From cf0233f193cc8694c202acf8d1261f6192fcd3c9 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 7 Jun 2026 07:59:31 +0200 Subject: [PATCH] refactor: architecture cleanup + fix scheduler Automation2Workflow error Fix: add missing Automation2Workflow/Automation2WorkflowRun imports to interfaceFeatureGraphicalEditor.py (caused scheduler crash on boot) Refactor: gdprDeletion via onUserDelete lifecycle hooks Refactor: i18nBootSync accounting labels via app.py parameter injection Refactor: serviceHub moved to serviceCenter/serviceHub.py Split: teamsbot/service.py, realEstate/main, routeTrustee, routeBilling Cleanup: remove obsolete methodTrustee, serviceExceptions shim Co-authored-by: Cursor --- app.py | 59 +- modules/datamodels/datamodelNavigation.py | 348 ++ .../serviceExceptions.py | 0 .../interfaceFeatureGraphicalEditor.py | 2 + .../neutralization/mainNeutralization.py | 24 + .../neutralization/neutralizePlayground.py | 2 +- .../features/realEstate/handlerRealEstate.py | 949 ++++++ modules/features/realEstate/mainRealEstate.py | 2900 +---------------- .../realEstate/routeFeatureRealEstate.py | 1369 +------- .../features/realEstate/serviceAiIntent.py | 1087 ++++++ modules/features/realEstate/serviceBzo.py | 725 +++++ .../features/realEstate/serviceGeometry.py | 817 +++++ modules/features/teamsbot/service.py | 2497 +------------- modules/features/teamsbot/serviceCommands.py | 305 ++ .../features/teamsbot/serviceConversation.py | 996 ++++++ modules/features/teamsbot/serviceWebSocket.py | 545 ++++ .../trustee/handlerTrusteeAccounting.py | 371 +++ modules/features/trustee/mainTrustee.py | 24 + .../features/trustee/routeFeatureTrustee.py | 451 +-- .../features/trustee/workflows/__init__.py | 3 + .../workflows}/methodTrustee/__init__.py | 0 .../methodTrustee/actions/extractFromFiles.py | 2 +- .../methodTrustee/actions/processDocuments.py | 0 .../methodTrustee/actions/queryData.py | 0 .../actions/refreshAccountingData.py | 0 .../methodTrustee/actions/syncToAccounting.py | 0 .../workflows}/methodTrustee/methodTrustee.py | 0 modules/interfaces/interfaceBootstrap.py | 10 +- modules/interfaces/interfaceDbApp.py | 4 +- modules/interfaces/interfaceDbSubscription.py | 2 +- modules/interfaces/interfaceFeatures.py | 2 +- modules/routes/billingWebhookHandler.py | 399 +++ modules/routes/routeBilling.py | 389 +-- modules/routes/routeClickup.py | 2 +- modules/routes/routeSharepoint.py | 2 +- modules/security/rootAccess.py | 44 +- .../core/serviceStreaming/eventManager.py | 224 +- modules/serviceCenter/serviceHub.py | 189 ++ .../serviceBilling/mainServiceBilling.py | 2 +- .../serviceGeneration/subDocumentUtility.py | 58 +- .../mainServiceSubscription.py | 2 +- modules/serviceHub/__init__.py | 196 +- modules/shared/documentUtils.py | 64 + modules/shared/eventManager.py | 167 + modules/shared/featureDiscovery.py | 59 + modules/system/gdprDeletion.py | 46 +- modules/system/i18nBootSync.py | 85 +- modules/system/mainSystem.py | 346 +- modules/system/registry.py | 52 +- .../workflows/automation2/executionEngine.py | 4 +- .../executors/actionNodeExecutor.py | 2 +- .../methods/methodAi/actions/consolidate.py | 2 +- .../methods/methodAi/actions/generateCode.py | 2 +- .../methodAi/actions/generateDocument.py | 2 +- .../methods/methodAi/actions/process.py | 2 +- .../methods/methodAi/actions/webResearch.py | 5 +- .../methodContext/actions/extractContent.py | 4 +- .../processing/shared/methodDiscovery.py | 103 +- modules/workflows/scheduler/mainScheduler.py | 55 +- 59 files changed, 7662 insertions(+), 8339 deletions(-) create mode 100644 modules/datamodels/datamodelNavigation.py rename modules/{shared => datamodels}/serviceExceptions.py (100%) create mode 100644 modules/features/realEstate/handlerRealEstate.py create mode 100644 modules/features/realEstate/serviceAiIntent.py create mode 100644 modules/features/realEstate/serviceBzo.py create mode 100644 modules/features/realEstate/serviceGeometry.py create mode 100644 modules/features/teamsbot/serviceCommands.py create mode 100644 modules/features/teamsbot/serviceConversation.py create mode 100644 modules/features/teamsbot/serviceWebSocket.py create mode 100644 modules/features/trustee/handlerTrusteeAccounting.py create mode 100644 modules/features/trustee/workflows/__init__.py rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/__init__.py (100%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/actions/extractFromFiles.py (99%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/actions/processDocuments.py (100%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/actions/queryData.py (100%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/actions/refreshAccountingData.py (100%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/actions/syncToAccounting.py (100%) rename modules/{workflows/methods => features/trustee/workflows}/methodTrustee/methodTrustee.py (100%) create mode 100644 modules/routes/billingWebhookHandler.py create mode 100644 modules/serviceCenter/serviceHub.py create mode 100644 modules/shared/documentUtils.py create mode 100644 modules/shared/eventManager.py create mode 100644 modules/shared/featureDiscovery.py diff --git a/app.py b/app.py index f43035c1..c91212e3 100644 --- a/app.py +++ b/app.py @@ -329,6 +329,14 @@ async def lifespan(app: FastAPI): catalogService = getCatalogService() registerAllFeaturesInCatalog(catalogService) logger.info("Feature catalog registration completed") + + # Register service center RBAC objects (Composition Root — avoids system→serviceCenter import) + try: + from modules.serviceCenter import registerServiceObjects + registerServiceObjects(catalogService) + except Exception as e: + logger.warning(f"Service center RBAC registration failed: {e}") + # Persist the in-memory feature registry into the Feature DB-table so # the FeatureInstance.featureCode FK has real targets. Without this # every FeatureInstance row would be flagged as orphan by the @@ -343,7 +351,22 @@ async def lifespan(app: FastAPI): # Sync gateway i18n registry to DB and load translation cache try: from modules.system.i18nBootSync import syncRegistryToDb, loadCache - await syncRegistryToDb() + from modules.serviceCenter.registry import IMPORTABLE_SERVICES + serviceLabels = [svc.get("label") for svc in IMPORTABLE_SERVICES.values()] + + accountingLabels = [] + try: + from modules.features.trustee.accounting.accountingRegistry import getAccountingRegistry + registry = getAccountingRegistry() + for connectorType, connector in (registry._connectors or {}).items(): + for field in connector.getRequiredConfigFields(): + label = getattr(field, "label", "") or "" + if label: + accountingLabels.append({"label": label, "connectorType": connectorType}) + except Exception: + pass + + await syncRegistryToDb(serviceLabels=serviceLabels, accountingLabels=accountingLabels) await loadCache() logger.info("i18n registry sync + cache load completed") except Exception as e: @@ -409,9 +432,41 @@ async def lifespan(app: FastAPI): try: main_loop = asyncio.get_running_loop() eventManager.set_event_loop(main_loop) - from modules.workflows.scheduler.mainScheduler import setMainLoop as setSchedulerMainLoop + from modules.workflows.scheduler.mainScheduler import setMainLoop as setSchedulerMainLoop, setOnRunFailedCallback setSchedulerMainLoop(main_loop) + # Inject run-failed notification callback (Composition Root — avoids workflows→serviceCenter import) + def _onRunFailed(workflowId, runId, error, mandateId=None, workflowLabel=None): + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelMessaging import MessagingEventParameters + + rootInterface = getRootInterface() + if not rootInterface: + return + eventUser = rootInterface.getUserByUsername("event") + if not eventUser: + return + ctx = ServiceCenterContext( + user=eventUser, + mandate_id=mandateId or "", + feature_instance_id="", + feature_code="graphicalEditor", + ) + messagingService = getService("messaging", ctx) + subscriptionId = "GraphicalEditorRunFailed" + eventParams = MessagingEventParameters(triggerData={ + "workflowId": workflowId, + "workflowLabel": workflowLabel or workflowId, + "runId": runId, + "error": error, + "mandateId": mandateId or "", + }) + messagingService.executeSubscription(subscriptionId, eventParams) + + setOnRunFailedCallback(_onRunFailed) + # Suppress noisy ConnectionResetError from ProactorEventLoop on Windows # when clients (browsers) close connections abruptly. This is a known # asyncio issue on Windows: https://bugs.python.org/issue39010 diff --git a/modules/datamodels/datamodelNavigation.py b/modules/datamodels/datamodelNavigation.py new file mode 100644 index 00000000..2fc278ef --- /dev/null +++ b/modules/datamodels/datamodelNavigation.py @@ -0,0 +1,348 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Navigation structure data (Layer L1 - datamodels). +Single source of truth for UI navigation sections used by RBAC and frontend. +""" + +from modules.shared.i18nRegistry import t + +# ============================================================================= +# Navigation Structure (Single Source of Truth) +# ============================================================================= +# +# Block Order (gemaess Navigation-API-Konzept): +# - System: 10 +# - : 15 (wird in routeSystem.py eingefuegt) +# - Basisdaten: 30 +# - Administration: 200 +# +# NOTE: Workflows and Migrate sections removed - now handled as features +# +# Item Order: Default-Abstand 10 pro Item +# uiComponent: Abgeleitet von objectKey (ui.system.home -> page.system.home) +# icon: Wird intern gehalten aber NICHT in der API Response zurueckgegeben + +NAVIGATION_SECTIONS = [ + # --- Meine Sicht (with top-level item + subgroups) --- + { + "id": "system", + "title": t("Meine Sicht"), + "order": 10, + "items": [ + { + "id": "home", + "objectKey": "ui.system.home", + "label": t("Start"), + "icon": "FaHome", + "path": "/", + "order": 10, + "public": True, + }, + ], + "subgroups": [ + { + "id": "system-overviews", + "title": t("Übersichten"), + "order": 15, + "items": [ + { + "id": "integrations", + "objectKey": "ui.system.integrations", + "label": t("Integrationen"), + "icon": "FaProjectDiagram", + "path": "/integrations", + "order": 10, + "public": True, + }, + { + "id": "compliance-audit", + "objectKey": "ui.system.complianceAudit", + "label": t("Compliance & Audit"), + "icon": "FaShieldAlt", + "path": "/compliance-audit", + "order": 20, + }, + ], + }, + { + "id": "system-basedata", + "title": t("Basisdaten"), + "order": 20, + "items": [ + { + "id": "connections", + "objectKey": "ui.system.connections", + "label": t("Verbindungen"), + "icon": "FaLink", + "path": "/basedata/connections", + "order": 10, + "public": True, + }, + { + "id": "files", + "objectKey": "ui.system.files", + "label": t("Dateien"), + "icon": "FaRegFileAlt", + "path": "/basedata/files", + "order": 20, + "public": True, + }, + { + "id": "prompts", + "objectKey": "ui.system.prompts", + "label": t("Prompts"), + "icon": "FaLightbulb", + "path": "/basedata/prompts", + "order": 30, + "public": True, + }, + ], + }, + { + "id": "system-usage", + "title": t("Nutzung"), + "order": 30, + "items": [ + { + "id": "billing-admin", + "objectKey": "ui.system.billingAdmin", + "label": t("Abrechnung"), + "icon": "FaMoneyBillAlt", + "path": "/billing/admin", + "order": 10, + }, + { + "id": "statistics", + "objectKey": "ui.system.statistics", + "label": t("Statistiken"), + "icon": "FaChartBar", + "path": "/billing/transactions", + "order": 20, + }, + { + "id": "automations", + "objectKey": "ui.system.automations", + "label": t("Automations"), + "icon": "FaRobot", + "path": "/automations", + "order": 30, + }, + { + "id": "rag-inventory", + "objectKey": "ui.system.ragInventory", + "label": t("RAG-Inventar"), + "icon": "FaDatabase", + "path": "/rag-inventory", + "order": 35, + }, + { + "id": "store", + "objectKey": "ui.system.store", + "label": t("Store"), + "icon": "FaStore", + "path": "/store", + "order": 40, + "public": True, + }, + { + "id": "settings", + "objectKey": "ui.system.settings", + "label": t("Einstellungen"), + "icon": "FaCog", + "path": "/settings", + "order": 50, + "public": True, + }, + ], + }, + ], + }, + # --- Administration (with subgroups) --- + { + "id": "admin", + "title": t("Administration"), + "order": 200, + "subgroups": [ + { + "id": "admin-wizards", + "title": t("Wizards"), + "order": 10, + "items": [ + { + "id": "admin-mandate-wizard", + "objectKey": "ui.admin.mandateWizard", + "label": t("Mandanten-Wizard"), + "icon": "FaMagic", + "path": "/admin/mandate-wizard", + "order": 10, + "adminOnly": True, + }, + { + "id": "admin-invitation-wizard", + "objectKey": "ui.admin.invitationWizard", + "label": t("Einladungs-Wizard"), + "icon": "FaEnvelopeOpenText", + "path": "/admin/invitation-wizard", + "order": 20, + "adminOnly": True, + }, + ], + }, + { + "id": "admin-users-group", + "title": t("Benutzer"), + "order": 20, + "items": [ + { + "id": "admin-users", + "objectKey": "ui.admin.users", + "label": t("Benutzer"), + "icon": "FaUsers", + "path": "/admin/users", + "order": 10, + "adminOnly": True, + }, + { + "id": "admin-invitations", + "objectKey": "ui.admin.invitations", + "label": t("Benutzer-Einladungen"), + "icon": "FaEnvelopeOpenText", + "path": "/admin/invitations", + "order": 20, + "adminOnly": True, + }, + { + "id": "admin-user-access-overview", + "objectKey": "ui.admin.userAccessOverview", + "label": t("Benutzer-Zugriffsübersicht"), + "icon": "FaClipboardList", + "path": "/admin/user-access-overview", + "order": 30, + "adminOnly": True, + }, + { + "id": "admin-subscriptions", + "objectKey": "ui.admin.subscriptions", + "label": t("Abonnements"), + "icon": "FaFileContract", + "path": "/admin/subscriptions", + "order": 40, + "adminOnly": True, + }, + ], + }, + { + "id": "admin-system-group", + "title": t("System"), + "order": 30, + "items": [ + { + "id": "admin-roles", + "objectKey": "ui.admin.roles", + "label": t("Rollen"), + "icon": "FaUserTag", + "path": "/admin/mandate-roles", + "order": 10, + "adminOnly": True, + }, + { + "id": "admin-mandate-role-permissions", + "objectKey": "ui.admin.mandateRolePermissions", + "label": t("Rollen-Berechtigungen"), + "icon": "FaKey", + "path": "/admin/mandate-role-permissions", + "order": 20, + "adminOnly": True, + }, + { + "id": "admin-mandates", + "objectKey": "ui.admin.mandates", + "label": t("Mandanten"), + "icon": "FaBuilding", + "path": "/admin/mandates", + "order": 30, + "adminOnly": True, + }, + { + "id": "admin-user-mandates", + "objectKey": "ui.admin.userMandates", + "label": t("Mandanten-Mitglieder"), + "icon": "FaUserFriends", + "path": "/admin/user-mandates", + "order": 40, + "adminOnly": True, + }, + { + "id": "admin-access", + "objectKey": "ui.admin.access", + "label": t("Zugriffsverwaltung"), + "icon": "FaBuilding", + "path": "/admin/access", + "order": 50, + "adminOnly": True, + }, + { + "id": "admin-feature-instances", + "objectKey": "ui.admin.featureInstances", + "label": t("Feature-Instanzen"), + "icon": "FaCubes", + "path": "/admin/feature-instances", + "order": 60, + "adminOnly": True, + }, + { + "id": "admin-feature-roles", + "objectKey": "ui.admin.featureRoles", + "label": t("Features Rollen-Vorlagen"), + "icon": "FaShieldAlt", + "path": "/admin/feature-roles", + "order": 70, + "adminOnly": True, + "sysAdminOnly": True, + }, + { + "id": "admin-logs", + "objectKey": "ui.admin.logs", + "label": t("Logs"), + "icon": "FaFileAlt", + "path": "/admin/logs", + "order": 90, + "adminOnly": True, + "sysAdminOnly": True, + }, + { + "id": "admin-languages", + "objectKey": "ui.admin.languages", + "label": t("UI-Sprachen"), + "icon": "FaGlobe", + "path": "/admin/languages", + "order": 95, + "adminOnly": True, + "sysAdminOnly": True, + }, + { + "id": "admin-database-health", + "objectKey": "ui.admin.databaseHealth", + "label": t("Datenbank-Gesundheit"), + "icon": "FaDatabase", + "path": "/admin/database-health", + "order": 98, + "adminOnly": True, + "sysAdminOnly": True, + }, + { + "id": "admin-demo-config", + "objectKey": "ui.admin.demoConfig", + "label": t("Demo Config"), + "icon": "FaCubes", + "path": "/admin/demo-config", + "order": 100, + "adminOnly": True, + "sysAdminOnly": True, + }, + ], + }, + ], + }, +] diff --git a/modules/shared/serviceExceptions.py b/modules/datamodels/serviceExceptions.py similarity index 100% rename from modules/shared/serviceExceptions.py rename to modules/datamodels/serviceExceptions.py diff --git a/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py b/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py index e58c7b18..aacc9e45 100644 --- a/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py +++ b/modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py @@ -46,6 +46,8 @@ from modules.datamodels.datamodelWorkflowAutomation import ( AutoRun, AutoStepLog, AutoTask, + Automation2Workflow, + Automation2WorkflowRun, ) from modules.features.graphicalEditor.entryPoints import invocations_synced_with_graph from modules.connectors.connectorDbPostgre import DatabaseConnector diff --git a/modules/features/neutralization/mainNeutralization.py b/modules/features/neutralization/mainNeutralization.py index 42f74cf3..2a8a6e19 100644 --- a/modules/features/neutralization/mainNeutralization.py +++ b/modules/features/neutralization/mainNeutralization.py @@ -279,3 +279,27 @@ def onMandateDelete(mandateId: str, instances: list) -> None: except Exception as e: logger.warning(f"Failed to cascade-delete neutralization data for mandate {mandateId}: {e}") + +def onUserDelete(userId: str, currentUser) -> dict: + """Delete/anonymize user data from the neutralization database (GDPR).""" + from modules.system.gdprDeletion import deleteUserDataFromDatabase + from modules.connectors.connectorDbPostgre import DatabaseConnector + from modules.shared.configuration import APP_CONFIG + + dbName = "poweron_neutralization" + try: + db = DatabaseConnector( + dbHost=APP_CONFIG.get("DB_HOST", "localhost"), + dbDatabase=dbName, + dbUser=APP_CONFIG.get("DB_USER"), + dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"), + dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), + userId=None, + ) + stats = deleteUserDataFromDatabase(db, userId, dbName) + db.close() + return stats + except Exception as e: + logger.warning(f"onUserDelete neutralization failed: {e}") + return {"database": dbName, "tablesProcessed": 0, "recordsDeleted": 0, "recordsAnonymized": 0, "errors": [str(e)]} + diff --git a/modules/features/neutralization/neutralizePlayground.py b/modules/features/neutralization/neutralizePlayground.py index eab0bdeb..0bd50b49 100644 --- a/modules/features/neutralization/neutralizePlayground.py +++ b/modules/features/neutralization/neutralizePlayground.py @@ -9,7 +9,7 @@ from urllib.parse import urlparse, unquote from modules.datamodels.datamodelUam import User from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig, DataNeutralizationSnapshot from .interfaceFeatureNeutralizer import getInterface as _getNeutralizerInterface -from modules.serviceHub import getInterface as getServices +from modules.serviceCenter.serviceHub import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/features/realEstate/handlerRealEstate.py b/modules/features/realEstate/handlerRealEstate.py new file mode 100644 index 00000000..e08ff6aa --- /dev/null +++ b/modules/features/realEstate/handlerRealEstate.py @@ -0,0 +1,949 @@ +""" +Handler functions for Real Estate feature routes. +Contains extracted business logic from route handlers. +""" + +import json +import logging +import re +import aiohttp +from typing import Optional, Dict, Any, List + +from fastapi import HTTPException, status + +from modules.datamodels.datamodelPagination import ( + PaginationParams, + PaginatedResponse, + PaginationMetadata, +) +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface +from .datamodelFeatureRealEstate import ( + Projekt, + Parzelle, + Dokument, + DokumentTyp, + Gemeinde, + Kanton, + Land, + Kontext, + StatusProzess, +) +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface +from .mainRealEstate import ( + create_project_with_parcel_data, + extract_bzo_information, +) +from .parcelSelectionService import is_parcel_adjacent_to_selection + +from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector +from modules.connectors.connectorOerebWfs import OerebWfsConnector +from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface +from modules.shared.i18nRegistry import apiRouteContext + +routeApiMsg = apiRouteContext("routeFeatureRealEstate") +logger = logging.getLogger(__name__) + + +# ============================================================================ +# GEMEINDE / BZO HANDLERS +# ============================================================================ + +async def processGemeindenSync(interface, instanceId: str, mandateId: str, onlyCurrent: bool = True) -> Dict[str, Any]: + """ + Fetch all Gemeinden from Swiss Topo and save to DB for an instance. + Creates Kantone as needed. + """ + try: + oerebConnector = OerebWfsConnector() + connector = SwissTopoMapServerConnector(oereb_connector=oerebConnector) + gemeindenData = await connector.get_all_gemeinden(only_current=onlyCurrent) + except Exception as e: + logger.error(f"Error fetching Gemeinden from Swiss Topo: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Error fetching Gemeinden: {str(e)}") + + gemeindenCreated = 0 + gemeindenSkipped = 0 + kantoneCreated = 0 + errors: List[str] = [] + kantonCache: Dict[str, str] = {} + + def _findGemeindeByBfsNummer(bfsNummer: str) -> Optional[Any]: + try: + gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId}) + for g in gemeinden: + for k in (g.kontextInformationen or []): + try: + data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt + if isinstance(data, dict) and str(data.get("bfs_nummer")) == str(bfsNummer): + return g + except (json.JSONDecodeError, AttributeError): + continue + except Exception as ex: + logger.error(f"Error finding Gemeinde by BFS {bfsNummer}: {ex}", exc_info=True) + return None + + def _getOrCreateKanton(kantonAbk: str) -> Optional[str]: + nonlocal kantoneCreated, errors + if not kantonAbk: + return None + if kantonAbk in kantonCache: + return kantonCache[kantonAbk] + kantone = interface.getKantone(recordFilter={"mandateId": mandateId, "abk": kantonAbk}) + if kantone: + kantonCache[kantonAbk] = kantone[0].id + return kantone[0].id + kantonNames = { + "AG": "Aargau", "AI": "Appenzell Innerrhoden", "AR": "Appenzell Ausserrhoden", + "BE": "Bern", "BL": "Basel-Landschaft", "BS": "Basel-Stadt", + "FR": "Freiburg", "GE": "Genf", "GL": "Glarus", "GR": "Graubünden", + "JU": "Jura", "LU": "Luzern", "NE": "Neuenburg", "NW": "Nidwalden", + "OW": "Obwalden", "SG": "St. Gallen", "SH": "Schaffhausen", "SO": "Solothurn", + "SZ": "Schwyz", "TG": "Thurgau", "TI": "Tessin", "UR": "Uri", + "VD": "Waadt", "VS": "Wallis", "ZG": "Zug", "ZH": "Zürich", + } + try: + kantonLabel = kantonNames.get(kantonAbk, kantonAbk) + kanton = Kanton( + mandateId=mandateId, + featureInstanceId=instanceId, + label=kantonLabel, + abk=kantonAbk, + ) + created = interface.createKanton(kanton) + if created and created.id: + kantonCache[kantonAbk] = created.id + kantoneCreated += 1 + return created.id + except Exception as ex: + errors.append(f"Error creating Kanton {kantonAbk}: {ex}") + return None + + savedGemeinden: List[Dict[str, Any]] = [] + for gd in gemeindenData: + try: + gemeindeName = gd.get("name") + bfsNummer = gd.get("bfs_nummer") + kantonAbk = gd.get("kanton") + if not gemeindeName or bfsNummer is None: + gemeindenSkipped += 1 + continue + existing = _findGemeindeByBfsNummer(str(bfsNummer)) + if existing: + gemeindenSkipped += 1 + savedGemeinden.append(existing.model_dump() if hasattr(existing, "model_dump") else existing) + continue + kantonId = _getOrCreateKanton(kantonAbk) if kantonAbk else None + gemeinde = Gemeinde( + mandateId=mandateId, + featureInstanceId=instanceId, + label=gemeindeName, + id_kanton=kantonId, + kontextInformationen=[ + Kontext(thema="BFS Nummer", inhalt=json.dumps({"bfs_nummer": bfsNummer}, ensure_ascii=False)) + ], + ) + created = interface.createGemeinde(gemeinde) + if created and created.id: + gemeindenCreated += 1 + savedGemeinden.append(created.model_dump() if hasattr(created, "model_dump") else created) + else: + errors.append(f"Failed to create Gemeinde {gemeindeName}") + gemeindenSkipped += 1 + except Exception as ex: + errors.append(f"Error processing {gd.get('name', 'Unknown')}: {str(ex)}") + gemeindenSkipped += 1 + + return { + "gemeinden": savedGemeinden, + "count": len(savedGemeinden), + "stats": { + "gemeinden_created": gemeindenCreated, + "gemeinden_skipped": gemeindenSkipped, + "kantone_created": kantoneCreated, + "error_count": len(errors), + "errors": errors[:10], + }, + } + + +async def processBzoDocumentsFetch(interface, componentInterface, mandateId: str, instanceId: str) -> Dict[str, Any]: + """Search for and download BZO documents for all Gemeinden of an instance.""" + from modules.features.realEstate.realEstateGemeindeService import fetch_bzo_for_gemeinde + + gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId}) + stats = {"gemeinden_processed": 0, "documents_created": 0, "documents_skipped": 0, "errors": []} + results: List[Dict[str, Any]] = [] + + for gemeinde in gemeinden: + gr = {"gemeinde_id": gemeinde.id, "gemeinde_label": gemeinde.label, "status": None, "dokument_ids": [], "error": None} + try: + stats["gemeinden_processed"] += 1 + fetched = await fetch_bzo_for_gemeinde( + interface, componentInterface, gemeinde, mandateId, instanceId + ) + if fetched: + gr["status"] = "created" + stats["documents_created"] += 1 + refreshed = interface.getGemeinde(gemeinde.id) + if refreshed and refreshed.dokumente: + for doc in refreshed.dokumente: + docId = getattr(doc, "id", None) or (doc.get("id") if isinstance(doc, dict) else None) + if docId: + gr["dokument_ids"].append(docId) + else: + gr["status"] = "skipped" + stats["documents_skipped"] += 1 + except Exception as ex: + gr["status"] = "error" + gr["error"] = str(ex) + stats["errors"].append(f"{gemeinde.label}: {str(ex)}") + results.append(gr) + + return {"success": True, "stats": stats, "results": results} + + +async def processParcelDocuments(interface, componentInterface, gemeindeName: str, bauzone: str, mandateId: str, instanceId: str) -> Dict[str, Any]: + """ + Ensure BZO document exists for Gemeinde, return documents for parcel info display. + Creates Gemeinde (Swiss Topo) and BZO (Tavily) if not in DB. + """ + from modules.features.realEstate.realEstateGemeindeService import ( + ensure_single_gemeinde, + fetch_bzo_for_gemeinde, + ) + + gemeindeObj = None + byLabel = interface.getGemeinden(recordFilter={"label": gemeindeName, "mandateId": mandateId}) + gemeindeObj = byLabel[0] if byLabel else None + + if not gemeindeObj: + allG = interface.getGemeinden(recordFilter={"mandateId": mandateId}) + gNorm = gemeindeName.strip().lower() + for g in allG: + gl = (g.label or "").strip().lower() + if gl == gNorm or gNorm in gl or gl in gNorm: + gemeindeObj = g + logger.debug(f"parcel-documents: Found Gemeinde by label match '{gemeindeName}' -> '{g.label}'") + break + + if gemeindeObj: + logger.debug(f"parcel-documents: Gemeinde '{gemeindeName}' resolved: {gemeindeObj.id}") + + if not gemeindeObj: + logger.info(f"parcel-documents: No Gemeinde for label '{gemeindeName}', ensuring via Swiss Topo...") + gemeindeObj = await ensure_single_gemeinde(interface, mandateId, instanceId, gemeinde_name=gemeindeName) + + if not gemeindeObj: + logger.warning(f"parcel-documents: Gemeinde '{gemeindeName}' nicht gefunden (mandateId={mandateId[:8]}...)") + return {"documents": [], "error": f"Gemeinde '{gemeindeName}' nicht gefunden"} + + bzoDocs = [] + if gemeindeObj.dokumente: + for doc in gemeindeObj.dokumente: + typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None) + if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] or str(typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision"]: + docId = doc.id if hasattr(doc, "id") else doc.get("id") + if docId: + full = interface.getDokument(docId) + if full and full.dokumentReferenz: + bzoDocs.append(full) + + if not bzoDocs: + logger.info(f"parcel-documents: No BZO for {gemeindeName}, fetching...") + fetched = await fetch_bzo_for_gemeinde(interface, componentInterface, gemeindeObj, mandateId, instanceId) + if fetched: + gemeindeObj = interface.getGemeinde(gemeindeObj.id) + if gemeindeObj and gemeindeObj.dokumente: + for doc in gemeindeObj.dokumente: + typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None) + if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION]: + docId = doc.id if hasattr(doc, "id") else doc.get("id") + if docId: + full = interface.getDokument(docId) + if full and full.dokumentReferenz: + bzoDocs.append(full) + + result = [] + for d in bzoDocs: + result.append({ + "id": d.id, + "label": d.label, + "fileId": d.dokumentReferenz, + "fileName": (d.label or "BZO") + ".pdf", + "mimeType": d.mimeType or "application/pdf", + }) + return {"documents": result, "gemeinde": gemeindeName, "bauzone": bauzone} + + +# ============================================================================ +# LEGACY TABLE HANDLERS +# ============================================================================ + +def processTableData(user, mandateId: Optional[str], table: str, pagination: Optional[str]) -> PaginatedResponse: + """Fetch and paginate table data for a real estate entity table.""" + tableMapping = { + "Projekt": (Projekt, "getProjekte"), + "Parzelle": (Parzelle, "getParzellen"), + "Dokument": (Dokument, "getDokumente"), + "Gemeinde": (Gemeinde, "getGemeinden"), + "Kanton": (Kanton, "getKantone"), + "Land": (Land, "getLaender"), + } + + if table not in tableMapping: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid table name '{table}'. Available tables: {', '.join(tableMapping.keys())}" + ) + + realEstateInterface = getRealEstateInterface(user, mandateId=mandateId) + modelClass, methodName = tableMapping[table] + getterMethod = getattr(realEstateInterface, methodName) + items = getterMethod(recordFilter=None) + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + paginationParams = PaginationParams(**paginationDict) if paginationDict else None + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid pagination parameter: {str(e)}" + ) + + if paginationParams: + if paginationParams.sort: + for sortField in reversed(paginationParams.sort): + fieldName = sortField.field + direction = sortField.direction.lower() + + def _sortKey(item, _fieldName=fieldName): + value = getattr(item, _fieldName, None) + if value is None: + return (1, None) + return (0, value) + + items.sort(key=_sortKey, reverse=(direction == "desc")) + + totalItems = len(items) + totalPages = (totalItems + paginationParams.pageSize - 1) // paginationParams.pageSize + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + paginatedItems = items[startIdx:endIdx] + + return PaginatedResponse( + items=paginatedItems, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + + return PaginatedResponse(items=items, pagination=None) + + +async def processCreateTableRecord(user, mandateId: Optional[str], table: str, data: Dict[str, Any]) -> Dict[str, Any]: + """Create a record in a real estate table, with special handling for Projekt+parcel.""" + if table == "Projekt" and ("parzelle" in data or "parzellen" in data): + logger.info(f"Creating Projekt with parcel data for user {user.id}") + + label = data.get("label") + if not label: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=routeApiMsg("label is required") + ) + + statusProzess = data.get("statusProzess", "Eingang") + + parzellenData = [] + if "parzellen" in data: + parzellenData = data.get("parzellen", []) + if not isinstance(parzellenData, list): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=routeApiMsg("parzellen must be an array") + ) + elif "parzelle" in data: + parzelleData = data.get("parzelle") + if parzelleData: + parzellenData = [parzelleData] + + if not parzellenData: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=routeApiMsg("parzelle or parzellen data is required") + ) + + result = await create_project_with_parcel_data( + currentUser=user, + mandateId=mandateId, + projekt_label=label, + parzellen_data=parzellenData, + status_prozess=statusProzess, + ) + return result.get("projekt", {}) + + tableMapping = { + "Projekt": (Projekt, "createProjekt"), + "Parzelle": (Parzelle, "createParzelle"), + "Dokument": (Dokument, "createDokument"), + "Gemeinde": (Gemeinde, "createGemeinde"), + "Kanton": (Kanton, "createKanton"), + "Land": (Land, "createLand"), + } + + if table not in tableMapping: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid table name '{table}'. Available tables: {', '.join(tableMapping.keys())}" + ) + + realEstateInterface = getRealEstateInterface(user, mandateId=mandateId) + modelClass, methodName = tableMapping[table] + createMethod = getattr(realEstateInterface, methodName) + + if "mandateId" not in data: + data["mandateId"] = mandateId + + try: + modelInstance = modelClass(**data) + except Exception as e: + logger.error(f"Error creating {table} model instance: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid data for {table}: {str(e)}" + ) + + createdRecord = createMethod(modelInstance) + if hasattr(createdRecord, 'model_dump'): + return createdRecord.model_dump() + return createdRecord + + +# ============================================================================ +# PARCEL SEARCH HANDLER +# ============================================================================ + +async def processParcelSearch(user, mandateId: Optional[str], location: str, includeBauzone: bool, includeAdjacent: bool) -> Dict[str, Any]: + """ + Search for parcel information by address or coordinates. + Resolves address, calculates geometry, optionally fetches adjacent parcels and bauzone. + """ + connector = SwissTopoMapServerConnector() + parcelData = await connector.search_parcel(location) + + if not parcelData: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"No parcel found for location: {location}" + ) + + extractedAttributes = connector.extract_parcel_attributes(parcelData) + attributes = parcelData.get("attributes", {}) + geometry = parcelData.get("geometry", {}) + + areaM2 = None + centroid = None + if extractedAttributes.get("perimeter"): + perimeter = extractedAttributes["perimeter"] + points = perimeter.get("punkte", []) + if len(points) >= 3: + area = 0 + for i in range(len(points)): + j = (i + 1) % len(points) + area += points[i]["x"] * points[j]["y"] + area -= points[j]["x"] * points[i]["y"] + areaM2 = abs(area / 2) + sumX = sum(p["x"] for p in points) + sumY = sum(p["y"] for p in points) + centroid = {"x": sumX / len(points), "y": sumY / len(points)} + + canton = attributes.get("ak", "") + municipalityName = None + fullAddress = None + plz = None + + geocodedAddress = parcelData.get('geocoded_address') + if geocodedAddress: + fullAddress = geocodedAddress.get('full_address') + plz = geocodedAddress.get('plz') + municipalityName = geocodedAddress.get('municipality') + logger.debug(f"Using geocoded address: {fullAddress}") + + queryCoords = parcelData.get('query_coordinates') + addressQueryCoords = queryCoords if queryCoords else centroid + + if not fullAddress and addressQueryCoords: + queryX = addressQueryCoords['x'] + queryY = addressQueryCoords['y'] + logger.debug(f"Querying address layer at query coordinates: ({queryX}, {queryY})") + + isCoordinateSearch = ',' in location and not any(c.isalpha() for c in location.split(',')[0]) + buildingTolerance = 1 if isCoordinateSearch else 10 + buildingResult = await connector._query_building_layer(queryX, queryY, tolerance=buildingTolerance, buffer=25) + + if buildingResult: + addrAttrs = buildingResult.get("attributes", {}) + logger.debug(f"Address layer attributes: {addrAttrs}") + addressInfo = connector._extract_address_from_building_attrs(addrAttrs) + fullAddress = addressInfo.get('full_address') + plz = addressInfo.get('plz') + municipalityName = addressInfo.get('municipality') + if fullAddress: + logger.debug(f"Constructed address: {fullAddress}") + + if not fullAddress: + if location and any(c.isalpha() for c in location) and "CH" not in location: + fullAddress = location + logger.debug(f"Using location as address: {fullAddress}") + + if not municipalityName and fullAddress: + plzMunicipalityMatch = re.search(r"\b(\d{4})\s+([A-ZÄÖÜ][a-zäöüß\s-]+)", fullAddress) + if plzMunicipalityMatch: + extractedMunicipality = plzMunicipalityMatch.group(2).strip() + extractedMunicipality = re.sub(r"[,;\.]+$", "", extractedMunicipality).strip() + if extractedMunicipality: + municipalityName = extractedMunicipality + if not plz: + plz = plzMunicipalityMatch.group(1) + logger.debug(f"Extracted municipality from address: {municipalityName}") + + bfsnr = attributes.get("bfsnr") + if not municipalityName and bfsnr and canton and mandateId: + try: + interface = getRealEstateInterface(user, mandateId=mandateId, featureInstanceId=None) + gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId}) + for g in gemeinden: + for k in (g.kontextInformationen or []): + try: + data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt + if isinstance(data, dict): + bfs = data.get("bfs_nummer") or data.get("bfsnr") or data.get("municipality_code") + if str(bfs) == str(bfsnr): + municipalityName = g.label + logger.debug(f"Found Gemeinde by BFS {bfsnr} in DB: {municipalityName}") + break + except (json.JSONDecodeError, AttributeError): + continue + if municipalityName: + break + except Exception as e: + logger.debug(f"Error querying Gemeinde by BFS: {e}") + + if not municipalityName and centroid and canton: + try: + geocodeUrl = "https://api3.geo.admin.ch/rest/services/api/MapServer/identify" + params = { + "geometry": f"{centroid['x']},{centroid['y']}", + "geometryType": "esriGeometryPoint", + "layers": "all:ch.swisstopo.swissboundaries3d-gemeinde-flaeche.fill", + "tolerance": "0", + "returnGeometry": "false", + "sr": "2056", + "f": "json", + } + async with aiohttp.ClientSession() as session: + async with session.get(geocodeUrl, params=params) as resp: + if resp.status == 200: + data = await resp.json() + results = data.get("results", []) + if results: + attrs = results[0].get("attributes", {}) + geoName = attrs.get("name") or attrs.get("gemeindename") or attrs.get("label") + if geoName: + municipalityName = connector._clean_municipality_name(str(geoName)) + logger.debug(f"Found municipality via Swiss Topo geocoding: {municipalityName}") + except Exception as e: + logger.debug(f"Error querying Swiss Topo geocoding: {e}") + + if not municipalityName and bfsnr: + commonMunicipalities = { + 261: "Zürich", 198: "Pfäffikon", 191: "Uster", 3203: "Winterthur", + 351: "Bern", 2701: "Basel", 6621: "Genève", 5586: "Lausanne", + 1061: "Luzern", 230: "St. Gallen", 5192: "Lugano", 1367: "Schwyz", + } + if bfsnr in commonMunicipalities: + municipalityName = commonMunicipalities[bfsnr] + logger.debug(f"Looked up municipality from common list: {municipalityName}") + elif canton and bfsnr: + municipalityName = f"{canton}-{bfsnr}" + logger.debug(f"Using fallback municipality: {municipalityName}") + + if fullAddress and fullAddress.startswith("CH") and len(fullAddress) == 14 and fullAddress[2:].isdigit(): + fullAddress = None + logger.debug("Removed EGRID from address field") + + bauzone = None + hasGeometry = geometry and (geometry.get("rings") or geometry.get("coordinates")) + if includeBauzone and canton and hasGeometry and centroid: + try: + logger.debug(f"Querying zone information for parcel {attributes.get('label')} in canton {canton}") + oerebConnector = OerebWfsConnector() + zoneResults = await oerebConnector.query_zone_layer( + egrid=attributes.get("egris_egrid", "") or "", + x=centroid["x"], + y=centroid["y"], + canton=canton, + geometry=geometry, + ) + if zoneResults and len(zoneResults) > 0: + zoneAttrs = zoneResults[0].get("attributes", {}) + typGdeAbkuerzung = zoneAttrs.get("typ_gde_abkuerzung") + if typGdeAbkuerzung: + bauzone = typGdeAbkuerzung + logger.debug(f"Found bauzone: {bauzone} for parcel {attributes.get('label')}") + except Exception as e: + logger.warning(f"Error querying zone information: {e}", exc_info=True) + + parcelInfo = { + "id": attributes.get("label") or attributes.get("number"), + "egrid": attributes.get("egris_egrid"), + "number": attributes.get("number"), + "name": attributes.get("name"), + "identnd": attributes.get("identnd"), + "canton": attributes.get("ak"), + "municipality_code": attributes.get("bfsnr"), + "municipality_name": municipalityName, + "address": fullAddress, + "plz": plz, + "perimeter": extractedAttributes.get("perimeter"), + "area_m2": areaM2, + "centroid": centroid, + "geoportal_url": attributes.get("geoportal_url"), + "realestate_type": attributes.get("realestate_type"), + "bauzone": bauzone, + } + + bbox = parcelData.get("bbox", []) + mapView = { + "center": centroid, + "zoom_bounds": { + "min_x": bbox[0] if len(bbox) >= 4 else None, + "min_y": bbox[1] if len(bbox) >= 4 else None, + "max_x": bbox[2] if len(bbox) >= 4 else None, + "max_y": bbox[3] if len(bbox) >= 4 else None + }, + "geometry_geojson": { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [[p["x"], p["y"]] for p in extractedAttributes["perimeter"]["punkte"]] + ] if extractedAttributes.get("perimeter") else [] + }, + "properties": { + "id": parcelInfo["id"], + "egrid": parcelInfo["egrid"], + "number": parcelInfo["number"] + } + } + } + + responseData = { + "parcel": parcelInfo, + "map_view": mapView + } + + if includeAdjacent and parcelData and parcelData.get("geometry"): + try: + selectedParcelId = parcelInfo["id"] + adjacentParcelsRaw = await connector.find_neighboring_parcels( + parcel_data=parcelData, + selected_parcel_id=selectedParcelId, + sample_distance=20.0, + max_sample_points=30, + max_neighbors=15, + max_concurrent=50, + ) + adjacentParcels = [_convertParcelGeometry(adjParcel) for adjParcel in adjacentParcelsRaw] + responseData["adjacent_parcels"] = adjacentParcels + logger.info(f"Found {len(adjacentParcels)} neighboring parcels for parcel {selectedParcelId}") + except Exception as e: + logger.warning(f"Error fetching adjacent parcels: {e}", exc_info=True) + responseData["adjacent_parcels"] = [] + + return responseData + + +def _convertParcelGeometry(adjParcel: Dict[str, Any]) -> Dict[str, Any]: + """Convert an adjacent parcel to include GeoJSON geometry.""" + adjParcelWithGeo = { + "id": adjParcel["id"], + "egrid": adjParcel.get("egrid"), + "number": adjParcel.get("number"), + "perimeter": adjParcel.get("perimeter") + } + + adjGeometry = adjParcel.get("geometry") + adjPerimeter = adjParcel.get("perimeter") + + if adjGeometry: + if "rings" in adjGeometry and adjGeometry["rings"]: + ring = adjGeometry["rings"][0] + coordinates = [[[p[0], p[1]] for p in ring]] + adjParcelWithGeo["geometry_geojson"] = { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": coordinates}, + "properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")} + } + elif adjGeometry.get("type") == "Polygon": + adjParcelWithGeo["geometry_geojson"] = { + "type": "Feature", + "geometry": adjGeometry, + "properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")} + } + + if "geometry_geojson" not in adjParcelWithGeo and adjPerimeter and adjPerimeter.get("punkte"): + punkte = adjPerimeter["punkte"] + coordinates = [[[p["x"], p["y"]] for p in punkte]] + adjParcelWithGeo["geometry_geojson"] = { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": coordinates}, + "properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")} + } + + return adjParcelWithGeo + + +# ============================================================================ +# ADD ADJACENT PARCEL HANDLER +# ============================================================================ + +async def processAddAdjacentParcel(location: Dict[str, Any], selectedParcels: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Add an adjacent parcel to the selection. Validates adjacency. + Returns parcel response with geometry. + """ + locStr = f"{location['x']},{location['y']}" + connector = SwissTopoMapServerConnector() + parcelData = await connector.search_parcel(locStr) + + if not parcelData: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=routeApiMsg("No parcel found at this location") + ) + + extracted = connector.extract_parcel_attributes(parcelData) + attributes = parcelData.get("attributes", {}) + geometry = parcelData.get("geometry", {}) + + areaM2 = None + centroid = None + if extracted.get("perimeter"): + perimeter = extracted["perimeter"] + points = perimeter.get("punkte", []) + if len(points) >= 3: + area = 0 + for i in range(len(points)): + j = (i + 1) % len(points) + area += points[i]["x"] * points[j]["y"] + area -= points[j]["x"] * points[i]["y"] + areaM2 = abs(area / 2) + sumX = sum(p["x"] for p in points) + sumY = sum(p["y"] for p in points) + centroid = {"x": sumX / len(points), "y": sumY / len(points)} + + parcelInfo = { + "id": attributes.get("label") or attributes.get("number"), + "egrid": attributes.get("egris_egrid"), + "number": attributes.get("number"), + "name": attributes.get("name"), + "identnd": attributes.get("identnd"), + "canton": attributes.get("ak"), + "municipality_code": attributes.get("bfsnr"), + "municipality_name": None, + "address": None, + "plz": None, + "perimeter": extracted.get("perimeter"), + "area_m2": areaM2, + "centroid": centroid, + "geoportal_url": attributes.get("geoportal_url"), + "realestate_type": attributes.get("realestate_type"), + "bauzone": None, + } + + mapView = { + "center": centroid, + "zoom_bounds": parcelData.get("bbox", []) and { + "min_x": parcelData["bbox"][0], + "min_y": parcelData["bbox"][1], + "max_x": parcelData["bbox"][2], + "max_y": parcelData["bbox"][3], + } or None, + "geometry_geojson": _buildGeometryGeojson(extracted, parcelInfo), + } + + newParcelResponse = {"parcel": parcelInfo, "map_view": mapView} + + if not is_parcel_adjacent_to_selection(newParcelResponse, selectedParcels): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=routeApiMsg("Nur angrenzende Parzellen können hinzugefügt werden") + ) + + bbox = parcelData.get("bbox", []) + mapView["zoom_bounds"] = { + "min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3] + } if len(bbox) >= 4 else None + + geocodedAddress = parcelData.get("geocoded_address") + if geocodedAddress: + parcelInfo["municipality_name"] = geocodedAddress.get("municipality") + parcelInfo["address"] = geocodedAddress.get("full_address") + parcelInfo["plz"] = geocodedAddress.get("plz") + + if centroid and attributes.get("ak"): + try: + oereb = OerebWfsConnector() + zoneResults = await oereb.query_zone_layer( + egrid=attributes.get("egris_egrid", "") or "", + x=centroid["x"], y=centroid["y"], + canton=attributes.get("ak"), + geometry=geometry, + ) + if zoneResults and len(zoneResults) > 0: + parcelInfo["bauzone"] = zoneResults[0].get("attributes", {}).get("typ_gde_abkuerzung") + except Exception as oe: + logger.debug(f"ÖREB zone query failed: {oe}") + + return newParcelResponse + + +def _buildGeometryGeojson(extracted: Dict[str, Any], parcelInfo: Dict[str, Any]) -> Dict[str, Any]: + """Build geometry_geojson from extracted perimeter.""" + coords = [] + if extracted.get("perimeter", {}).get("punkte"): + coords = [[[p["x"], p["y"]] for p in extracted["perimeter"]["punkte"]]] + return { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": coords}, + "properties": {"id": parcelInfo["id"], "egrid": parcelInfo["egrid"], "number": parcelInfo["number"]}, + } + + +# ============================================================================ +# ADD PARCEL TO PROJECT HANDLER +# ============================================================================ + +async def processAddParcelToProject(user, mandateId: Optional[str], projektId: str, body: Dict[str, Any]) -> Dict[str, Any]: + """ + Add a parcel to an existing project. + Supports linking existing, creating from location, or creating from custom data. + """ + realEstateInterface = getRealEstateInterface(user, mandateId=mandateId) + + recordFilter = {"id": projektId} + if mandateId: + recordFilter["mandateId"] = mandateId + projekte = realEstateInterface.getProjekte(recordFilter=recordFilter) + if not projekte: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Projekt {projektId} not found" + ) + projekt = projekte[0] + + parcelId = body.get("parcelId") + location = body.get("location") + parcelDataDict = body.get("parcelData") + parzelle = None + + if parcelId: + logger.info(f"Linking existing parcel {parcelId}") + parcelFilter = {"id": parcelId} + if mandateId: + parcelFilter["mandateId"] = mandateId + parcels = realEstateInterface.getParzellen(recordFilter=parcelFilter) + if not parcels: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Parzelle {parcelId} not found" + ) + parzelle = parcels[0] + + elif location: + logger.info(f"Creating parcel from location: {location}") + connector = SwissTopoMapServerConnector() + parcelData = await connector.search_parcel(location) + if not parcelData: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"No parcel found at location: {location}" + ) + extractedAttributes = connector.extract_parcel_attributes(parcelData) + attributes = parcelData.get("attributes", {}) + + parzelleCreateData = { + "mandateId": mandateId, + "label": extractedAttributes.get("label") or attributes.get("number") or "Unknown", + "parzellenAliasTags": [attributes.get("egris_egrid")] if attributes.get("egris_egrid") else [], + "eigentuemerschaft": None, + "strasseNr": location if not location.replace(",", "").replace(".", "").replace(" ", "").isdigit() else None, + "plz": None, + "perimeter": extractedAttributes.get("perimeter"), + "baulinie": None, + "kontextGemeinde": None, + "bauzone": None, + "az": None, + "bz": None, + "vollgeschossZahl": None, + "anrechenbarDachgeschoss": None, + "anrechenbarUntergeschoss": None, + "gebaeudehoeheMax": None, + "regelnGrenzabstand": [], + "regelnMehrlaengenzuschlag": [], + "regelnMehrhoehenzuschlag": [], + "parzelleBebaut": None, + "parzelleErschlossen": None, + "parzelleHanglage": None, + "laermschutzzone": None, + "hochwasserschutzzone": None, + "grundwasserschutzzone": None, + "parzellenNachbarschaft": [], + "dokumente": [], + "kontextInformationen": [ + Kontext( + thema="Swiss Topo Data", + inhalt=json.dumps({ + "egrid": attributes.get("egris_egrid"), + "identnd": attributes.get("identnd"), + "canton": attributes.get("ak"), + "municipality_code": attributes.get("bfsnr"), + "geoportal_url": attributes.get("geoportal_url") + }, ensure_ascii=False) + ) + ] + } + parzelleInstance = Parzelle(**parzelleCreateData) + parzelle = realEstateInterface.createParzelle(parzelleInstance) + + elif parcelDataDict: + logger.info(f"Creating parcel from custom data") + parcelDataDict["mandateId"] = mandateId + parzelleInstance = Parzelle(**parcelDataDict) + parzelle = realEstateInterface.createParzelle(parzelleInstance) + + else: + raise ValueError("One of 'parcelId', 'location', or 'parcelData' is required") + + if parzelle not in projekt.parzellen: + projekt.parzellen.append(parzelle) + + if not projekt.perimeter and parzelle.perimeter: + projekt.perimeter = parzelle.perimeter + + updatedProjekt = realEstateInterface.updateProjekt(projekt) + logger.info(f"Added Parzelle {parzelle.id} to Projekt {projektId}") + + return { + "projekt": updatedProjekt.model_dump(), + "parzelle": parzelle.model_dump() + } diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 7ab7d8d5..b9af4827 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -1,13 +1,14 @@ """ -Real Estate feature main logic. -Handles database operations with AI-powered natural language processing. -Stateless implementation without session management. +Real Estate feature main entry point. +Handles feature definition, RBAC registration, and lifecycle hooks. -This module also handles feature initialization and RBAC catalog registration. +Service logic is split into dedicated modules: +- serviceGeometry: Geometry utilities and project creation with parcel data +- serviceAiIntent: AI-based intent recognition and CRUD operations +- serviceBzo: BZO information extraction and filtering """ import logging -import re from modules.shared.i18nRegistry import t @@ -77,6 +78,12 @@ TEMPLATE_ROLES = [ }, ] +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Feature Definition & RBAC Registration +# --------------------------------------------------------------------------- def getFeatureDefinition(): """Return the feature definition for registration.""" @@ -121,15 +128,18 @@ def registerFeature(catalogService) -> bool: meta=resObj.get("meta") ) - # Sync template roles to database (with AccessRules) _syncTemplateRolesToDb() return True except Exception as e: - logging.getLogger(__name__).error(f"Failed to register feature '{FEATURE_CODE}': {e}") + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") return False +# --------------------------------------------------------------------------- +# Internal RBAC Helpers +# --------------------------------------------------------------------------- + def _syncTemplateRolesToDb() -> int: """ Sync template roles and their AccessRules to the database. @@ -169,16 +179,16 @@ def _syncTemplateRolesToDb() -> int: roleId = createdRole.get("id") existingRoleLabels[roleLabel] = roleId _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) - logging.getLogger(__name__).info(f"Created template role '{roleLabel}' with ID {roleId}") + logger.info(f"Created template role '{roleLabel}' with ID {roleId}") createdCount += 1 if createdCount > 0: - logging.getLogger(__name__).info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") + logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") _repairInstanceRolesAccessRules(rootInterface, existingRoleLabels) return createdCount except Exception as e: - logging.getLogger(__name__).error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") + logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") return 0 @@ -256,2822 +266,6 @@ def _ensureAccessRulesForRole(rootInterface, roleId: str, ruleTemplates: list) - return createdCount -import json -from typing import Optional, Dict, Any, List -from fastapi import HTTPException, status -from shapely.geometry import Polygon -from shapely.ops import unary_union -from modules.datamodels.datamodelUam import User -from .datamodelFeatureRealEstate import ( - Projekt, - Parzelle, - StatusProzess, - GeoPolylinie, - GeoPunkt, - Kontext, - Gemeinde, - Kanton, - Land, - DokumentTyp, -) -from modules.serviceHub import getInterface as getServices -from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface -from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface -from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector -from modules.features.realEstate.bzoDocumentRetriever import BZODocumentRetriever -from modules.features.realEstate.bzoExtraction import run_extraction, run_bzo_params_extraction -from modules.features.realEstate.parcelSelectionService import compute_selection_summary -from modules.features.realEstate.realEstateGemeindeService import ( - ensure_single_gemeinde, - fetch_bzo_for_gemeinde, -) - -logger = logging.getLogger(__name__) - - -# ===== Geometry Utilities ===== - -def geopolylinie_to_shapely_polygon(geopolylinie: GeoPolylinie) -> Polygon: - """ - Convert GeoPolylinie to Shapely Polygon. - - Args: - geopolylinie: GeoPolylinie instance with punkte list - - Returns: - Shapely Polygon object - """ - if not geopolylinie or not geopolylinie.punkte: - raise ValueError("GeoPolylinie must have at least one point") - - # Extract coordinates from punkte - coordinates = [] - for punkt in geopolylinie.punkte: - coordinates.append((punkt.x, punkt.y)) - - # Ensure polygon is closed (first point == last point) - if len(coordinates) < 3: - raise ValueError("Polygon must have at least 3 points") - - # Close polygon if not already closed - if coordinates[0] != coordinates[-1]: - coordinates.append(coordinates[0]) - - return Polygon(coordinates) - - -def shapely_polygon_to_geopolylinie(polygon: Polygon) -> GeoPolylinie: - """ - Convert Shapely Polygon to GeoPolylinie. - - Args: - polygon: Shapely Polygon object - - Returns: - GeoPolylinie instance with LV95 coordinate system - """ - if not polygon or polygon.is_empty: - raise ValueError("Polygon must not be empty") - - # Extract exterior coordinates - exterior_coords = list(polygon.exterior.coords) - - # Remove duplicate last point if present (Shapely includes it) - if len(exterior_coords) > 1 and exterior_coords[0] == exterior_coords[-1]: - exterior_coords = exterior_coords[:-1] - - # Convert to GeoPunkt list - punkte = [] - for coord in exterior_coords: - punkt = GeoPunkt( - koordinatensystem="LV95", - x=float(coord[0]), - y=float(coord[1]), - z=None - ) - punkte.append(punkt) - - return GeoPolylinie( - closed=True, - punkte=punkte - ) - - -def combine_parcel_geometries(geometries: List[GeoPolylinie]) -> GeoPolylinie: - """ - Combine multiple parcel geometries into a single outer outline. - - Uses Shapely union operation to merge polygons and automatically - removes internal edges. The result is a clean outer boundary. - - Args: - geometries: List of GeoPolylinie instances to combine - - Returns: - Combined GeoPolylinie representing the outer outline - - Raises: - ValueError: If geometries list is empty or invalid - """ - if not geometries or len(geometries) == 0: - raise ValueError("At least one geometry is required") - - if len(geometries) == 1: - # Single geometry - return as-is - return geometries[0] - - # Convert all geometries to Shapely Polygons - shapely_polygons = [] - for geo in geometries: - try: - polygon = geopolylinie_to_shapely_polygon(geo) - if not polygon.is_empty: - shapely_polygons.append(polygon) - except Exception as e: - logger.warning(f"Error converting geometry to Shapely Polygon: {e}") - continue - - if not shapely_polygons: - raise ValueError("No valid geometries to combine") - - if len(shapely_polygons) == 1: - # Only one valid polygon - convert back - return shapely_polygon_to_geopolylinie(shapely_polygons[0]) - - # Perform union operation - automatically removes internal edges - try: - combined = unary_union(shapely_polygons) - - # Handle MultiPolygon case (disconnected parcels) - if hasattr(combined, 'geoms'): - # Multiple separate polygons - combine their exteriors - # For now, take the largest polygon or combine all exteriors - # In practice, we might want to keep them separate or combine differently - largest = max(combined.geoms, key=lambda p: p.area) - combined = largest - - # Extract outer boundary - if combined.is_empty: - raise ValueError("Union resulted in empty geometry") - - # Convert back to GeoPolylinie - result = shapely_polygon_to_geopolylinie(combined) - logger.info(f"Combined {len(geometries)} geometries into single outline with {len(result.punkte)} points") - return result - - except Exception as e: - logger.error(f"Error combining geometries: {e}", exc_info=True) - raise ValueError(f"Failed to combine geometries: {str(e)}") - - -def filter_neighbor_parcels( - neighbors: List[Dict[str, Any]], - selected_geometries: List[GeoPolylinie] -) -> List[Dict[str, Any]]: - """ - Filter neighbor parcels to exclude those that are part of the selected parcels. - - Uses geometric comparison to check if neighbor parcels intersect or touch - any of the selected parcel geometries. - - Args: - neighbors: List of neighbor parcel dictionaries (must have 'perimeter' or 'geometry_geojson') - selected_geometries: List of GeoPolylinie instances representing selected parcels - - Returns: - Filtered list of neighbor parcels (excluding selected ones) - """ - if not neighbors or not selected_geometries: - return neighbors - - # Convert selected geometries to Shapely Polygons for comparison - selected_polygons = [] - for geo in selected_geometries: - try: - polygon = geopolylinie_to_shapely_polygon(geo) - if not polygon.is_empty: - selected_polygons.append(polygon) - except Exception as e: - logger.warning(f"Error converting selected geometry for filtering: {e}") - continue - - if not selected_polygons: - # No valid selected geometries - return all neighbors - return neighbors - - # Filter neighbors - filtered_neighbors = [] - for neighbor in neighbors: - try: - # Try to get geometry from neighbor - neighbor_geometry = None - - # Check for perimeter (GeoPolylinie format) - if neighbor.get("perimeter"): - perimeter = neighbor["perimeter"] - if isinstance(perimeter, dict) and perimeter.get("punkte"): - # Convert to GeoPolylinie - punkte = [] - for p in perimeter["punkte"]: - punkt = GeoPunkt( - koordinatensystem=p.get("koordinatensystem", "LV95"), - x=float(p.get("x", 0)), - y=float(p.get("y", 0)), - z=p.get("z") - ) - punkte.append(punkt) - neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte) - - # Check for geometry_geojson - elif neighbor.get("geometry_geojson"): - geo_json = neighbor["geometry_geojson"] - geometry = geo_json.get("geometry") if isinstance(geo_json, dict) else geo_json - - if geometry and geometry.get("type") == "Polygon": - coordinates = geometry.get("coordinates", []) - if coordinates and len(coordinates) > 0: - ring = coordinates[0] # Outer ring - punkte = [] - for coord in ring: - if len(coord) >= 2: - punkt = GeoPunkt( - koordinatensystem="LV95", - x=float(coord[0]), - y=float(coord[1]), - z=float(coord[2]) if len(coord) > 2 else None - ) - punkte.append(punkt) - neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte) - - if not neighbor_geometry: - # No geometry available - include neighbor (can't filter without geometry) - filtered_neighbors.append(neighbor) - continue - - # Convert neighbor geometry to Shapely Polygon - neighbor_polygon = geopolylinie_to_shapely_polygon(neighbor_geometry) - - # Check if neighbor intersects or touches any selected parcel - is_selected = False - for selected_polygon in selected_polygons: - if neighbor_polygon.intersects(selected_polygon) or neighbor_polygon.touches(selected_polygon): - # Check if they're actually the same (within tolerance) - # If areas are very similar, it's likely the same parcel - area_diff = abs(neighbor_polygon.area - selected_polygon.area) - if area_diff < 1.0: # Less than 1 m² difference - is_selected = True - break - # Also check if one contains the other (shouldn't happen for neighbors, but check anyway) - if neighbor_polygon.contains(selected_polygon) or selected_polygon.contains(neighbor_polygon): - is_selected = True - break - - if not is_selected: - filtered_neighbors.append(neighbor) - else: - logger.debug(f"Filtered out neighbor parcel {neighbor.get('id')} - part of selected parcels") - - except Exception as e: - logger.warning(f"Error filtering neighbor parcel {neighbor.get('id')}: {e}") - # On error, include neighbor (better to show too many than too few) - filtered_neighbors.append(neighbor) - - logger.info(f"Filtered {len(neighbors)} neighbors to {len(filtered_neighbors)} (removed {len(neighbors) - len(filtered_neighbors)} selected parcels)") - return filtered_neighbors - - -# ===== Swisstopo Integration ===== - -async def fetch_parcel_polygon_from_swisstopo( - gemeinde: str, - parzellen_nr: str, - sr: int = 2056 -) -> Optional[Dict[str, Any]]: - """ - Holt die vollständige Polygon-Geometrie einer Parzelle von Swisstopo API. - - Args: - gemeinde: Name der Gemeinde (z.B. "Bern") - parzellen_nr: Parzellennummer (z.B. "1234") - sr: Koordinatensystem (2056=LV95, 4326=WGS84) - - Returns: - Dictionary mit GeoPolylinie-Format für perimeter-Feld, oder None wenn nicht gefunden - Format: {"closed": True, "punkte": [{"koordinatensystem": "LV95", "x": ..., "y": ..., "z": None}, ...]} - """ - try: - connector = SwissTopoMapServerConnector() - - # Get GeoJSON feature from Swisstopo - feature = await connector.get_parcel_polygon(gemeinde, parzellen_nr, sr) - - if not feature: - logger.warning(f"Parzelle {gemeinde} {parzellen_nr} nicht gefunden in Swisstopo") - return None - - # Convert GeoJSON to GeoPolylinie format - geometry = feature.get("geometry", {}) - if geometry.get("type") == "Polygon": - coordinates = geometry.get("coordinates", []) - if coordinates and len(coordinates) > 0: - ring = coordinates[0] # Outer ring - - punkte = [] - for coord in ring: - if len(coord) >= 2: - punkt = { - "koordinatensystem": "LV95" if sr == 2056 else "WGS84", - "x": coord[0], # GeoJSON: [x, y] = [easting, northing] - "y": coord[1], - "z": coord[2] if len(coord) > 2 else None - } - punkte.append(punkt) - - logger.info(f"Successfully fetched polygon with {len(punkte)} points for {gemeinde} {parzellen_nr}") - - return { - "closed": True, - "punkte": punkte - } - - logger.warning(f"Unexpected geometry type in Swisstopo response: {geometry.get('type')}") - return None - - except Exception as e: - logger.error(f"Error fetching parcel polygon from Swisstopo: {e}", exc_info=True) - return None - - -# ===== Direkte Query-Ausführung (stateless) ===== - -async def executeDirectQuery( - currentUser: User, - mandateId: str, - queryText: str, - parameters: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """ - Execute a database query directly without session management. - - Args: - currentUser: Current authenticated user - mandateId: Mandate context (from RequestContext / X-Mandate-Id header) - queryText: SQL query text - parameters: Optional parameters for parameterized queries - - Returns: - Dictionary containing query result (rows, columns, rowCount) - - Note: - - No session or query history is saved - - Query is executed directly and result is returned - - For production, validate and sanitize queries before execution - """ - try: - logger.info(f"Executing direct query for user {currentUser.id} (mandate: {mandateId})") - logger.debug(f"Query text: {queryText}") - if parameters: - logger.debug(f"Query parameters: {parameters}") - - # Execute query via Real Estate interface (stateless) - realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) - result = realEstateInterface.executeQuery(queryText, parameters) - - logger.info( - f"Query executed successfully: {result['rowCount']} rows in {result.get('executionTime', 0):.3f}s" - ) - - return { - "status": "success", - "rows": result["rows"], - "columns": result["columns"], - "rowCount": result["rowCount"], - "executionTime": result.get("executionTime", 0), - } - - except Exception as e: - logger.error(f"Error executing query: {str(e)}", exc_info=True) - raise - - -# ===== AI-basierte Intent-Erkennung und CRUD-Operationen ===== - -def _formatEntitySummary(entity_type: str, items: List[Dict[str, Any]], filters: Dict[str, Any]) -> str: - """ - Format a human-readable summary of query results. - - Args: - entity_type: Type of entity (Projekt, Parzelle, etc.) - items: List of entity data dictionaries - filters: Filter parameters used in the query - - Returns: - Human-readable summary string - """ - if not items: - return f"Keine {entity_type} gefunden" - - count = len(items) - filter_desc = "" - if filters: - # Format filter description - if "kontextGemeinde" in filters: - filter_desc = f" in {filters['kontextGemeinde']}" - elif "plz" in filters: - filter_desc = f" mit PLZ {filters['plz']}" - elif "location_filter" in filters: - filter_desc = f" in {filters['location_filter']}" - - # Start with count - summary = f"Gefunden: {count} {entity_type}{filter_desc}" - - # Add details based on entity type - if entity_type == "Parzelle": - summary += "\n\nDetails:" - for i, item in enumerate(items[:10], 1): # Limit to first 10 - parts = [] - - # Add label or ID - if item.get("label"): - parts.append(f"Parzelle '{item['label']}'") - elif item.get("id"): - parts.append(f"Parzelle {item['id'][:8]}...") - - # Add address - if item.get("strasseNr"): - parts.append(item["strasseNr"]) - - # Add PLZ and municipality - location_parts = [] - if item.get("plz"): - location_parts.append(item["plz"]) - if item.get("kontextGemeinde"): - location_parts.append(item["kontextGemeinde"]) - if location_parts: - parts.append(" ".join(location_parts)) - - # Add building zone - if item.get("bauzone"): - parts.append(f"Bauzone: {item['bauzone']}") - - summary += f"\n{i}. {', '.join(parts)}" - - if count > 10: - summary += f"\n... und {count - 10} weitere" - - elif entity_type == "Projekt": - summary += "\n\nDetails:" - for i, item in enumerate(items[:10], 1): - parts = [] - - # Add label - if item.get("label"): - parts.append(f"'{item['label']}'") - - # Add status - if item.get("statusProzess"): - parts.append(f"Status: {item['statusProzess']}") - - # Add parcel count - parzellen = item.get("parzellen", []) - if parzellen: - parts.append(f"{len(parzellen)} Parzelle(n)") - - summary += f"\n{i}. {' - '.join(parts)}" - - if count > 10: - summary += f"\n... und {count - 10} weitere" - - elif entity_type == "Gemeinde": - summary += "\n\nDetails:" - for i, item in enumerate(items[:10], 1): - parts = [] - - if item.get("label"): - parts.append(item["label"]) - if item.get("plz"): - parts.append(f"PLZ: {item['plz']}") - if item.get("abk"): - parts.append(f"Abk: {item['abk']}") - - summary += f"\n{i}. {', '.join(parts)}" - - if count > 10: - summary += f"\n... und {count - 10} weitere" - - elif entity_type == "Dokument": - summary += "\n\nDetails:" - for i, item in enumerate(items[:10], 1): - parts = [] - - if item.get("label"): - parts.append(item["label"]) - if item.get("dokumentTyp"): - parts.append(f"Typ: {item['dokumentTyp']}") - if item.get("quelle"): - parts.append(f"Quelle: {item['quelle']}") - - summary += f"\n{i}. {', '.join(parts)}" - - if count > 10: - summary += f"\n... und {count - 10} weitere" - - else: - # Generic format for other entity types - if count <= 5: - summary += "\n\nDetails:" - for i, item in enumerate(items, 1): - label = item.get("label") or item.get("id", "") - if label: - summary += f"\n{i}. {label}" - - return summary - - -async def processNaturalLanguageCommand( - currentUser: User, - mandateId: str, - userInput: str, -) -> Dict[str, Any]: - """ - Process natural language user input and execute corresponding CRUD operations. - - Uses AI to analyze user intent and extract parameters, then executes the appropriate - CRUD operation through the interface. Works stateless without session management. - - Args: - currentUser: Current authenticated user - mandateId: Mandate context (from RequestContext / X-Mandate-Id header) - userInput: Natural language command from user - - Returns: - Dictionary containing operation result and metadata - - Example user inputs: - - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - - "Zeige mir alle Projekte in Zürich" - - "Aktualisiere Projekt XYZ mit Status 'Planung'" - - "Lösche Parzelle ABC" - - "SELECT * FROM Projekt WHERE plz = '8000'" - """ - try: - logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {mandateId})") - logger.debug(f"User input: {userInput}") - - # Initialize services for AI access - services = getServices(currentUser, workflow=None, mandateId=mandateId) - aiService = services.ai - - # Step 1: Analyze user intent with AI - intentAnalysis = await analyzeUserIntent(aiService, userInput) - - logger.info(f"Intent analysis result: intent={intentAnalysis.get('intent')}, entity={intentAnalysis.get('entity')}") - - # Step 2: Execute CRUD operation based on intent - result = await executeIntentBasedOperation( - currentUser=currentUser, - mandateId=mandateId, - intent=intentAnalysis["intent"], - entity=intentAnalysis.get("entity"), - parameters=intentAnalysis.get("parameters", {}), - ) - - # Build user-friendly response - response = { - "success": True, - "intent": intentAnalysis["intent"], - "entity": intentAnalysis.get("entity"), - "result": result, - } - - # Add human-readable summary for operations - if intentAnalysis["intent"] == "CREATE" and isinstance(result, dict): - # Add confirmation message for CREATE operations - operation_result = result.get("result") - if isinstance(operation_result, dict): - entity_name = intentAnalysis.get('entity', 'Eintrag') - label = operation_result.get("label", operation_result.get("id", "")) - - # Build detailed message - msg_parts = [f"✅ {entity_name} '{label}' erfolgreich erstellt"] - - if entity_name == "Parzelle": - if operation_result.get("plz"): - msg_parts.append(f"PLZ: {operation_result['plz']}") - if operation_result.get("kontextGemeinde"): - msg_parts.append(f"Gemeinde: {operation_result['kontextGemeinde']}") - if operation_result.get("bauzone"): - msg_parts.append(f"Bauzone: {operation_result['bauzone']}") - - kontext_items = operation_result.get("kontextInformationen", []) - if kontext_items: - msg_parts.append(f"\n📋 {len(kontext_items)} Kontextinformationen gespeichert:") - for kontext in kontext_items[:5]: # Show first 5 - thema = kontext.get("thema", "") - inhalt = kontext.get("inhalt", "") - if thema and inhalt: - msg_parts.append(f" • {thema}: {inhalt}") - if len(kontext_items) > 5: - msg_parts.append(f" • ... und {len(kontext_items) - 5} weitere") - - elif entity_name == "Projekt": - if operation_result.get("statusProzess"): - msg_parts.append(f"Status: {operation_result['statusProzess']}") - parzellen = operation_result.get("parzellen", []) - if parzellen: - msg_parts.append(f"{len(parzellen)} Parzelle(n)") - - response["message"] = "\n".join(msg_parts) - - elif intentAnalysis["intent"] == "READ" and isinstance(result, dict): - operation_result = result.get("result") - if isinstance(operation_result, list): - response["count"] = len(operation_result) - entity_name = intentAnalysis.get('entity', 'Einträge') - - if len(operation_result) == 0: - # Provide helpful message for empty results - filter_info = intentAnalysis.get('parameters', {}) - if filter_info: - filter_desc = ", ".join([f"{k}={v}" for k, v in filter_info.items()]) - response["message"] = f"Keine {entity_name} gefunden mit Filter: {filter_desc}. Möglicherweise sind noch keine Daten vorhanden oder der Filter ist zu spezifisch." - else: - response["message"] = f"Keine {entity_name} vorhanden. Erstellen Sie zuerst neue Einträge." - else: - # Create detailed summary based on entity type - response["message"] = _formatEntitySummary( - entity_name, - operation_result, - intentAnalysis.get('parameters', {}) - ) - elif isinstance(operation_result, dict): - response["count"] = 1 - # Format single entity - entity_name = intentAnalysis.get('entity', 'Eintrag') - response["message"] = _formatEntitySummary(entity_name, [operation_result], {}) - - return response - - except Exception as e: - logger.error(f"Error processing natural language command: {str(e)}", exc_info=True) - raise - - -async def analyzeUserIntent( - aiService, - userInput: str -) -> Dict[str, Any]: - """ - Use AI to analyze user input and extract intent, entity, and parameters. - - Args: - aiService: AI service instance - userInput: Natural language user input - - Returns: - Dictionary with 'intent', 'entity', and 'parameters' - """ - # Create a structured prompt for intent analysis with accurate field information - intentPrompt = f""" -Analyze the following user command and extract the intent, entity, and parameters. - -User Command: "{userInput}" - -Available intents: -- CREATE: User wants to create a new entity -- READ: User wants to read/query entities -- UPDATE: User wants to update an existing entity -- DELETE: User wants to delete an entity -- QUERY: User wants to execute a database query (SQL statements) - -Available entities and their fields: - -**Projekt** (Real estate project): -- id: string (primary key) -- mandateId: string (mandate ID) -- label: string (project designation/name) -- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) -- perimeter: GeoPolylinie (geographic boundary, JSONB) -- baulinie: GeoPolylinie (building line, JSONB) -- parzellen: List[Parzelle] (plots belonging to project, JSONB) -- dokumente: List[Dokument] (documents, JSONB) -- kontextInformationen: List[Kontext] (context info, JSONB) - -**Parzelle** (Plot/parcel): -- id: string (primary key) -- mandateId: string (mandate ID) -- label: string (plot designation) -- strasseNr: string (street and house number) -- plz: string (postal code) -- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) -- bauzone: string (building zone, e.g. W3, WG2) -- az: float (Ausnützungsziffer) -- bz: float (Bebauungsziffer) -- vollgeschossZahl: int (number of allowed full floors) -- gebaeudehoeheMax: float (maximum building height in meters) -- laermschutzzone: string (noise protection zone) -- hochwasserschutzzone: string (flood protection zone) -- grundwasserschutzzone: string (groundwater protection zone) -- parzelleBebaut: JaNein enum (is plot built) -- parzelleErschlossen: JaNein enum (is plot developed) -- parzelleHanglage: JaNein enum (is plot on slope) -- kontextInformationen: List[Kontext] (metadata - each item has 'thema' and 'inhalt' fields only) - -**Kontext** (Context information for metadata): -- thema: string (topic/subject, e.g. "EGRID", "Fläche", "Zentrum") -- inhalt: string (content as text, e.g. "CH887199917793", "6514.99 m²", "X: 123, Y: 456") - -**Important relationships:** -- Projekte contain Parzellen (projects have plots) -- Parzelle links to Gemeinde (via kontextGemeinde) -- Gemeinde links to Kanton (via id_kanton) -- Kanton links to Land (via id_land) -- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) -- Projekt does NOT have location fields directly - location is stored in associated Parzellen - -Return a JSON object with the following structure: -{{ - "intent": "CREATE|READ|UPDATE|DELETE|QUERY", - "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", - "parameters": {{ - // Extracted parameters from user input - // For CREATE/UPDATE: include all relevant fields using EXACT field names from above - // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) - // For DELETE: include entity ID if mentioned - // For QUERY: include queryText if SQL is detected - // IMPORTANT: Use only field names that exist in the entity definition above - }}, - "confidence": 0.0-1.0 // Confidence score for the analysis -}} - -Examples: -- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - Output: {{"intent": "CREATE", "entity": "Projekt", "parameters": {{"label": "Hauptstrasse 42"}}, "confidence": 0.95}} - -- Input: "Erstelle eine Parzelle mit Label 123, PLZ 8000, Gemeinde Zürich, Bauzone W3" - Output: {{"intent": "CREATE", "entity": "Parzelle", "parameters": {{"label": "123", "plz": "8000", "kontextGemeinde": "Zürich", "bauzone": "W3"}}, "confidence": 0.95}} - -- Input: "Parzellen-Informationen: ID:AA1704, Nummer:AA1704, EGRID:CH887199917793, Kanton:ZH, Gemeinde:Zürich, Gemeinde-Code:261, Fläche:6514.99 m², Zentrum:2682951.44,1247622.91" - Output: {{ - "intent": "CREATE", - "entity": "Parzelle", - "parameters": {{ - "label": "AA1704", - "parzellenAliasTags": ["AA1704"], - "kontextGemeinde": "Zürich", - "kontextInformationen": [ - {{"thema": "EGRID", "inhalt": "CH887199917793"}}, - {{"thema": "Kanton", "inhalt": "ZH"}}, - {{"thema": "BFS-Nummer", "inhalt": "261"}}, - {{"thema": "Fläche", "inhalt": "6514.99 m²"}}, - {{"thema": "Zentrum (LV95)", "inhalt": "X: 2682951.44 m, Y: 1247622.91 m (EPSG:2056)"}} - ] - }}, - "confidence": 0.9 - }} - Note: Extract structured data from detailed input. Use kontextInformationen for metadata. Each item has 'thema' (topic) and 'inhalt' (content as text). - -- Input: "Zeige mir alle Projekte" - Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{}}, "confidence": 0.9}} - -- Input: "Zeige mir Projekte in Zürich" or "Wie viele Projekte in Zürich" - Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{"location_filter": "Zürich"}}, "confidence": 0.9}} - Note: For project location queries, use Projekt entity with location_filter parameter - -- Input: "Zeige mir Parzellen mit PLZ 8000" - Output: {{"intent": "READ", "entity": "Parzelle", "parameters": {{"plz": "8000"}}, "confidence": 0.95}} - -- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" - Output: {{"intent": "UPDATE", "entity": "Projekt", "parameters": {{"id": "XYZ", "statusProzess": "Planung"}}, "confidence": 0.85}} - -- Input: "SELECT * FROM Projekt WHERE label = 'Test'" - Output: {{"intent": "QUERY", "entity": null, "parameters": {{"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}}, "confidence": 1.0}} - -- Input: "Lösche Parzelle ABC" - Output: {{"intent": "DELETE", "entity": "Parzelle", "parameters": {{"id": "ABC"}}, "confidence": 0.9}} - -IMPORTANT EXTRACTION RULES: -1. For CREATE operations, extract ALL mentioned data fields from the user input -2. Use kontextInformationen array for metadata that doesn't have dedicated fields (EGRID, BFS numbers, area, coordinates, etc.) -3. Each kontextInformationen item MUST have exactly two fields: 'thema' (topic/subject) and 'inhalt' (content as text string) -4. Format kontextInformationen values as readable text strings, including units (e.g., "6514.99 m²", "X: 123, Y: 456") -5. Match field names EXACTLY to the entity definition above -6. Convert data types correctly (strings for text, numbers for numeric values) -7. Extract coordinates, areas, and other numeric values from text -8. When multiple values are mentioned for the same concept (ID, Nummer, Name), use the most relevant one for 'label' and put alternatives in parzellenAliasTags -""" - - try: - # Use AI planning call for structured JSON response - response = await aiService.callAiPlanning( - prompt=intentPrompt, - debugType="intentanalysis" - ) - - # Extract JSON from response (handles markdown code blocks) - jsonStart = response.find('{') - jsonEnd = response.rfind('}') + 1 - - if jsonStart == -1 or jsonEnd == 0: - raise ValueError("No JSON found in AI response") - - jsonStr = response[jsonStart:jsonEnd] - - # Parse JSON response - intentData = json.loads(jsonStr) - - # Validate response structure - if "intent" not in intentData: - raise ValueError("Invalid intent analysis response: missing 'intent' field") - - # Ensure parameters exists - if "parameters" not in intentData: - intentData["parameters"] = {} - - logger.debug(f"Parsed intent analysis: {intentData}") - - return intentData - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse AI intent analysis response: {e}") - logger.error(f"Raw response: {response}") - raise ValueError(f"AI returned invalid JSON: {str(e)}") - except Exception as e: - logger.error(f"Error analyzing user intent: {str(e)}", exc_info=True) - raise - - -async def executeIntentBasedOperation( - currentUser: User, - mandateId: str, - intent: str, - entity: Optional[str], - parameters: Dict[str, Any], -) -> Dict[str, Any]: - """ - Execute CRUD operation based on analyzed intent. - - Args: - currentUser: Current authenticated user - mandateId: Mandate context (from RequestContext / X-Mandate-Id header) - intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) - entity: Entity type from AI analysis - parameters: Extracted parameters from AI analysis - - Returns: - Operation result - - Note: - - Supports CREATE, READ, UPDATE, DELETE, QUERY intents - - Entity types: Projekt, Parzelle, Gemeinde, Kanton, Land, Dokument - """ - try: - logger.info(f"Executing intent-based operation: intent={intent}, entity={entity}") - logger.debug(f"Parameters: {parameters}") - - if intent == "QUERY": - # Execute database query directly (stateless) - queryText = parameters.get("queryText", "") - - if not queryText: - raise ValueError("QUERY intent requires queryText in parameters") - - result = await executeDirectQuery( - currentUser=currentUser, - mandateId=mandateId, - queryText=queryText, - parameters=parameters.get("queryParameters"), - ) - return result - - elif intent == "CREATE": - # Create new entity - realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) - - if entity == "Projekt": - # Create Projekt from parameters - projekt = Projekt( - mandateId=mandateId, - label=parameters.get("label", ""), - statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, - ) - created = realEstateInterface.createProjekt(projekt) - return { - "operation": "CREATE", - "entity": "Projekt", - "result": created.model_dump() - } - - elif entity == "Parzelle": - # Create Parzelle from parameters - # Import Kontext for kontextInformationen - from modules.features.realestate.datamodelFeatureRealEstate import Kontext, GeoPolylinie - - # Build parzelle data with all extracted parameters - parzelle_data = { - "mandateId": mandateId, - "label": parameters.get("label", ""), - } - - # Add optional fields if present - optional_fields = [ - "parzellenAliasTags", "eigentuemerschaft", "strasseNr", "plz", - "bauzone", "az", "bz", "vollgeschossZahl", "anrechenbarDachgeschoss", - "anrechenbarUntergeschoss", "gebaeudehoeheMax", "kontextGemeinde", - "regelnGrenzabstand", "regelnMehrlaengenzuschlag", "regelnMehrhoehenzuschlag", - "parzelleBebaut", "parzelleErschlossen", "parzelleHanglage", - "laermschutzzone", "hochwasserschutzzone", "grundwasserschutzzone" - ] - - for field in optional_fields: - if field in parameters and parameters[field] is not None: - parzelle_data[field] = parameters[field] - - # Handle complex objects - if "perimeter" in parameters and parameters["perimeter"]: - parzelle_data["perimeter"] = GeoPolylinie(**parameters["perimeter"]) - elif "kontextGemeinde" in parameters and parameters.get("kontextGemeinde"): - # Try to fetch polygon from Swisstopo if gemeinde and parzellen_nr are available - gemeinde = parameters.get("kontextGemeinde") - parzellen_nr = parameters.get("label") or parameters.get("parzellen_nr") or parameters.get("parzellennummer") - - if gemeinde and parzellen_nr: - logger.info(f"Attempting to fetch polygon from Swisstopo for {gemeinde} {parzellen_nr}") - try: - # Try to resolve gemeinde name if it's an ID - gemeinde_name = gemeinde - if len(gemeinde) == 36: # UUID format - # Try to get gemeinde name from interface (realEstateInterface already initialized above) - gemeinde_obj = realEstateInterface.getGemeinde(gemeinde) - if gemeinde_obj: - gemeinde_name = gemeinde_obj.label - - polygon_data = await fetch_parcel_polygon_from_swisstopo( - gemeinde=gemeinde_name, - parzellen_nr=str(parzellen_nr), - sr=2056 - ) - - if polygon_data: - parzelle_data["perimeter"] = GeoPolylinie(**polygon_data) - logger.info(f"Successfully fetched and set perimeter from Swisstopo") - else: - logger.warning(f"Could not fetch polygon from Swisstopo for {gemeinde_name} {parzellen_nr}") - except Exception as e: - logger.warning(f"Error fetching polygon from Swisstopo (continuing without): {e}") - - if "baulinie" in parameters and parameters["baulinie"]: - parzelle_data["baulinie"] = GeoPolylinie(**parameters["baulinie"]) - - # Handle kontextInformationen (convert dicts to Kontext objects) - if "kontextInformationen" in parameters and parameters["kontextInformationen"]: - kontext_list = [] - for kontext_data in parameters["kontextInformationen"]: - if isinstance(kontext_data, dict): - # Ensure only thema and inhalt are passed (Kontext model only has these fields) - kontext_obj = Kontext( - thema=kontext_data.get("thema", ""), - inhalt=kontext_data.get("inhalt", "") - ) - kontext_list.append(kontext_obj) - else: - kontext_list.append(kontext_data) - parzelle_data["kontextInformationen"] = kontext_list - - parzelle = Parzelle(**parzelle_data) - created = realEstateInterface.createParzelle(parzelle) - - logger.info(f"Created Parzelle '{created.label}' with {len(created.kontextInformationen)} context items") - - return { - "operation": "CREATE", - "entity": "Parzelle", - "result": created.model_dump() - } - elif entity == "Gemeinde": - # Create Gemeinde from parameters - from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde - gemeinde = Gemeinde( - mandateId=mandateId, - label=parameters.get("label", ""), - id_kanton=parameters.get("id_kanton"), - plz=parameters.get("plz"), - ) - created = realEstateInterface.createGemeinde(gemeinde) - return { - "operation": "CREATE", - "entity": "Gemeinde", - "result": created.model_dump() - } - elif entity == "Kanton": - # Create Kanton from parameters - from modules.features.realestate.datamodelFeatureRealEstate import Kanton - kanton = Kanton( - mandateId=mandateId, - label=parameters.get("label", ""), - id_land=parameters.get("id_land"), - abk=parameters.get("abk"), - ) - created = realEstateInterface.createKanton(kanton) - return { - "operation": "CREATE", - "entity": "Kanton", - "result": created.model_dump() - } - elif entity == "Land": - # Create Land from parameters - from modules.features.realestate.datamodelFeatureRealEstate import Land - land = Land( - mandateId=mandateId, - label=parameters.get("label", ""), - abk=parameters.get("abk"), - ) - created = realEstateInterface.createLand(land) - return { - "operation": "CREATE", - "entity": "Land", - "result": created.model_dump() - } - elif entity == "Dokument": - # Create Dokument from parameters - from modules.features.realestate.datamodelFeatureRealEstate import Dokument - dokument = Dokument( - mandateId=mandateId, - label=parameters.get("label", ""), - dokumentReferenz=parameters.get("dokumentReferenz", ""), - versionsbezeichnung=parameters.get("versionsbezeichnung"), - dokumentTyp=parameters.get("dokumentTyp"), - quelle=parameters.get("quelle"), - mimeType=parameters.get("mimeType"), - ) - created = realEstateInterface.createDokument(dokument) - return { - "operation": "CREATE", - "entity": "Dokument", - "result": created.model_dump() - } - else: - raise ValueError(f"CREATE operation not supported for entity: {entity}") - - elif intent == "READ": - # Read entities - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projektId = parameters.get("id") - if projektId: - # Get single Projekt by ID - projekt = realEstateInterface.getProjekt(projektId) - if not projekt: - raise ValueError(f"Projekt {projektId} not found") - return { - "operation": "READ", - "entity": "Projekt", - "result": projekt.model_dump() - } - else: - # List all Projekte (with optional filters) - # Validate filter fields against Projekt model - validProjektFields = {"id", "mandateId", "label", "statusProzess"} - recordFilter = { - k: v for k, v in parameters.items() - if k != "id" and k in validProjektFields - } - - # Handle location_filter specially (filter projects by parcel location) - location_filter = parameters.get("location_filter") - - # Get all projects first - projekte = realEstateInterface.getProjekte(recordFilter=recordFilter if recordFilter else None) - - # If location filter is present, filter by parcels in that location - if location_filter: - logger.info(f"Filtering projects by location: {location_filter}") - - # Try to resolve location to Gemeinde ID for UUID comparison - location_id = None - try: - # Check if it's already a UUID - uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE) - if not uuid_pattern.match(location_filter): - # Try to resolve name to ID - gemeinde_records = realEstateInterface.getGemeinden(recordFilter={"label": location_filter}) - if gemeinde_records: - location_id = gemeinde_records[0].id - logger.debug(f"Resolved location '{location_filter}' to ID '{location_id}'") - except Exception as e: - logger.debug(f"Could not resolve location filter: {e}") - - filtered_projekte = [] - - for projekt in projekte: - # Check if any parcel in the project matches the location - for parzelle in projekt.parzellen: - # Check kontextGemeinde (both UUID and string), plz, or strasseNr for location match - location_lower = location_filter.lower() - matches = False - - # Check if kontextGemeinde matches (as UUID or string) - if parzelle.kontextGemeinde: - if (parzelle.kontextGemeinde == location_id or # UUID match - parzelle.kontextGemeinde == location_filter or # Exact match - location_lower in parzelle.kontextGemeinde.lower()): # Partial string match - matches = True - - # Check PLZ or address - if not matches and ( - (parzelle.plz and location_lower in parzelle.plz) or - (parzelle.strasseNr and location_lower in parzelle.strasseNr.lower()) - ): - matches = True - - if matches: - filtered_projekte.append(projekt) - break # Found a matching parcel, no need to check more - - projekte = filtered_projekte - logger.info(f"Found {len(projekte)} projects in location '{location_filter}'") - - return { - "operation": "READ", - "entity": "Projekt", - "result": [p.model_dump() for p in projekte], - "count": len(projekte) - } - elif entity == "Parzelle": - parzelleId = parameters.get("id") - if parzelleId: - # Get single Parzelle by ID - parzelle = realEstateInterface.getParzelle(parzelleId) - if not parzelle: - raise ValueError(f"Parzelle {parzelleId} not found") - return { - "operation": "READ", - "entity": "Parzelle", - "result": parzelle.model_dump() - } - else: - # List all Parzellen (with optional filters) - # Validate filter fields against Parzelle model - # Note: kontextKanton and kontextLand are NOT direct fields on Parzelle - # Parzelle links to Gemeinde, Gemeinde links to Kanton, Kanton links to Land - validParzelleFields = { - "id", "mandateId", "label", "strasseNr", "plz", - "kontextGemeinde", # Only direct link - Gemeinde → Kanton → Land - "bauzone", "az", "bz", "vollgeschossZahl", "gebaeudehoeheMax", - "laermschutzzone", "hochwasserschutzzone", "grundwasserschutzzone", - "parzelleBebaut", "parzelleErschlossen", "parzelleHanglage" - } - recordFilter = { - k: v for k, v in parameters.items() - if k != "id" and k in validParzelleFields - } - # Warn about invalid fields - invalidFields = {k: v for k, v in parameters.items() if k not in validParzelleFields and k != "id"} - if invalidFields: - logger.warning(f"Invalid filter fields for Parzelle ignored: {list(invalidFields.keys())}") - - parzellen = realEstateInterface.getParzellen(recordFilter=recordFilter if recordFilter else None) - - # Debug logging for empty results - if not parzellen and recordFilter: - logger.info(f"No Parzellen found matching filter: {recordFilter}") - # Get total count to help debug - all_parzellen = realEstateInterface.getParzellen(recordFilter=None) - logger.info(f"Total Parzellen in database: {len(all_parzellen)}") - if all_parzellen: - # Show some sample kontextGemeinde values - sample_gemeinden = set() - for p in all_parzellen[:10]: - if p.kontextGemeinde: - sample_gemeinden.add(p.kontextGemeinde) - logger.info(f"Sample kontextGemeinde values in database: {sample_gemeinden}") - - return { - "operation": "READ", - "entity": "Parzelle", - "result": [p.model_dump() for p in parzellen], - "count": len(parzellen) - } - elif entity == "Gemeinde": - from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde - gemeindeId = parameters.get("id") - if gemeindeId: - gemeinde = realEstateInterface.getGemeinde(gemeindeId) - if not gemeinde: - raise ValueError(f"Gemeinde {gemeindeId} not found") - return { - "operation": "READ", - "entity": "Gemeinde", - "result": gemeinde.model_dump() - } - else: - recordFilter = {k: v for k, v in parameters.items() if k != "id"} - gemeinden = realEstateInterface.getGemeinden(recordFilter=recordFilter if recordFilter else None) - return { - "operation": "READ", - "entity": "Gemeinde", - "result": [g.model_dump() for g in gemeinden], - "count": len(gemeinden) - } - elif entity == "Kanton": - from modules.features.realestate.datamodelFeatureRealEstate import Kanton - kantonId = parameters.get("id") - if kantonId: - kanton = realEstateInterface.getKanton(kantonId) - if not kanton: - raise ValueError(f"Kanton {kantonId} not found") - return { - "operation": "READ", - "entity": "Kanton", - "result": kanton.model_dump() - } - else: - recordFilter = {k: v for k, v in parameters.items() if k != "id"} - kantone = realEstateInterface.getKantone(recordFilter=recordFilter if recordFilter else None) - return { - "operation": "READ", - "entity": "Kanton", - "result": [k.model_dump() for k in kantone], - "count": len(kantone) - } - elif entity == "Land": - from modules.features.realestate.datamodelFeatureRealEstate import Land - landId = parameters.get("id") - if landId: - land = realEstateInterface.getLand(landId) - if not land: - raise ValueError(f"Land {landId} not found") - return { - "operation": "READ", - "entity": "Land", - "result": land.model_dump() - } - else: - recordFilter = {k: v for k, v in parameters.items() if k != "id"} - laender = realEstateInterface.getLaender(recordFilter=recordFilter if recordFilter else None) - return { - "operation": "READ", - "entity": "Land", - "result": [l.model_dump() for l in laender], - "count": len(laender) - } - elif entity == "Dokument": - from modules.features.realestate.datamodelFeatureRealEstate import Dokument - dokumentId = parameters.get("id") - if dokumentId: - dokument = realEstateInterface.getDokument(dokumentId) - if not dokument: - raise ValueError(f"Dokument {dokumentId} not found") - return { - "operation": "READ", - "entity": "Dokument", - "result": dokument.model_dump() - } - else: - recordFilter = {k: v for k, v in parameters.items() if k != "id"} - dokumente = realEstateInterface.getDokumente(recordFilter=recordFilter if recordFilter else None) - return { - "operation": "READ", - "entity": "Dokument", - "result": [d.model_dump() for d in dokumente], - "count": len(dokumente) - } - else: - raise ValueError(f"READ operation not supported for entity: {entity}") - - elif intent == "UPDATE": - # Update existing entity - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projektId = parameters.get("id") - if not projektId: - raise ValueError("UPDATE operation requires entity ID") - - # Get existing projekt - projekt = realEstateInterface.getProjekt(projektId) - if not projekt: - raise ValueError(f"Projekt {projektId} not found") - - # Update fields - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateProjekt(projektId, updateData) - return { - "operation": "UPDATE", - "entity": "Projekt", - "result": updated.model_dump() - } - elif entity == "Parzelle": - parzelleId = parameters.get("id") - if not parzelleId: - raise ValueError("UPDATE operation requires entity ID") - - # Get existing parzelle - parzelle = realEstateInterface.getParzelle(parzelleId) - if not parzelle: - raise ValueError(f"Parzelle {parzelleId} not found") - - # Update fields - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateParzelle(parzelleId, updateData) - return { - "operation": "UPDATE", - "entity": "Parzelle", - "result": updated.model_dump() - } - elif entity == "Gemeinde": - from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde - gemeindeId = parameters.get("id") - if not gemeindeId: - raise ValueError("UPDATE operation requires entity ID") - - gemeinde = realEstateInterface.getGemeinde(gemeindeId) - if not gemeinde: - raise ValueError(f"Gemeinde {gemeindeId} not found") - - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateGemeinde(gemeindeId, updateData) - return { - "operation": "UPDATE", - "entity": "Gemeinde", - "result": updated.model_dump() - } - elif entity == "Kanton": - from modules.features.realestate.datamodelFeatureRealEstate import Kanton - kantonId = parameters.get("id") - if not kantonId: - raise ValueError("UPDATE operation requires entity ID") - - kanton = realEstateInterface.getKanton(kantonId) - if not kanton: - raise ValueError(f"Kanton {kantonId} not found") - - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateKanton(kantonId, updateData) - return { - "operation": "UPDATE", - "entity": "Kanton", - "result": updated.model_dump() - } - elif entity == "Land": - from modules.features.realestate.datamodelFeatureRealEstate import Land - landId = parameters.get("id") - if not landId: - raise ValueError("UPDATE operation requires entity ID") - - land = realEstateInterface.getLand(landId) - if not land: - raise ValueError(f"Land {landId} not found") - - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateLand(landId, updateData) - return { - "operation": "UPDATE", - "entity": "Land", - "result": updated.model_dump() - } - elif entity == "Dokument": - from modules.features.realestate.datamodelFeatureRealEstate import Dokument - dokumentId = parameters.get("id") - if not dokumentId: - raise ValueError("UPDATE operation requires entity ID") - - dokument = realEstateInterface.getDokument(dokumentId) - if not dokument: - raise ValueError(f"Dokument {dokumentId} not found") - - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateDokument(dokumentId, updateData) - return { - "operation": "UPDATE", - "entity": "Dokument", - "result": updated.model_dump() - } - else: - raise ValueError(f"UPDATE operation not supported for entity: {entity}") - - elif intent == "DELETE": - # Delete entity - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projektId = parameters.get("id") - if not projektId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteProjekt(projektId) - return { - "operation": "DELETE", - "entity": "Projekt", - "success": success - } - elif entity == "Parzelle": - parzelleId = parameters.get("id") - if not parzelleId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteParzelle(parzelleId) - return { - "operation": "DELETE", - "entity": "Parzelle", - "success": success - } - elif entity == "Gemeinde": - from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde - gemeindeId = parameters.get("id") - if not gemeindeId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteGemeinde(gemeindeId) - return { - "operation": "DELETE", - "entity": "Gemeinde", - "success": success - } - elif entity == "Kanton": - from modules.features.realestate.datamodelFeatureRealEstate import Kanton - kantonId = parameters.get("id") - if not kantonId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteKanton(kantonId) - return { - "operation": "DELETE", - "entity": "Kanton", - "success": success - } - elif entity == "Land": - from modules.features.realestate.datamodelFeatureRealEstate import Land - landId = parameters.get("id") - if not landId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteLand(landId) - return { - "operation": "DELETE", - "entity": "Land", - "success": success - } - elif entity == "Dokument": - from modules.features.realestate.datamodelFeatureRealEstate import Dokument - dokumentId = parameters.get("id") - if not dokumentId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteDokument(dokumentId) - return { - "operation": "DELETE", - "entity": "Dokument", - "success": success - } - else: - raise ValueError(f"DELETE operation not supported for entity: {entity}") - - else: - raise ValueError(f"Unknown intent: {intent}") - - except Exception as e: - logger.error(f"Error executing intent-based operation: {str(e)}", exc_info=True) - raise - - -# ===== Project Creation with Parcel Data ===== - -async def create_project_with_parcel_data( - currentUser: User, - mandateId: str, - projekt_label: str, - parzellen_data: List[Dict[str, Any]], - status_prozess: Optional[str] = None, -) -> Dict[str, Any]: - """ - Create a Projekt with one or more Parzellen from provided parcel data. - - Args: - currentUser: Current authenticated user - mandateId: Mandate context (from RequestContext / X-Mandate-Id header) - projekt_label: Label for the Projekt - parzellen_data: List of dictionaries containing parcel information from request - status_prozess: Optional project status (defaults to "Eingang") - - Returns: - Dictionary containing created Projekt and list of Parzellen - - Raises: - HTTPException: If Gemeinde or Kanton not found, or validation fails - """ - try: - logger.info(f"Creating project '{projekt_label}' with {len(parzellen_data)} parcel(s) for user {currentUser.id}") - - # Get interface with mandate context - realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) - - # Validate required fields - if not projekt_label: - raise ValueError("Projekt label is required") - - if not parzellen_data or len(parzellen_data) == 0: - raise ValueError("At least one Parzelle data is required") - - # Validate all parcels have required fields - for idx, parzelle_data in enumerate(parzellen_data): - if not parzelle_data.get("perimeter"): - raise ValueError(f"Parzelle {idx + 1} perimeter is required") - - # Helper function to convert GeoJSON geometry to GeoPolylinie (defined early for use in geometry collection) - def convert_geojson_to_geopolylinie(geometry_data: Dict[str, Any]) -> Optional[GeoPolylinie]: - """Convert GeoJSON geometry to GeoPolylinie format.""" - if not geometry_data: - return None - - # Handle nested geometry structure (geometry.geometry.coordinates) - if "geometry" in geometry_data: - geometry_data = geometry_data["geometry"] - - geometry_type = geometry_data.get("type") - coordinates = geometry_data.get("coordinates") - - if not coordinates or geometry_type != "Polygon": - return None - - # Extract outer ring (first array of coordinates) - if not coordinates or len(coordinates) == 0: - return None - - ring = coordinates[0] # Outer ring - - # Convert coordinates to GeoPunkt list - punkte = [] - for coord in ring: - if len(coord) >= 2: - punkt = GeoPunkt( - koordinatensystem="LV95", - x=float(coord[0]), - y=float(coord[1]), - z=float(coord[2]) if len(coord) > 2 else None - ) - punkte.append(punkt) - - if not punkte: - return None - - return GeoPolylinie( - closed=True, - punkte=punkte - ) - - # First pass: Collect all parcel geometries for neighbor filtering - # Convert all perimeters to GeoPolylinie format - all_parcel_geometries = [] - for parzelle_data in parzellen_data: - perimeter = parzelle_data.get("perimeter") - if perimeter: - # Convert to GeoPolylinie if needed - if isinstance(perimeter, dict): - if "punkte" in perimeter and "closed" in perimeter: - try: - geo_perimeter = GeoPolylinie(**perimeter) - all_parcel_geometries.append(geo_perimeter) - except Exception as e: - logger.warning(f"Error converting perimeter to GeoPolylinie: {e}") - else: - # Try GeoJSON conversion - converted = convert_geojson_to_geopolylinie(perimeter) - if converted: - all_parcel_geometries.append(converted) - elif isinstance(perimeter, GeoPolylinie): - all_parcel_geometries.append(perimeter) - - # Process all parcels - create each one or use existing - created_parzellen = [] - parcel_perimeters = [] # Collect all parcel perimeters for baulinie calculation - - for idx, parzelle_data in enumerate(parzellen_data): - logger.info(f"Processing Parzelle {idx + 1}/{len(parzellen_data)}") - - # Determine parcel label for uniqueness check - parcel_label = parzelle_data.get("id") or parzelle_data.get("number") or parzelle_data.get("label") or "Unknown" - - # Check if Parzelle with this label already exists - existing_parzellen = realEstateInterface.getParzellen( - recordFilter={"label": parcel_label, "mandateId": mandateId} - ) - - if existing_parzellen and len(existing_parzellen) > 0: - # Parzelle already exists - use existing one - existing_parzelle = existing_parzellen[0] - logger.info(f"Parzelle with label '{parcel_label}' already exists (ID: {existing_parzelle.id}), reusing it") - - # Collect perimeter for baulinie calculation - if existing_parzelle.perimeter: - parcel_perimeters.append(existing_parzelle.perimeter) - - # Add to list of created parcels (actually existing) - created_parzellen.append(existing_parzelle) - continue # Skip creation, use existing - - # Parzelle does not exist - create new one - logger.info(f"Parzelle with label '{parcel_label}' does not exist, creating new one") - - # Resolve Gemeinde and Kanton for this parcel (create if not found) - gemeinde_id = None - canton_abk = parzelle_data.get("canton") - municipality_name = parzelle_data.get("municipality_name") - - logger.debug(f"Resolving Gemeinde/Kanton: canton='{canton_abk}', municipality='{municipality_name}'") - - if municipality_name and canton_abk: - # Mapping of canton abbreviations to full names - canton_names = { - "ZH": "Zürich", "BE": "Bern", "LU": "Luzern", "UR": "Uri", "SZ": "Schwyz", - "OW": "Obwalden", "NW": "Nidwalden", "GL": "Glarus", "ZG": "Zug", "FR": "Freiburg", - "SO": "Solothurn", "BS": "Basel-Stadt", "BL": "Basel-Landschaft", "SH": "Schaffhausen", - "AR": "Appenzell Ausserrhoden", "AI": "Appenzell Innerrhoden", "SG": "St. Gallen", - "GR": "Graubünden", "AG": "Aargau", "TG": "Thurgau", "TI": "Tessin", - "VD": "Waadt", "VS": "Wallis", "NE": "Neuenburg", "GE": "Genf", "JU": "Jura" - } - - # First, ensure Land "Schweiz" exists - logger.debug("Ensuring Land 'Schweiz' exists") - laender = realEstateInterface.getLaender(recordFilter={"label": "Schweiz"}) - if not laender: - logger.info("Creating Land 'Schweiz'") - land = Land( - mandateId=mandateId, - label="Schweiz", - abk="CH" - ) - land = realEstateInterface.createLand(land) - logger.info(f"Created Land 'Schweiz' with ID: {land.id}") - else: - land = laender[0] - logger.debug(f"Found Land 'Schweiz' with ID: {land.id}") - - # Then, lookup or create Kanton - logger.debug(f"Looking up Kanton with abk='{canton_abk}'") - kantone = realEstateInterface.getKantone(recordFilter={"abk": canton_abk}) - logger.debug(f"Found {len(kantone)} Kanton(e) with abk='{canton_abk}'") - if not kantone: - logger.info(f"Kanton '{canton_abk}' not found, creating it") - kanton_label = canton_names.get(canton_abk, canton_abk) # Use mapping or fallback to abk - kanton = Kanton( - mandateId=mandateId, - label=kanton_label, - abk=canton_abk, - id_land=land.id - ) - kanton = realEstateInterface.createKanton(kanton) - logger.info(f"Created Kanton '{kanton_label}' ({canton_abk}) with ID: {kanton.id}") - else: - kanton = kantone[0] - logger.debug(f"Found Kanton: ID={kanton.id}, Label={kanton.label}, abk={kanton.abk}") - - # Then, lookup or create Gemeinde - logger.debug(f"Looking up Gemeinde with label='{municipality_name}' and id_kanton='{kanton.id}'") - gemeinden = realEstateInterface.getGemeinden( - recordFilter={"label": municipality_name, "id_kanton": kanton.id} - ) - logger.debug(f"Found {len(gemeinden)} Gemeinde(n) with label='{municipality_name}' and id_kanton='{kanton.id}'") - if not gemeinden: - logger.info(f"Gemeinde '{municipality_name}' in Kanton '{canton_abk}' not found, creating it") - gemeinde = Gemeinde( - mandateId=mandateId, - label=municipality_name, - id_kanton=kanton.id, - plz=parzelle_data.get("plz") # Use PLZ directly from Swiss Topo API - ) - gemeinde = realEstateInterface.createGemeinde(gemeinde) - logger.info(f"Created Gemeinde '{municipality_name}' with ID: {gemeinde.id}") - else: - gemeinde = gemeinden[0] - logger.debug(f"Found Gemeinde: ID={gemeinde.id}, Label={gemeinde.label}") - - gemeinde_id = gemeinde.id - logger.info(f"Resolved Gemeinde '{municipality_name}' to ID '{gemeinde_id}'") - else: - logger.warning(f"Missing Gemeinde/Kanton data: municipality_name={municipality_name}, canton={canton_abk}") - - # Build parzellenAliasTags - alias_tags = [] - if parzelle_data.get("egrid"): - alias_tags.append(parzelle_data["egrid"]) - if parzelle_data.get("number") and parzelle_data["number"] != parzelle_data.get("id"): - alias_tags.append(parzelle_data["number"]) - - # Extract address information from Swiss Topo API data - # Each parcel should have its own address data from Swiss Topo API - # The address comes from the parcel search API response for THIS specific parcel - strasse_nr = None - plz = None - - # Use address from Swiss Topo API - this is specific to THIS parcel - # The address field contains the full address string from Swiss Topo - address = parzelle_data.get("address") - if address: - # Swiss Topo provides full address string like "Street Number, PLZ City" - # Parse to extract street and number (before comma) - parts = address.split(",") - if len(parts) >= 1: - strasse_nr = parts[0].strip() - # PLZ is provided separately by Swiss Topo API - plz = parzelle_data.get("plz") - - # Log address info for debugging - logger.debug(f"Parzelle {idx + 1} address data: strasse_nr='{strasse_nr}', plz='{plz}', full_address='{address}'") - - # If no address found, log warning but continue - if not strasse_nr and not plz: - logger.warning(f"No address data found for Parzelle {idx + 1} (label: {parcel_label})") - - # Build kontextInformationen - kontext_items = [] - - if parzelle_data.get("egrid"): - kontext_items.append(Kontext( - thema="EGRID", - inhalt=parzelle_data["egrid"] - )) - - if parzelle_data.get("identnd"): - kontext_items.append(Kontext( - thema="IdentND", - inhalt=parzelle_data["identnd"] - )) - - if parzelle_data.get("area_m2"): - kontext_items.append(Kontext( - thema="Fläche", - inhalt=f"{parzelle_data['area_m2']} m²" - )) - - if parzelle_data.get("centroid"): - centroid = parzelle_data["centroid"] - kontext_items.append(Kontext( - thema="Zentrum (LV95)", - inhalt=f"X: {centroid.get('x')} m, Y: {centroid.get('y')} m (EPSG:2056)" - )) - - if parzelle_data.get("geoportal_url"): - kontext_items.append(Kontext( - thema="Geoportal URL", - inhalt=parzelle_data["geoportal_url"] - )) - - if parzelle_data.get("municipality_code"): - kontext_items.append(Kontext( - thema="BFS-Nummer", - inhalt=str(parzelle_data["municipality_code"]) - )) - - # Handle adjacent parcels - filter out selected parcels geometrically - adjacent_parcel_refs = [] - if parzelle_data.get("adjacent_parcels"): - # Filter neighbors to exclude selected parcels - neighbors_to_filter = [] - for adj_parcel in parzelle_data["adjacent_parcels"]: - if isinstance(adj_parcel, dict): - neighbors_to_filter.append(adj_parcel) - elif isinstance(adj_parcel, str): - neighbors_to_filter.append({"id": adj_parcel}) - - # Filter using geometry comparison if we have geometries - if all_parcel_geometries and neighbors_to_filter: - try: - filtered_neighbors = filter_neighbor_parcels( - neighbors_to_filter, - all_parcel_geometries - ) - # Extract IDs from filtered neighbors - for filtered_neighbor in filtered_neighbors: - adj_id = filtered_neighbor.get("id") - if adj_id: - adjacent_parcel_refs.append({"id": adj_id}) - except Exception as e: - logger.warning(f"Error filtering neighbor parcels: {e}, including all neighbors") - # Fallback: include all neighbors if filtering fails - for adj_parcel in parzelle_data["adjacent_parcels"]: - if isinstance(adj_parcel, dict): - adj_id = adj_parcel.get("id") - if adj_id: - adjacent_parcel_refs.append({"id": adj_id}) - elif isinstance(adj_parcel, str): - adjacent_parcel_refs.append({"id": adj_parcel}) - else: - # No geometries available - include all neighbors - for adj_parcel in parzelle_data["adjacent_parcels"]: - if isinstance(adj_parcel, dict): - adj_id = adj_parcel.get("id") - if adj_id: - adjacent_parcel_refs.append({"id": adj_id}) - elif isinstance(adj_parcel, str): - adjacent_parcel_refs.append({"id": adj_parcel}) - - # Convert perimeter to GeoPolylinie if needed - perimeter = parzelle_data.get("perimeter") - if isinstance(perimeter, dict): - # Check if it's already in GeoPolylinie format (has punkte and closed) - if "punkte" in perimeter and "closed" in perimeter: - try: - perimeter = GeoPolylinie(**perimeter) - except Exception as e: - raise ValueError(f"Invalid perimeter format: {str(e)}") - else: - # Try to convert from GeoJSON format - converted = convert_geojson_to_geopolylinie(perimeter) - if converted: - perimeter = converted - else: - raise ValueError("Invalid perimeter format: cannot convert to GeoPolylinie") - elif isinstance(perimeter, GeoPolylinie): - # Already a GeoPolylinie instance, use as-is - pass - else: - raise ValueError("Invalid perimeter type: must be dict or GeoPolylinie") - - # Extract baulinie from geometry if provided - baulinie = None - geometry = parzelle_data.get("geometry") - logger.debug(f"Geometry present: {geometry is not None}") - if geometry: - logger.debug(f"Geometry type: {type(geometry)}, keys: {list(geometry.keys()) if isinstance(geometry, dict) else 'not a dict'}") - baulinie = convert_geojson_to_geopolylinie(geometry) - if baulinie: - logger.info(f"Extracted baulinie from geometry with {len(baulinie.punkte)} points") - else: - logger.warning("Failed to extract baulinie from geometry") - else: - logger.warning("No geometry found in parzelle_data") - - # Build Parzelle data - parzelle_create_data = { - "mandateId": mandateId, - "label": parcel_label, # Use the label we determined earlier for uniqueness check - "parzellenAliasTags": alias_tags, - "eigentuemerschaft": None, - "strasseNr": strasse_nr, - "plz": plz, - "perimeter": perimeter, - "baulinie": baulinie, - "kontextGemeinde": gemeinde_id, - "bauzone": None, - "az": None, - "bz": None, - "vollgeschossZahl": None, - "anrechenbarDachgeschoss": None, - "anrechenbarUntergeschoss": None, - "gebaeudehoeheMax": None, - "regelnGrenzabstand": [], - "regelnMehrlaengenzuschlag": [], - "regelnMehrhoehenzuschlag": [], - "parzelleBebaut": None, - "parzelleErschlossen": None, - "parzelleHanglage": None, - "laermschutzzone": None, - "hochwasserschutzzone": None, - "grundwasserschutzzone": None, - "parzellenNachbarschaft": adjacent_parcel_refs, - "dokumente": [], - "kontextInformationen": kontext_items, - } - - # Create Parzelle instance - logger.debug(f"Creating Parzelle with label: {parzelle_create_data.get('label')}") - logger.debug(f"Parzelle mandateId: {parzelle_create_data.get('mandateId')}") - logger.debug(f"Parzelle perimeter present: {parzelle_create_data.get('perimeter') is not None}") - - try: - parzelle_instance = Parzelle(**parzelle_create_data) - logger.debug(f"Parzelle instance created successfully with ID: {parzelle_instance.id}") - except Exception as e: - logger.error(f"Error creating Parzelle instance: {str(e)}", exc_info=True) - raise - - # Create Parzelle in database - try: - logger.info(f"Calling createParzelle for Parzelle '{parzelle_instance.label}' (ID: {parzelle_instance.id})") - logger.debug(f"Parzelle instance before createParzelle: {parzelle_instance.model_dump(mode='json', exclude={'perimeter', 'baulinie', 'kontextInformationen'})}") - - # Use model_dump with mode='json' to ensure nested Pydantic models are serialized - parzelle_dict = parzelle_instance.model_dump(mode='json') - logger.debug(f"Parzelle dict keys: {list(parzelle_dict.keys())}") - - # Create Parzelle using the interface, which will handle serialization - created_parzelle = realEstateInterface.createParzelle(parzelle_instance) - - logger.info(f"createParzelle returned: ID={created_parzelle.id if created_parzelle else 'None'}, Label={created_parzelle.label if created_parzelle else 'None'}") - - # Verify Parzelle was created successfully - if not created_parzelle: - raise ValueError("Failed to create Parzelle - createParzelle returned None") - - if not created_parzelle.id: - raise ValueError("Failed to create Parzelle - no ID returned") - - logger.info(f"Parzelle created with ID: {created_parzelle.id}") - - # Verify Parzelle exists in database by fetching it - logger.debug(f"Verifying Parzelle {created_parzelle.id} exists in database...") - verify_parzelle = realEstateInterface.getParzelle(created_parzelle.id) - if not verify_parzelle: - logger.error(f"Parzelle {created_parzelle.id} was not found in database after creation") - # Try to get all Parzellen to see what's in the database - all_parzellen = realEstateInterface.getParzellen(recordFilter=None) - logger.error(f"Total Parzellen in database: {len(all_parzellen)}") - if all_parzellen: - logger.error(f"Sample Parzelle IDs: {[p.id for p in all_parzellen[:5]]}") - raise ValueError(f"Parzelle {created_parzelle.id} was not found in database after creation") - - logger.info(f"Verified Parzelle {created_parzelle.id} exists in database") - # Use the verified Parzelle from database to ensure it has all fields - created_parzelle = verify_parzelle - - # Collect perimeter for baulinie calculation - if created_parzelle.perimeter: - parcel_perimeters.append(created_parzelle.perimeter) - - # Add to list of created parcels - created_parzellen.append(created_parzelle) - - except Exception as e: - logger.error(f"Error creating Parzelle {idx + 1}: {str(e)}", exc_info=True) - raise - - if not created_parzellen: - raise ValueError("No Parzellen were successfully created") - - logger.info(f"Successfully created {len(created_parzellen)} Parzelle(n)") - - # Calculate combined baulinie from all parcel perimeters - project_baulinie = None - if len(parcel_perimeters) > 0: - try: - if len(parcel_perimeters) == 1: - # Single parcel - use its perimeter as baulinie - project_baulinie = parcel_perimeters[0] - logger.info("Using single parcel perimeter as baulinie") - else: - # Multiple parcels - combine geometries to create outer outline - logger.info(f"Combining {len(parcel_perimeters)} parcel geometries to create baulinie") - project_baulinie = combine_parcel_geometries(parcel_perimeters) - logger.info(f"Created combined baulinie with {len(project_baulinie.punkte)} points") - except Exception as e: - logger.error(f"Error combining parcel geometries for baulinie: {e}", exc_info=True) - # Fallback: use first parcel's perimeter - if parcel_perimeters: - project_baulinie = parcel_perimeters[0] - logger.warning("Using first parcel perimeter as fallback baulinie") - - # Convert status_prozess to enum - status_prozess_enum = None - if status_prozess: - try: - # Try to convert string to enum - if isinstance(status_prozess, str): - status_prozess_enum = StatusProzess(status_prozess) - elif isinstance(status_prozess, StatusProzess): - status_prozess_enum = status_prozess - except (ValueError, KeyError): - logger.warning(f"Invalid statusProzess '{status_prozess}', using default 'Eingang'") - status_prozess_enum = StatusProzess.EINGANG - else: - status_prozess_enum = StatusProzess.EINGANG - - # Create Projekt with combined baulinie - # Use the verified Parzelle instance (from database) to ensure it has all fields properly set - logger.debug(f"Preparing Projekt creation with baulinie: {project_baulinie is not None}") - if project_baulinie: - logger.debug(f"Baulinie has {len(project_baulinie.punkte)} points") - - # Use first parcel's perimeter for project perimeter (or combine if needed) - project_perimeter = created_parzellen[0].perimeter if created_parzellen else None - - projekt_create_data = { - "mandateId": mandateId, - "label": projekt_label, - "statusProzess": status_prozess_enum, - "perimeter": project_perimeter, # Use first parcel perimeter as project perimeter - "baulinie": project_baulinie, # Set baulinie from first parcel geometry - "parzellen": created_parzellen, # Link all created Parzelle instances - "dokumente": [], - "kontextInformationen": [], - } - - logger.debug(f"Projekt data prepared: label={projekt_label}, parzellen_count={len(projekt_create_data['parzellen'])}, baulinie={'present' if project_baulinie else 'None'}") - - try: - projekt_instance = Projekt(**projekt_create_data) - logger.debug(f"Projekt instance created successfully with ID: {projekt_instance.id}") - except Exception as e: - logger.error(f"Error creating Projekt instance: {str(e)}", exc_info=True) - raise - - # Log before creation for debugging - logger.debug(f"Creating Projekt with {len(projekt_instance.parzellen)} Parzelle(n)") - if projekt_instance.parzellen: - for idx, p in enumerate(projekt_instance.parzellen): - logger.debug(f" Parzelle {idx}: ID={p.id}, Label={p.label}") - - logger.debug(f"Projekt baulinie before save: {projekt_instance.baulinie is not None}") - if projekt_instance.baulinie: - logger.debug(f"Projekt baulinie has {len(projekt_instance.baulinie.punkte)} points") - - try: - created_projekt = realEstateInterface.createProjekt(projekt_instance) - logger.info(f"Created Projekt '{created_projekt.label}' (ID: {created_projekt.id})") - logger.debug(f"Created Projekt baulinie: {created_projekt.baulinie is not None}") - except Exception as e: - logger.error(f"Error calling createProjekt: {str(e)}", exc_info=True) - raise - - # Verify Projekt was created - if not created_projekt or not created_projekt.id: - raise ValueError("Failed to create Projekt - no ID returned") - - # Verify Parzelle is linked in the created Projekt - if not created_projekt.parzellen or len(created_projekt.parzellen) == 0: - logger.warning(f"Projekt {created_projekt.id} created but no Parzellen linked") - # Try to fetch the Projekt from database to see if Parzellen are there - verify_projekt = realEstateInterface.getProjekt(created_projekt.id) - if verify_projekt and verify_projekt.parzellen: - logger.info(f"Parzellen found when fetching Projekt from database: {len(verify_projekt.parzellen)}") - created_projekt = verify_projekt - else: - raise ValueError(f"Projekt {created_projekt.id} has no Parzellen linked after creation") - else: - logger.info(f"Projekt {created_projekt.id} successfully linked to {len(created_projekt.parzellen)} Parzelle(n)") - # Log Parzelle details - for idx, p in enumerate(created_projekt.parzellen): - logger.debug(f" Linked Parzelle {idx}: ID={p.id if hasattr(p, 'id') else 'NO ID'}, Label={p.label if hasattr(p, 'label') else 'NO LABEL'}") - - return { - "projekt": created_projekt.model_dump(), - "parzellen": [p.model_dump() for p in created_parzellen], - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error creating project with parcel data: {str(e)}", exc_info=True) - raise - - -# ===== BZO Information Extraction for Parcels ===== - -async def extract_bzo_information( - currentUser: User, - gemeinde: str, - bauzone: str, - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - total_area_m2: Optional[float] = None, - parcels: Optional[List[Dict[str, Any]]] = None, -) -> Dict[str, Any]: - """ - Extract BZO information from PDF documents for a specific Bauzone in a Gemeinde. - - Retrieves BZO documents for the specified Gemeinde, extracts content using - the BZO extraction pipeline, filters by Bauzone, and uses AI to find relevant information. - When total_area_m2 or parcels are provided, runs Machbarkeitsstudie for structured output. - - Args: - currentUser: Current authenticated user - gemeinde: Gemeinde name (e.g., "Zürich") or ID - bauzone: Bauzone code (e.g., "W3", "W2/30") - mandateId: Optional mandate ID for instance-scoped data (defaults to currentUser.mandateId) - featureInstanceId: Optional feature instance ID for instance-scoped data - total_area_m2: Optional total parcel area (m²) for Machbarkeitsstudie - parcels: Optional list of parcel dicts; total area computed via compute_selection_summary if not total_area_m2 - - Returns: - Dictionary containing: - - bauzone, gemeinde, extracted_content, ai_summary, relevant_rules, documents_processed - - machbarkeitsstudie: Structured Machbarkeitsstudie output when total_area_m2/parcels provided - """ - try: - _mandateId = mandateId or (str(currentUser.mandateId) if currentUser.mandateId else None) - logger.info(f"Extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}' (user: {currentUser.id}, mandate: {_mandateId})") - - # Get interfaces (instance-scoped when mandateId/featureInstanceId provided) - realEstateInterface = getRealEstateInterface( - currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId - ) - componentInterface = getComponentInterface( - currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId - ) - - # Get Gemeinde - try by ID first, then by label - logger.debug(f"Attempting to retrieve Gemeinde '{gemeinde}' for mandate {_mandateId}") - gemeinde_obj = realEstateInterface.getGemeinde(gemeinde) - - # If not found by ID, try searching by label - if not gemeinde_obj: - logger.debug(f"Gemeinde not found by ID, trying to search by label: {gemeinde}") - record_filter = {"label": gemeinde} - if _mandateId: - record_filter["mandateId"] = _mandateId - gemeinden_by_label = realEstateInterface.getGemeinden( - recordFilter=record_filter - ) - if gemeinden_by_label and len(gemeinden_by_label) > 0: - gemeinde_obj = gemeinden_by_label[0] - logger.info(f"Found Gemeinde by label '{gemeinde}' with ID: {gemeinde_obj.id}") - - # If still not found: fetch only this Gemeinde from Swiss Topo and create it - if not gemeinde_obj and _mandateId and featureInstanceId: - logger.info(f"Gemeinde '{gemeinde}' not in DB - fetching from Swiss Topo (this Gemeinde only)") - gemeinde_obj = await ensure_single_gemeinde( - realEstateInterface, _mandateId, featureInstanceId, gemeinde_name=gemeinde - ) - - if not gemeinde_obj: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Gemeinde '{gemeinde}' not found or not accessible" - ) - - gemeinde_id = gemeinde_obj.id - - # Get BZO documents directly from Gemeinde's dokumente field - bzo_documents = [] - if gemeinde_obj.dokumente: - for doc in gemeinde_obj.dokumente: - # Handle both dict and object formats - if isinstance(doc, dict): - doc_id = doc.get("id") - doc_typ = doc.get("dokumentTyp") - else: - doc_id = doc.id if hasattr(doc, "id") else None - doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None - - # Check if it's a BZO document type - if doc_typ: - # Handle enum, string, or dict formats - if isinstance(doc_typ, DokumentTyp): - is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] - elif isinstance(doc_typ, str): - is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] - else: - doc_typ_str = str(doc_typ) - is_bzo = doc_typ_str in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] - - if is_bzo: - # Get full document object - if doc_id: - full_doc = realEstateInterface.getDokument(doc_id) - if full_doc: - bzo_documents.append(full_doc) - else: - logger.warning(f"Document {doc_id} referenced in Gemeinde but not found in database") - - # If no BZO documents: auto-fetch from Tavily, then retry - if not bzo_documents and _mandateId and featureInstanceId: - logger.info(f"No BZO documents for Gemeinde '{gemeinde_obj.label}' - fetching from web") - fetched = await fetch_bzo_for_gemeinde( - realEstateInterface, componentInterface, gemeinde_obj, _mandateId, featureInstanceId - ) - if fetched: - # Reload Gemeinde to get updated dokumente - gemeinde_obj = realEstateInterface.getGemeinde(gemeinde_obj.id) - bzo_documents = [] - if gemeinde_obj and gemeinde_obj.dokumente: - for doc in gemeinde_obj.dokumente: - if isinstance(doc, dict): - doc_id = doc.get("id") - doc_typ = doc.get("dokumentTyp") - else: - doc_id = doc.id if hasattr(doc, "id") else None - doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None - if doc_typ: - if isinstance(doc_typ, DokumentTyp): - is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] - elif isinstance(doc_typ, str): - is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] - else: - is_bzo = str(doc_typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] - if is_bzo and doc_id: - full_doc = realEstateInterface.getDokument(doc_id) - if full_doc: - bzo_documents.append(full_doc) - - if not bzo_documents: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"No BZO documents found for Gemeinde '{gemeinde_obj.label}'" - ) - - logger.info(f"Found {len(bzo_documents)} BZO document(s) for Gemeinde '{gemeinde_obj.label}'") - - # Initialize document retriever - document_retriever = BZODocumentRetriever(realEstateInterface, componentInterface) - - # Extract content from all documents - all_extracted_content = { - "articles": [], - "zones": [], - "rules": [], - "zone_parameter_tables": [], - "errors": [], - "warnings": [] - } - documents_processed = [] - - for dokument in bzo_documents: - try: - logger.info(f"Processing document {dokument.id}: {dokument.label}") - - # Retrieve PDF content - pdf_bytes = document_retriever.retrieve_pdf_content(dokument) - if not pdf_bytes: - logger.warning(f"Could not retrieve PDF content for dokument {dokument.id}") - all_extracted_content["warnings"].append( - f"Could not retrieve PDF content for document '{dokument.label}'" - ) - continue - - # Run extraction using the BZO extraction pipeline - extraction_result = run_extraction( - pdf_bytes=pdf_bytes, - pdf_id=dokument.dokumentReferenz or f"dok_{dokument.id}", - dokument_id=dokument.id - ) - - # Combine results - all_extracted_content["articles"].extend(extraction_result.get("articles", [])) - all_extracted_content["zones"].extend(extraction_result.get("zones", [])) - all_extracted_content["rules"].extend(extraction_result.get("rules", [])) - all_extracted_content["zone_parameter_tables"].extend(extraction_result.get("zone_parameter_tables", [])) - all_extracted_content["errors"].extend(extraction_result.get("errors", [])) - all_extracted_content["warnings"].extend(extraction_result.get("warnings", [])) - - documents_processed.append({ - "id": dokument.id, - "label": dokument.label, - "dokumentTyp": dokument.dokumentTyp.value if dokument.dokumentTyp else None - }) - - except Exception as e: - logger.error(f"Error processing document {dokument.id}: {str(e)}", exc_info=True) - all_extracted_content["errors"].append( - f"Error processing document '{dokument.label}': {str(e)}" - ) - continue - - # Filter rules by Bauzone - only rules explicitly associated with this zone - relevant_rules = filter_rules_by_bauzone( - all_extracted_content["rules"], - bauzone - ) - logger.info(f"Extracting for Bauzone {bauzone}: {len(relevant_rules)} zone-specific rules, " - f"{len([t for t in all_extracted_content.get('zone_parameter_tables', []) if bauzone.upper() in str(t.get('zones', [])).upper()])} tables with zone data") - - # Filter zones by Bauzone - relevant_zones = filter_zones_by_bauzone( - all_extracted_content["zones"], - bauzone - ) - - # Filter articles that mention the Bauzone - relevant_articles = filter_articles_by_bauzone( - all_extracted_content.get("articles", []), - bauzone - ) - - # Compute total_area_m2 from parcels if not provided - _total_area_m2 = total_area_m2 - if _total_area_m2 is None and parcels: - selection_summary = compute_selection_summary(parcels) - _total_area_m2 = selection_summary.get("total_area_m2") or 0.0 - - # Extract BZO parameters for Wohnzone via LLM (bullet list with sources) - bzo_params_result = None - try: - services = getServices( - currentUser, workflow=None, mandateId=_mandateId, featureInstanceId=featureInstanceId - ) - ai_service = services.ai - bzo_params_result = await run_bzo_params_extraction( - extracted_content=all_extracted_content, - bauzone=bauzone, - ai_service=ai_service, - gemeinde=gemeinde_obj.label, - relevant_rules=relevant_rules, - relevant_articles=relevant_articles, - total_area_m2=_total_area_m2, - ) - except Exception as me: - logger.warning(f"BZO parameter extraction failed: {me}", exc_info=True) - all_extracted_content["warnings"] = all_extracted_content.get("warnings", []) + [ - f"BZO-Parameter konnten nicht extrahiert werden: {str(me)}" - ] - - # Use AI to generate summary and find additional information - ai_summary = await generate_bauzone_ai_summary( - currentUser=currentUser, - bauzone=bauzone, - gemeinde=gemeinde_obj.label, - extracted_content=all_extracted_content, - relevant_rules=relevant_rules, - relevant_zones=relevant_zones, - mandateId=_mandateId, - featureInstanceId=featureInstanceId, - ) - - # Build unified summary that includes zones and articles - unified_summary = ai_summary - - # Append zone and article information to the summary if not already included - # The AI should have integrated this, but we add it as backup if needed - summary_lower = unified_summary.lower() - - # Check if zones are mentioned in summary - zones_mentioned = any(zone.get("zone_code", "").upper() in summary_lower for zone in relevant_zones) - if not zones_mentioned and relevant_zones: - unified_summary += "\n\n=== ZONENDEFINITIONEN ===\n" - for zone in relevant_zones: - zone_code = zone.get("zone_code", "") - zone_name = zone.get("zone_name", "") - zone_category = zone.get("zone_category", "") - geschosszahl = zone.get("geschosszahl") - gewerbeerleichterung = zone.get("gewerbeerleichterung", False) - page_num = zone.get("page", 0) - source_article = zone.get("source_article", "") - - zone_info = f"{zone_code}: {zone_name}" - if zone_category: - zone_info += f"\nKategorie: {zone_category}" - if geschosszahl: - zone_info += f"\nGeschosszahl: {geschosszahl}" - if gewerbeerleichterung: - zone_info += "\nGewerbeerleichterung: Ja" - if source_article: - zone_info += f"\nQuelle: {source_article} (Seite {page_num})" - unified_summary += zone_info + "\n\n" - - # Check if articles are mentioned in summary - articles_mentioned = any(article.get("article_label", "") in summary_lower for article in relevant_articles) - if not articles_mentioned and relevant_articles: - unified_summary += "\n\n=== RELEVANTE ARTIKEL ===\n" - for article in relevant_articles: - article_label = article.get("article_label", "") - article_title = article.get("article_title", "") - article_text = article.get("text", "") - page_start = article.get("page_start", 0) - page_end = article.get("page_end", 0) - page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}" - - unified_summary += f"{article_label}" - if article_title: - unified_summary += f": {article_title}" - unified_summary += f" ({page_range})\n" - # Include first 500 chars of article text - if article_text: - preview = article_text[:500] + "..." if len(article_text) > 500 else article_text - unified_summary += f"{preview}\n\n" - - return { - "bauzone": bauzone, - "gemeinde": { - "id": gemeinde_obj.id, - "label": gemeinde_obj.label, - "plz": gemeinde_obj.plz - }, - "extracted_content": { - "zones": relevant_zones, - "rules": relevant_rules, - "articles": relevant_articles, - "zone_parameter_tables": _filter_tables_by_bauzone( - all_extracted_content.get("zone_parameter_tables", []), - bauzone - ), - "total_zones": len(all_extracted_content.get("zones", [])), - "total_rules": len(all_extracted_content.get("rules", [])), - "total_articles": len(all_extracted_content.get("articles", [])), - "total_tables": len(all_extracted_content.get("zone_parameter_tables", [])) - }, - "ai_summary": unified_summary, - "relevant_rules": relevant_rules, - "documents_processed": documents_processed, - "errors": all_extracted_content.get("errors", []), - "warnings": all_extracted_content.get("warnings", []), - "machbarkeitsstudie": bzo_params_result, # Same key for frontend compatibility - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}': {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error extracting BZO information: {str(e)}" - ) - - -def filter_rules_by_bauzone(rules: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: - """ - Filter rules by Bauzone code. Only keeps rules from SINGLE-zone articles to avoid - wrong values (e.g. article with W2,W3,W5 has different values per zone - we cannot - associate a rule value with a specific zone from article text alone). - """ - relevant_rules = [] - bauzone_upper = bauzone.upper() - - def _zone_matches(z: str) -> bool: - zu = (z or "").upper().strip() - if not zu: - return False - if bauzone_upper in zu: - return True - if zu in bauzone_upper and len(zu) >= 2: - return True - return False - - for rule in rules: - table_zones = rule.get("table_zones", []) or [] - zone_raw = rule.get("zone_raw") - - # Rule must be zone-associated - has_zone = bool(zone_raw) or bool(table_zones) - if not has_zone: - continue - - # CRITICAL: Only use rules from single-zone articles. Multi-zone articles - # (e.g. table with W2,W3,W5) have different values per zone - we cannot - # know which value applies to our zone from article text. - if len(table_zones) > 1: - # Check if ALL zones in article match our bauzone (e.g. W5, W5/50) - unlikely - matches_all = all(_zone_matches(str(z)) for z in table_zones) - if not matches_all: - continue # Ambiguous: exclude - - # Zone must match our bauzone - matches = False - if zone_raw and _zone_matches(zone_raw): - matches = True - if not matches and table_zones: - for tz in table_zones: - if _zone_matches(str(tz)): - matches = True - break - if not matches: - ts = (rule.get("text_snippet") or "").upper() - if bauzone_upper in ts and len(table_zones) <= 1: - matches = True - - if matches: - relevant_rules.append(rule) - - logger.info(f"Filtered {len(relevant_rules)} rules for Bauzone {bauzone} from {len(rules)} total (multi-zone articles excluded)") - return relevant_rules - - -def filter_zones_by_bauzone(zones: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: - """ - Filter zones by Bauzone code. - - Args: - zones: List of zone dictionaries from extraction - bauzone: Bauzone code to filter by - - Returns: - Filtered list of zones that match the Bauzone - """ - relevant_zones = [] - bauzone_upper = bauzone.upper() - - for zone in zones: - zone_code = zone.get("zone_code", "") - if bauzone_upper in zone_code.upper(): - relevant_zones.append(zone) - - logger.info(f"Filtered {len(relevant_zones)} zones for Bauzone {bauzone} from {len(zones)} total zones") - return relevant_zones - - -def filter_articles_by_bauzone(articles: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: - """ - Filter articles that mention the Bauzone. - - Args: - articles: List of article dictionaries from extraction - bauzone: Bauzone code to filter by - - Returns: - Filtered list of articles that mention the Bauzone - """ - relevant_articles = [] - bauzone_upper = bauzone.upper() - - for article in articles: - text = article.get("text", "") - zone_raw = article.get("zone_raw") - - # Check if article mentions the Bauzone - text_matches = bauzone_upper in text.upper() if text else False - zone_matches = bauzone_upper in zone_raw.upper() if zone_raw else False - - if text_matches or zone_matches: - relevant_articles.append(article) - - logger.info(f"Filtered {len(relevant_articles)} articles for Bauzone {bauzone} from {len(articles)} total articles") - return relevant_articles - - -def _filter_tables_by_bauzone(tables: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: - """ - Filter zone-parameter tables to include only those containing the specified Bauzone. - - Args: - tables: List of zone-parameter table dictionaries - bauzone: Bauzone code to filter by - - Returns: - Filtered list of tables containing the Bauzone - """ - relevant_tables = [] - bauzone_upper = bauzone.upper() - - for table in tables: - zones = table.get("zones", []) - # Check if any zone in the table matches the Bauzone - matching_zones = [z for z in zones if bauzone_upper in str(z).upper()] - - if matching_zones: - # Create filtered version with only relevant zone columns - filtered_table = { - "page": table.get("page"), - "zones": matching_zones, - "parameters": [] - } - - # Filter parameters to only include values for matching zones - for param in table.get("parameters", []): - values_by_zone = param.get("values_by_zone", {}) - filtered_values = { - zone: values_by_zone[zone] - for zone in matching_zones - if zone in values_by_zone - } - - if filtered_values: - filtered_table["parameters"].append({ - "parameter": param.get("parameter"), - "values_by_zone": filtered_values - }) - - if filtered_table["parameters"]: - relevant_tables.append(filtered_table) - - logger.info(f"Filtered {len(relevant_tables)} tables for Bauzone {bauzone} from {len(tables)} total tables") - return relevant_tables - - -async def generate_bauzone_ai_summary( - currentUser: User, - bauzone: str, - gemeinde: str, - extracted_content: Dict[str, Any], - relevant_rules: List[Dict[str, Any]], - relevant_zones: List[Dict[str, Any]], - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, -) -> str: - """ - Use AI to generate a summary of relevant information for a Bauzone. - - Args: - currentUser: Current authenticated user - bauzone: Bauzone code - gemeinde: Gemeinde name - extracted_content: All extracted content from PDFs - relevant_rules: Rules filtered by Bauzone - relevant_zones: Zones filtered by Bauzone - - Returns: - AI-generated summary string - """ - try: - # Initialize AI service (mandateId required for billing) - services = getServices( - currentUser, workflow=None, mandateId=mandateId, featureInstanceId=featureInstanceId - ) - aiService = services.ai - - # Build context from extracted content, prioritizing zone-parameter tables - context_parts = [] - - # Extract and format zone-parameter table values for the specific Bauzone - zone_parameter_tables = extracted_content.get("zone_parameter_tables", []) - table_values_for_bauzone = [] - - if zone_parameter_tables: - context_parts.append("=== BUILDING REGULATIONS TABLE VALUES FOR BAUZONE (INCLUDE THESE EXACT VALUES IN YOUR SUMMARY) ===") - for table in zone_parameter_tables: - page_num = table.get("page", 0) - article_ref = table.get("article", "Unknown article") - zones_in_table = table.get("zones", []) - - # Check if this table contains the requested Bauzone - matching_zones = [z for z in zones_in_table if bauzone.upper() in str(z).upper()] - - if matching_zones: - context_parts.append(f"\nTabelle aus {article_ref} (Seite {page_num}):") - - for param in table.get("parameters", []): - param_name = param.get("parameter", "") - values_by_zone = param.get("values_by_zone", {}) - - # Extract values for the requested Bauzone - for zone, values in values_by_zone.items(): - if bauzone.upper() in zone.upper(): - if isinstance(values, list) and len(values) > 0: - # Take the first value (most relevant) - val_entry = values[0] - value = val_entry.get("value", "") - unit = val_entry.get("unit", "") - unit_str = f" {unit}" if unit else "" - - # Format parameter name nicely - formatted_param = param_name - if "Ausnützungsziffer" in param_name or "ausnützungsziffer" in param_name.lower(): - formatted_param = "Ausnützungsziffer max." - elif "Vollgeschosse" in param_name or "vollgeschosse" in param_name.lower(): - formatted_param = "Vollgeschosse max." - elif "Gebäudelänge" in param_name or "gebäudelänge" in param_name.lower(): - formatted_param = "Gebäudelänge max." - elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Grundabstand" in param_name or "grundabstand" in param_name.lower()): - formatted_param = "Grenzabstand - Grundabstand min." - elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Mehrlängen" in param_name or "mehrlängen" in param_name.lower()): - formatted_param = "Grenzabstand - Mehrlängen-zuschlag" - elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Höchstmass" in param_name or "höchstmass" in param_name.lower() or "Höchstmaß" in param_name): - formatted_param = "Grenzabstand - Höchstmass max." - elif "Fassadenhöhen" in param_name or "fassadenhöhen" in param_name.lower(): - formatted_param = "Fassadenhöhen max." - elif "Dachgeschosse" in param_name or "dachgeschosse" in param_name.lower(): - formatted_param = "anrechenbare Dachgeschosse max." - elif "Attikageschoss" in param_name or "attikageschoss" in param_name.lower(): - formatted_param = "anrechenbares Attikageschoss max." - elif "Untergeschoss" in param_name or "untergeschoss" in param_name.lower(): - formatted_param = "anrechenbares Untergeschoss max." - - table_values_for_bauzone.append({ - "parameter": formatted_param, - "value": value, - "unit": unit_str, - "article": article_ref, - "page": page_num - }) - context_parts.append(f" • {formatted_param}: {value}{unit_str} (Quelle: {article_ref}, Seite {page_num})") - - # Also check for multiple values (e.g., Fassadenhöhen with footnote values) - if len(values) > 1: - for idx, val_entry in enumerate(values[1:], 1): - value_extra = val_entry.get("value", "") - unit_extra = val_entry.get("unit", "") - unit_str_extra = f" {unit_extra}" if unit_extra else "" - context_parts.append(f" (Alternative: {value_extra}{unit_str_extra})") - - # Add zone information with all details - if relevant_zones: - context_parts.append("\n=== ZONE DEFINITIONS ===") - for zone in relevant_zones: - zone_code = zone.get("zone_code", "") - zone_name = zone.get("zone_name", "") - zone_category = zone.get("zone_category", "") - geschosszahl = zone.get("geschosszahl") - gewerbeerleichterung = zone.get("gewerbeerleichterung", False) - page_num = zone.get("page", 0) - source_article = zone.get("source_article", "") - - zone_info = f"- {zone_code}: {zone_name}" - if zone_category: - zone_info += f" (Kategorie: {zone_category})" - if geschosszahl: - zone_info += f", Geschosszahl: {geschosszahl}" - if gewerbeerleichterung: - zone_info += ", Gewerbeerleichterung: Ja" - if source_article: - zone_info += f" - Quelle: {source_article} (Seite {page_num})" - context_parts.append(zone_info) - - # Add article information with full text previews - relevant_articles = filter_articles_by_bauzone(extracted_content.get("articles", []), bauzone) - if relevant_articles: - context_parts.append("\n=== RELEVANT ARTICLES (full content) ===") - for article in relevant_articles: - article_label = article.get("article_label", "") - article_title = article.get("article_title", "") - article_text = article.get("text", "") - page_start = article.get("page_start", 0) - page_end = article.get("page_end", 0) - page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}" - - context_parts.append(f"\n{article_label}: {article_title or 'Kein Titel'}") - context_parts.append(f"Lage: {page_range}") - # Include full article text (truncated if too long) - if len(article_text) > 1000: - context_parts.append(f"Inhalt: {article_text[:1000]}...") - else: - context_parts.append(f"Inhalt: {article_text}") - - # Add relevant rules (only if not already covered in tables) - if relevant_rules: - # Filter out rules that are likely already in tables - table_parameter_names = set() - for table in zone_parameter_tables: - for param in table.get("parameters", []): - param_name = param.get("parameter", "").lower() - table_parameter_names.add(param_name) - - unique_rules = [] - for rule in relevant_rules[:15]: - rule_type = rule.get("rule_type", "").lower() - # Skip if this rule type is likely in tables - if not any(tp in rule_type for tp in table_parameter_names): - unique_rules.append(rule) - - if unique_rules: - context_parts.append("\n=== ADDITIONAL BUILDING REGULATIONS (from text) ===") - for rule in unique_rules[:8]: - rule_type = rule.get("rule_type", "") - value_numeric = rule.get("value_numeric") - value_text = rule.get("value_text", "") - unit = rule.get("unit", "") - page_num = rule.get("page", 0) - - rule_desc = f"- {rule_type}: " - if value_numeric is not None: - rule_desc += f"{value_numeric}" - if unit: - rule_desc += f" {unit}" - else: - rule_desc += value_text - rule_desc += f" (Seite {page_num})" - - context_parts.append(rule_desc) - - context = "\n".join(context_parts) - - # Create AI prompt with explicit instructions to include all table values - prompt = f""" -Analyze the following building zone (Bauzone) information extracted from BZO (Bau- und Zonenordnung) documents for {gemeinde}, specifically for Bauzone {bauzone}. - -Extracted Content: -{context} - -CRITICAL INSTRUCTIONS: -1. You MUST include ALL actual values from the tables in your summary - do NOT just say "see tables on page X" -2. List ALL parameters with their actual values: Ausnützungsziffer, Vollgeschosse, Gebäudelänge, Grenzabstand (Grundabstand, Mehrlängen-zuschlag, Höchstmass), Fassadenhöhen, etc. -3. Integrate zone definitions and article information INTO the summary text - do NOT create separate sections -4. Always cite WHERE each piece of information was found (article number and page number) -5. Combine everything into ONE unified, flowing summary - no separate sections for zones/articles -6. Be comprehensive - include all relevant details from zones, articles, and tables -7. Format as a single, well-structured German text document - -Please provide a comprehensive, unified summary that includes: - -1. General description of Bauzone {bauzone}: - - Zone category (Wohnzonen, Zentrumszonen, etc.) - - Geschosszahl (number of full storeys) - - Gewerbeerleichterung status (Ja/Nein) - - Where defined (article and page number) - -2. ALL building regulations with ACTUAL VALUES from tables (you MUST include the exact values): - - Ausnützungsziffer max.: [ACTUAL PERCENTAGE VALUE]% (from article, page) - - Vollgeschosse max.: [ACTUAL NUMBER] (from article, page) - - anrechenbare Dachgeschosse max.: [ACTUAL NUMBER] (from article, page) - - anrechenbares Attikageschoss max.: [ACTUAL NUMBER] (from article, page) - - anrechenbares Untergeschoss max.: [ACTUAL NUMBER] (from article, page) - - Gebäudelänge max.: [ACTUAL VALUE] m (from article, page) - - Grenzabstand - Grundabstand min.: [ACTUAL VALUE] m (from article, page) - - Grenzabstand - Mehrlängen-zuschlag: [ACTUAL FRACTION] (from article, page) - - Grenzabstand - Höchstmass max.: [ACTUAL VALUE] m (from article, page) - - Fassadenhöhen max.: [ACTUAL VALUE] m (from article, page, include footnote values if present) - -3. Zone definitions: Integrate information about where this zone is defined (which articles mention it, with page numbers) - -4. Relevant articles: Integrate key content from relevant articles naturally into the summary, citing article numbers and page numbers - -5. Special conditions: Any special requirements or exceptions mentioned in articles - -CRITICAL: You MUST include the actual numeric values from the tables in your summary. Do NOT say "see tables" - list the actual values. Format everything as ONE unified, flowing German text document without separate sections. Integrate zones and articles naturally into the narrative. -""" - - # Call AI service - logger.info(f"Generating AI summary for Bauzone {bauzone} in {gemeinde}") - ai_response = await aiService.callAiPlanning( - prompt=prompt, - debugType="bzo_summary" - ) - - return ai_response.strip() - - except Exception as e: - logger.error(f"Error generating AI summary: {str(e)}", exc_info=True) - # Return a basic summary if AI fails - return f"Summary generation failed: {str(e)}. Found {len(relevant_rules)} relevant rules and {len(relevant_zones)} zones for Bauzone {bauzone}." - - # --------------------------------------------------------------------------- # Feature Lifecycle Hooks # --------------------------------------------------------------------------- @@ -3124,3 +318,57 @@ def onMandateDelete(mandateId: str, instances: list) -> None: logger.warning(f"Failed to cascade-delete realEstate data for mandate {mandateId}: {e}") +def onUserDelete(userId: str, currentUser) -> dict: + """Delete/anonymize user data from the realEstate database (GDPR).""" + from modules.system.gdprDeletion import deleteUserDataFromDatabase + from modules.connectors.connectorDbPostgre import DatabaseConnector + from modules.shared.configuration import APP_CONFIG + + dbName = "poweron_realestate" + try: + db = DatabaseConnector( + dbHost=APP_CONFIG.get("DB_HOST", "localhost"), + dbDatabase=dbName, + dbUser=APP_CONFIG.get("DB_USER"), + dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"), + dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), + userId=None, + ) + stats = deleteUserDataFromDatabase(db, userId, dbName) + db.close() + return stats + except Exception as e: + logger.warning(f"onUserDelete realEstate failed: {e}") + return {"database": dbName, "tablesProcessed": 0, "recordsDeleted": 0, "recordsAnonymized": 0, "errors": [str(e)]} + + +# --------------------------------------------------------------------------- +# Re-exports for backward compatibility +# --------------------------------------------------------------------------- + +from .serviceGeometry import ( # noqa: F401, E402 + geopolylinie_to_shapely_polygon, + shapely_polygon_to_geopolylinie, + combine_parcel_geometries, + filter_neighbor_parcels, + fetch_parcel_polygon_from_swisstopo, + create_project_with_parcel_data, + convert_geojson_to_geopolylinie, +) + +from .serviceAiIntent import ( # noqa: F401, E402 + executeDirectQuery, + _formatEntitySummary, + processNaturalLanguageCommand, + analyzeUserIntent, + executeIntentBasedOperation, +) + +from .serviceBzo import ( # noqa: F401, E402 + extract_bzo_information, + filter_rules_by_bauzone, + filter_zones_by_bauzone, + filter_articles_by_bauzone, + _filter_tables_by_bauzone, + generate_bauzone_ai_summary, +) diff --git a/modules/features/realEstate/routeFeatureRealEstate.py b/modules/features/realEstate/routeFeatureRealEstate.py index a1cfdb8b..b78ee2ee 100644 --- a/modules/features/realEstate/routeFeatureRealEstate.py +++ b/modules/features/realEstate/routeFeatureRealEstate.py @@ -3,13 +3,9 @@ Real Estate routes for the backend API. Implements stateless endpoints for real estate database operations with AI-powered natural language processing. """ -import asyncio import json import logging -import re -import aiohttp -import requests -from typing import Optional, Dict, Any, List, Union +from typing import Optional, Dict, Any, List from fastapi import APIRouter, HTTPException, Depends, Body, Request, Query, Path, status from fastapi.responses import JSONResponse @@ -29,12 +25,9 @@ from .datamodelFeatureRealEstate import ( Projekt, Parzelle, Dokument, - DokumentTyp, Gemeinde, Kanton, Land, - Kontext, - StatusProzess, ) # Import interfaces @@ -43,19 +36,27 @@ from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface # Import feature logic for AI-powered commands from .mainRealEstate import ( processNaturalLanguageCommand, - create_project_with_parcel_data, extract_bzo_information, ) -from .parcelSelectionService import compute_selection_summary, is_parcel_adjacent_to_selection +from .parcelSelectionService import compute_selection_summary -# Import Swiss Topo MapServer, ÖREB and Zurich WFS connectors -from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector -from modules.connectors.connectorOerebWfs import OerebWfsConnector +# Import connectors still used directly in route file from modules.connectors.connectorZhWfsParcels import ZhWfsParcelsConnector -# Import ComponentObjects and Tavily for BZO document fetch +# Import ComponentObjects interface for BZO routes from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface -from modules.aicore.aicorePluginTavily import AiTavily + +# Import handler functions for complex business logic +from .handlerRealEstate import ( + processGemeindenSync, + processBzoDocumentsFetch, + processParcelDocuments, + processTableData, + processCreateTableRecord, + processParcelSearch, + processAddAdjacentParcel, + processAddParcelToProject, +) # Import attribute utilities for model schema from modules.shared.attributeUtils import getModelAttributeDefinitions @@ -141,6 +142,31 @@ _REALESTATE_ENTITY_MODELS = { } +def _validateCsrfToken(request: "Request", routePath: str, userId: str) -> None: + """Validate CSRF token from request headers (format + hex check).""" + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for {routePath} from user {userId}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") + ) + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for {routePath} from user {userId}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("Invalid CSRF token format") + ) + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for {routePath} from user {userId}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=routeApiMsg("Invalid CSRF token format") + ) + + # ============================================================================ # INSTANCE-ID ROUTES (backend-driven, analog to Trustee) # ============================================================================ @@ -535,116 +561,7 @@ async def get_instance_gemeinden( interface = getRealEstateInterface( context.user, mandateId=mandateId, featureInstanceId=instanceId ) - try: - oereb_connector = OerebWfsConnector() - connector = SwissTopoMapServerConnector(oereb_connector=oereb_connector) - gemeinden_data = await connector.get_all_gemeinden(only_current=only_current) - except Exception as e: - logger.error(f"Error fetching Gemeinden from Swiss Topo: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Error fetching Gemeinden: {str(e)}") - gemeinden_created = 0 - gemeinden_skipped = 0 - kantone_created = 0 - errors: List[str] = [] - kanton_cache: Dict[str, str] = {} - - def find_gemeinde_by_bfs_nummer(bfs_nummer: str) -> Optional[Any]: - try: - gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId}) - for g in gemeinden: - for k in (g.kontextInformationen or []): - try: - data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt - if isinstance(data, dict) and str(data.get("bfs_nummer")) == str(bfs_nummer): - return g - except (json.JSONDecodeError, AttributeError): - continue - except Exception as ex: - logger.error(f"Error finding Gemeinde by BFS {bfs_nummer}: {ex}", exc_info=True) - return None - - def get_or_create_kanton(kanton_abk: str) -> Optional[str]: - nonlocal kantone_created, errors - if not kanton_abk: - return None - if kanton_abk in kanton_cache: - return kanton_cache[kanton_abk] - kantone = interface.getKantone(recordFilter={"mandateId": mandateId, "abk": kanton_abk}) - if kantone: - kanton_cache[kanton_abk] = kantone[0].id - return kantone[0].id - kanton_names = { - "AG": "Aargau", "AI": "Appenzell Innerrhoden", "AR": "Appenzell Ausserrhoden", - "BE": "Bern", "BL": "Basel-Landschaft", "BS": "Basel-Stadt", - "FR": "Freiburg", "GE": "Genf", "GL": "Glarus", "GR": "Graubünden", - "JU": "Jura", "LU": "Luzern", "NE": "Neuenburg", "NW": "Nidwalden", - "OW": "Obwalden", "SG": "St. Gallen", "SH": "Schaffhausen", "SO": "Solothurn", - "SZ": "Schwyz", "TG": "Thurgau", "TI": "Tessin", "UR": "Uri", - "VD": "Waadt", "VS": "Wallis", "ZG": "Zug", "ZH": "Zürich", - } - try: - kanton_label = kanton_names.get(kanton_abk, kanton_abk) - kanton = Kanton( - mandateId=mandateId, - featureInstanceId=instanceId, - label=kanton_label, - abk=kanton_abk, - ) - created = interface.createKanton(kanton) - if created and created.id: - kanton_cache[kanton_abk] = created.id - kantone_created += 1 - return created.id - except Exception as ex: - errors.append(f"Error creating Kanton {kanton_abk}: {ex}") - return None - - saved_gemeinden: List[Dict[str, Any]] = [] - for gd in gemeinden_data: - try: - gemeinde_name = gd.get("name") - bfs_nummer = gd.get("bfs_nummer") - kanton_abk = gd.get("kanton") - if not gemeinde_name or bfs_nummer is None: - gemeinden_skipped += 1 - continue - existing = find_gemeinde_by_bfs_nummer(str(bfs_nummer)) - if existing: - gemeinden_skipped += 1 - saved_gemeinden.append(existing.model_dump() if hasattr(existing, "model_dump") else existing) - continue - kanton_id = get_or_create_kanton(kanton_abk) if kanton_abk else None - gemeinde = Gemeinde( - mandateId=mandateId, - featureInstanceId=instanceId, - label=gemeinde_name, - id_kanton=kanton_id, - kontextInformationen=[ - Kontext(thema="BFS Nummer", inhalt=json.dumps({"bfs_nummer": bfs_nummer}, ensure_ascii=False)) - ], - ) - created = interface.createGemeinde(gemeinde) - if created and created.id: - gemeinden_created += 1 - saved_gemeinden.append(created.model_dump() if hasattr(created, "model_dump") else created) - else: - errors.append(f"Failed to create Gemeinde {gemeinde_name}") - gemeinden_skipped += 1 - except Exception as ex: - errors.append(f"Error processing {gd.get('name', 'Unknown')}: {str(ex)}") - gemeinden_skipped += 1 - - return { - "gemeinden": saved_gemeinden, - "count": len(saved_gemeinden), - "stats": { - "gemeinden_created": gemeinden_created, - "gemeinden_skipped": gemeinden_skipped, - "kantone_created": kantone_created, - "error_count": len(errors), - "errors": errors[:10], - }, - } + return await processGemeindenSync(interface, instanceId, mandateId, onlyCurrent=only_current) @router.post("/{instanceId}/gemeinden/fetch-bzo-documents", response_model=Dict[str, Any]) @@ -662,38 +579,7 @@ async def fetch_instance_bzo_documents( componentInterface = getComponentInterface( context.user, mandateId=mandateId, featureInstanceId=instanceId ) - from modules.features.realEstate.realEstateGemeindeService import fetch_bzo_for_gemeinde - - gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId}) - stats = {"gemeinden_processed": 0, "documents_created": 0, "documents_skipped": 0, "errors": []} - results: List[Dict[str, Any]] = [] - - for gemeinde in gemeinden: - gr = {"gemeinde_id": gemeinde.id, "gemeinde_label": gemeinde.label, "status": None, "dokument_ids": [], "error": None} - try: - stats["gemeinden_processed"] += 1 - fetched = await fetch_bzo_for_gemeinde( - interface, componentInterface, gemeinde, mandateId, instanceId - ) - if fetched: - gr["status"] = "created" - stats["documents_created"] += 1 - refreshed = interface.getGemeinde(gemeinde.id) - if refreshed and refreshed.dokumente: - for doc in refreshed.dokumente: - doc_id = getattr(doc, "id", None) or (doc.get("id") if isinstance(doc, dict) else None) - if doc_id: - gr["dokument_ids"].append(doc_id) - else: - gr["status"] = "skipped" - stats["documents_skipped"] += 1 - except Exception as ex: - gr["status"] = "error" - gr["error"] = str(ex) - stats["errors"].append(f"{gemeinde.label}: {str(ex)}") - results.append(gr) - - return {"success": True, "stats": stats, "results": results} + return await processBzoDocumentsFetch(interface, componentInterface, mandateId, instanceId) @router.get("/{instanceId}/parcel-documents", response_model=Dict[str, Any]) @@ -717,65 +603,7 @@ async def get_parcel_documents( componentInterface = getComponentInterface( context.user, mandateId=mandateId, featureInstanceId=instanceId ) - from modules.features.realEstate.realEstateGemeindeService import ( - ensure_single_gemeinde, - fetch_bzo_for_gemeinde, - ) - gemeinde_obj = None - by_label = interface.getGemeinden(recordFilter={"label": gemeinde, "mandateId": mandateId}) - gemeinde_obj = by_label[0] if by_label else None - if not gemeinde_obj: - # Fallback: match by normalized label (e.g. DB has "Stadt Uster", request has "Uster") - all_g = interface.getGemeinden(recordFilter={"mandateId": mandateId}) - g_norm = gemeinde.strip().lower() - for g in all_g: - gl = (g.label or "").strip().lower() - if gl == g_norm or g_norm in gl or gl in g_norm: - gemeinde_obj = g - logger.debug(f"parcel-documents: Found Gemeinde by label match '{gemeinde}' -> '{g.label}'") - break - if gemeinde_obj: - logger.debug(f"parcel-documents: Gemeinde '{gemeinde}' resolved: {gemeinde_obj.id}") - if not gemeinde_obj: - logger.info(f"parcel-documents: No Gemeinde for label '{gemeinde}', ensuring via Swiss Topo...") - gemeinde_obj = await ensure_single_gemeinde(interface, mandateId, instanceId, gemeinde_name=gemeinde) - if not gemeinde_obj: - logger.warning(f"parcel-documents: Gemeinde '{gemeinde}' nicht gefunden (mandateId={mandateId[:8]}...)") - return {"documents": [], "error": f"Gemeinde '{gemeinde}' nicht gefunden"} - bzo_docs = [] - if gemeinde_obj.dokumente: - for doc in gemeinde_obj.dokumente: - typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None) - if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] or str(typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision"]: - doc_id = doc.id if hasattr(doc, "id") else doc.get("id") - if doc_id: - full = interface.getDokument(doc_id) - if full and full.dokumentReferenz: - bzo_docs.append(full) - if not bzo_docs: - logger.info(f"parcel-documents: No BZO for {gemeinde}, fetching...") - fetched = await fetch_bzo_for_gemeinde(interface, componentInterface, gemeinde_obj, mandateId, instanceId) - if fetched: - gemeinde_obj = interface.getGemeinde(gemeinde_obj.id) - if gemeinde_obj and gemeinde_obj.dokumente: - for doc in gemeinde_obj.dokumente: - typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None) - if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION]: - doc_id = doc.id if hasattr(doc, "id") else doc.get("id") - if doc_id: - full = interface.getDokument(doc_id) - if full and full.dokumentReferenz: - bzo_docs.append(full) - result = [] - for d in bzo_docs: - result.append({ - "id": d.id, - "label": d.label, - "fileId": d.dokumentReferenz, - "fileName": (d.label or "BZO") + ".pdf", - "mimeType": d.mimeType or "application/pdf", - }) - return {"documents": result, "gemeinde": gemeinde, "bauzone": bauzone} + return await processParcelDocuments(interface, componentInterface, gemeinde, bauzone, mandateId, instanceId) @router.get("/{instanceId}/bzo-information", response_model=Dict[str, Any]) @@ -831,57 +659,13 @@ async def process_command( Uses AI to analyze user intent and extract parameters, then executes the appropriate CRUD operation. Works stateless without session management. - - Example user inputs: - - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - - "Zeige mir alle Projekte in Zuerich" - - "Aktualisiere Projekt XYZ mit Status 'Planung'" - - "Loesche Parzelle ABC" - - "SELECT * FROM Projekt WHERE plz = '8000'" - - Headers: - - X-CSRF-Token: CSRF token (required for security) - - Returns: - { - "success": true, - "intent": "CREATE|READ|UPDATE|DELETE|QUERY", - "entity": "Projekt|Parzelle|...|null", - "result": {...} - } """ try: - # Validate CSRF token (middleware also checks, but explicit validation for better error messages) - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/command from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - - # Basic CSRF token format validation - if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for POST /api/realestate/command from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - - # Validate token is hex string - try: - int(csrf_token, 16) - except ValueError: - logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/command from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) + _validateCsrfToken(request, "POST /api/realestate/command", str(context.user.id)) logger.info(f"Processing command request from user {context.user.id} (mandate: {context.mandateId})") logger.debug(f"User input: {userInput}") - # Process natural language command with AI result = await processNaturalLanguageCommand( currentUser=context.user, mandateId=str(context.mandateId), @@ -911,85 +695,22 @@ def get_available_tables( request: Request, context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """ - Get all available real estate tables. - - Returns a list of available table names with their descriptions. - - Headers: - - X-CSRF-Token: CSRF token (required for security) - - Example: - - GET /api/realestate/tables - """ + """Get all available real estate tables.""" try: - # Validate CSRF token if provided - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/tables from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - - # Basic CSRF token format validation - if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for GET /api/realestate/tables from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - - # Validate token is hex string - try: - int(csrf_token, 16) - except ValueError: - logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/tables from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) + _validateCsrfToken(request, "GET /api/realestate/tables", str(context.user.id)) logger.info(f"Getting available tables for user {context.user.id} (mandate: {context.mandateId})") - # Define available tables with descriptions tables = [ - { - "name": "Projekt", - "description": "Real estate projects", - "model": "Projekt" - }, - { - "name": "Parzelle", - "description": "Plots/parcels", - "model": "Parzelle" - }, - { - "name": "Dokument", - "description": "Documents", - "model": "Dokument" - }, - { - "name": "Gemeinde", - "description": "Municipalities", - "model": "Gemeinde" - }, - { - "name": "Kanton", - "description": "Cantons", - "model": "Kanton" - }, - { - "name": "Land", - "description": "Countries", - "model": "Land" - }, + {"name": "Projekt", "description": "Real estate projects", "model": "Projekt"}, + {"name": "Parzelle", "description": "Plots/parcels", "model": "Parzelle"}, + {"name": "Dokument", "description": "Documents", "model": "Dokument"}, + {"name": "Gemeinde", "description": "Municipalities", "model": "Gemeinde"}, + {"name": "Kanton", "description": "Cantons", "model": "Kanton"}, + {"name": "Land", "description": "Countries", "model": "Land"}, ] - return { - "tables": tables, - "count": len(tables) - } + return {"tables": tables, "count": len(tables)} except HTTPException: raise @@ -1009,142 +730,12 @@ def get_table_data( pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), context: RequestContext = Depends(getRequestContext) ) -> PaginatedResponse[Dict[str, Any]]: - """ - Get all data from a specific real estate table with optional pagination. - - Available tables: - - Projekt: Real estate projects - - Parzelle: Plots/parcels - - Dokument: Documents - - Gemeinde: Municipalities - - Kanton: Cantons - - Land: Countries - - Query Parameters: - - pagination: JSON-encoded PaginationParams object, or None for no pagination - - Headers: - - X-CSRF-Token: CSRF token (required for security) - - Examples: - - GET /api/realestate/table/Projekt (no pagination - returns all items) - - GET /api/realestate/table/Parzelle?pagination={"page":1,"pageSize":10,"sort":[]} - - GET /api/realestate/table/Gemeinde?pagination={"page":2,"pageSize":20,"sort":[{"field":"label","direction":"asc"}]} - """ + """Get all data from a specific real estate table with optional pagination.""" try: - # Validate CSRF token if provided - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - - # Basic CSRF token format validation - if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for GET /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - - # Validate token is hex string - try: - int(csrf_token, 16) - except ValueError: - logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - + _validateCsrfToken(request, f"GET /api/realestate/table/{table}", str(context.user.id)) logger.info(f"Getting table data for '{table}' from user {context.user.id} (mandate: {context.mandateId})") - - # Map table names to model classes and getter methods - table_mapping = { - "Projekt": (Projekt, "getProjekte"), - "Parzelle": (Parzelle, "getParzellen"), - "Dokument": (Dokument, "getDokumente"), - "Gemeinde": (Gemeinde, "getGemeinden"), - "Kanton": (Kanton, "getKantone"), - "Land": (Land, "getLaender"), - } - - # Validate table name - if table not in table_mapping: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid table name '{table}'. Available tables: {', '.join(table_mapping.keys())}" - ) - - # Get interface and fetch data - realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - model_class, method_name = table_mapping[table] - getter_method = getattr(realEstateInterface, method_name) - - # Fetch all records (no filter for now) - records = getter_method(recordFilter=None) - - # Keep records as model instances (like routeDataFiles does with FileItem) - # FastAPI will automatically serialize Pydantic models to JSON - items = records - - # Parse pagination parameter - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - paginationParams = PaginationParams(**paginationDict) if paginationDict else None - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid pagination parameter: {str(e)}" - ) - - # Apply pagination if requested - if paginationParams: - # Apply sorting if specified - if paginationParams.sort: - for sort_field in reversed(paginationParams.sort): # Reverse to apply in priority order - field_name = sort_field.field - direction = sort_field.direction.lower() - - def sort_key(item): - # Access attribute from model instance - value = getattr(item, field_name, None) - # Handle None values - put them at the end for asc, at the start for desc - if value is None: - return (1, None) # Use tuple to ensure None values sort consistently - return (0, value) - - items.sort(key=sort_key, reverse=(direction == "desc")) - - # Apply pagination - total_items = len(items) - total_pages = (total_items + paginationParams.pageSize - 1) // paginationParams.pageSize # Ceiling division - start_idx = (paginationParams.page - 1) * paginationParams.pageSize - end_idx = start_idx + paginationParams.pageSize - paginated_items = items[start_idx:end_idx] - - return PaginatedResponse( - items=paginated_items, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=total_items, - totalPages=total_pages, - sort=paginationParams.sort, - filters=paginationParams.filters - ) - ) - else: - # No pagination - return all items (as model instances, like routeDataFiles) - return PaginatedResponse( - items=items, - pagination=None - ) - + mandateId = str(context.mandateId) if context.mandateId else None + return processTableData(context.user, mandateId, table, pagination) except HTTPException: raise except Exception as e: @@ -1163,186 +754,12 @@ async def create_table_record( data: Dict[str, Any] = Body(..., description="Record data to create"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """ - Create a new record in a specific real estate table. - - Available tables: - - Projekt: Real estate projects (with parcel data support) - - Parzelle: Plots/parcels - - Dokument: Documents - - Gemeinde: Municipalities - - Kanton: Cantons - - Land: Countries - - Request Body: - For Projekt: - { - "label": "Projekt Bezeichnung", - "statusProzess": "Eingang", // Optional - "parzelle": { - "id": "OE5913", - "egrid": "CH252699779137", - "perimeter": {...}, - "geometry": {...}, // Used for baulinie - ... - } - } - - For other tables: - - JSON object with fields matching the table's data model - - Headers: - - X-CSRF-Token: CSRF token (required for security) - - Examples: - - POST /api/realestate/table/Projekt - Body: {"label": "Hauptstrasse 42", "parzelle": {...}} - - POST /api/realestate/table/Parzelle - Body: {"label": "Parzelle 1", "strasseNr": "Hauptstrasse 42", "plz": "8000", "bauzone": "W3"} - """ + """Create a new record in a specific real estate table.""" try: - # Validate CSRF token - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - - # Basic CSRF token format validation - if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - logger.warning(f"Invalid CSRF token format for POST /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - - # Validate token is hex string - try: - int(csrf_token, 16) - except ValueError: - logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/table/{table} from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - - # Special handling for Projekt with parcel data - if table == "Projekt" and ("parzelle" in data or "parzellen" in data): - logger.info(f"Creating Projekt with parcel data for user {context.user.id} (mandate: {context.mandateId})") - - # Extract fields - label = data.get("label") - if not label: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=routeApiMsg("label is required") - ) - - status_prozess = data.get("statusProzess", "Eingang") - - # Support both single parzelle and multiple parzellen - parzellen_data = [] - if "parzellen" in data: - # Multiple parcels - parzellen_data = data.get("parzellen", []) - if not isinstance(parzellen_data, list): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=routeApiMsg("parzellen must be an array") - ) - elif "parzelle" in data: - # Single parcel - parzelle_data = data.get("parzelle") - if parzelle_data: - parzellen_data = [parzelle_data] - - if not parzellen_data: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=routeApiMsg("parzelle or parzellen data is required") - ) - - # Use helper function to create project with parcel data - try: - result = await create_project_with_parcel_data( - currentUser=context.user, - mandateId=str(context.mandateId), - projekt_label=label, - parzellen_data=parzellen_data, - status_prozess=status_prozess, - ) - - # Return in format expected by frontend (single record, not nested) - return result.get("projekt", {}) - except HTTPException: - # Re-raise HTTPExceptions directly - raise - except Exception as e: - logger.error(f"Error creating Projekt with parcel data: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error creating Projekt: {str(e)}" - ) - - # Standard handling for other tables or Projekt without parcel data + _validateCsrfToken(request, f"POST /api/realestate/table/{table}", str(context.user.id)) logger.info(f"Creating record in table '{table}' for user {context.user.id} (mandate: {context.mandateId})") - logger.debug(f"Record data: {data}") - - # Map table names to model classes and create methods - table_mapping = { - "Projekt": (Projekt, "createProjekt"), - "Parzelle": (Parzelle, "createParzelle"), - "Dokument": (Dokument, "createDokument"), - "Gemeinde": (Gemeinde, "createGemeinde"), - "Kanton": (Kanton, "createKanton"), - "Land": (Land, "createLand"), - } - - # Validate table name - if table not in table_mapping: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid table name '{table}'. Available tables: {', '.join(table_mapping.keys())}" - ) - - # Get interface - realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - model_class, method_name = table_mapping[table] - create_method = getattr(realEstateInterface, method_name) - - # Ensure mandateId is set from context - if "mandateId" not in data: - data["mandateId"] = str(context.mandateId) if context.mandateId else None - - # Create model instance from data - try: - model_instance = model_class(**data) - except Exception as e: - logger.error(f"Error creating {table} model instance: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid data for {table}: {str(e)}" - ) - - # Create record - try: - created_record = create_method(model_instance) - - # Convert to dictionary for response - if hasattr(created_record, 'model_dump'): - return created_record.model_dump() - else: - return created_record - - except Exception as e: - logger.error(f"Error creating {table} record: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error creating {table} record: {str(e)}" - ) - + mandateId = str(context.mandateId) if context.mandateId else None + return await processCreateTableRecord(context.user, mandateId, table, data) except HTTPException: raise except Exception as e: @@ -1389,380 +806,13 @@ async def search_parcel( ) -> Dict[str, Any]: """ Search for parcel information by address or coordinates. - - Returns comprehensive parcel information including: - - Parcel identification (number, EGRID, etc.) - - Precise boundary geometry for map display - - Administrative context (canton, municipality) - - Bauzone (zone code from ÖREB WFS when include_bauzone=True) - - Link to official cadastral map - - Optional: Adjacent parcels - - Query Parameters: - - location: Either coordinates as "x,y" (LV95/EPSG:2056) or address string - - include_adjacent: If true, fetches information about adjacent parcels (slower) - - include_bauzone: If true, queries ÖREB WFS for zone info (Bauzone/Wohnzone) - - Headers: - - X-CSRF-Token: CSRF token (required for security) + Returns comprehensive parcel information including geometry, administrative context, and bauzone. """ try: - # Validate CSRF token - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for GET /api/realestate/parcel/search from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - + _validateCsrfToken(request, "GET /api/realestate/parcel/search", str(context.user.id)) logger.info(f"Searching parcel for user {context.user.id} (mandate: {context.mandateId}) with location: {location}") - - # Initialize connector - connector = SwissTopoMapServerConnector() - - # Search for parcel - parcel_data = await connector.search_parcel(location) - - if not parcel_data: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"No parcel found for location: {location}" - ) - - # Extract and normalize attributes - extracted_attributes = connector.extract_parcel_attributes(parcel_data) - attributes = parcel_data.get("attributes", {}) - geometry = parcel_data.get("geometry", {}) - - # Calculate parcel area from perimeter - area_m2 = None - centroid = None - if extracted_attributes.get("perimeter"): - perimeter = extracted_attributes["perimeter"] - points = perimeter.get("punkte", []) - - # Calculate area using shoelace formula - if len(points) >= 3: - area = 0 - for i in range(len(points)): - j = (i + 1) % len(points) - area += points[i]["x"] * points[j]["y"] - area -= points[j]["x"] * points[i]["y"] - area_m2 = abs(area / 2) - - # Calculate centroid - sum_x = sum(p["x"] for p in points) - sum_y = sum(p["y"] for p in points) - centroid = { - "x": sum_x / len(points), - "y": sum_y / len(points) - } - - # Extract canton early (needed for bauzone query and municipality resolution) - canton = attributes.get("ak", "") - - # Extract municipality name and address from Swiss Topo data - municipality_name = None - full_address = None - plz = None - - # First, try to use geocoded address info if available (more accurate than centroid query) - geocoded_address = parcel_data.get('geocoded_address') - if geocoded_address: - full_address = geocoded_address.get('full_address') - plz = geocoded_address.get('plz') - municipality_name = geocoded_address.get('municipality') - logger.debug(f"Using geocoded address: {full_address}") - - # If geocoded address not available, try to get address by querying the address layer - # Use query coordinates (where user clicked/geocoded) instead of parcel centroid - # This ensures we get the address at the exact location, not at the parcel center - query_coords = parcel_data.get('query_coordinates') - address_query_coords = query_coords if query_coords else centroid - - if not full_address and address_query_coords: - query_x = address_query_coords['x'] - query_y = address_query_coords['y'] - logger.debug(f"Querying address layer at query coordinates: ({query_x}, {query_y})") - - # Check if this was a coordinate search (not geocoded address) - is_coordinate_search = ',' in location and not any(c.isalpha() for c in location.split(',')[0]) - - # Use connector's helper method to query building layer - # Use tolerance=1 (minimum) for coordinate searches to get exact building - building_tolerance = 1 if is_coordinate_search else 10 - building_result = await connector._query_building_layer(query_x, query_y, tolerance=building_tolerance, buffer=25) - - if building_result: - addr_attrs = building_result.get("attributes", {}) - logger.debug(f"Address layer attributes: {addr_attrs}") - - # Extract address using connector's helper method - address_info = connector._extract_address_from_building_attrs(addr_attrs) - full_address = address_info.get('full_address') - plz = address_info.get('plz') - municipality_name = address_info.get('municipality') - - if full_address: - logger.debug(f"Constructed address: {full_address}") - - # If address not found via building layer, try to construct from available data - if not full_address: - # Check if location was provided as an address string - if location and any(c.isalpha() for c in location) and "CH" not in location: - # Location looks like an address (not an EGRID) - full_address = location - logger.debug(f"Using location as address: {full_address}") - - # Try to extract municipality name from address string (e.g. "Forchstrasse 6c, 8610 Uster") - if not municipality_name and full_address: - plz_municipality_match = re.search(r"\b(\d{4})\s+([A-ZÄÖÜ][a-zäöüß\s-]+)", full_address) - if plz_municipality_match: - extracted_municipality = plz_municipality_match.group(2).strip() - extracted_municipality = re.sub(r"[,;\.]+$", "", extracted_municipality).strip() - if extracted_municipality: - municipality_name = extracted_municipality - if not plz: - plz = plz_municipality_match.group(1) - logger.debug(f"Extracted municipality from address: {municipality_name}") - - # Try to extract municipality name from BFSNR if not found - bfsnr = attributes.get("bfsnr") - if not municipality_name and bfsnr and canton and context.mandateId: - try: - interface = getRealEstateInterface( - context.user, mandateId=str(context.mandateId), featureInstanceId=None - ) - gemeinden = interface.getGemeinden(recordFilter={"mandateId": str(context.mandateId)}) - for g in gemeinden: - for k in (g.kontextInformationen or []): - try: - data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt - if isinstance(data, dict): - bfs = data.get("bfs_nummer") or data.get("bfsnr") or data.get("municipality_code") - if str(bfs) == str(bfsnr): - municipality_name = g.label - logger.debug(f"Found Gemeinde by BFS {bfsnr} in DB: {municipality_name}") - break - except (json.JSONDecodeError, AttributeError): - continue - if municipality_name: - break - except Exception as e: - logger.debug(f"Error querying Gemeinde by BFS: {e}") - - # Swiss Topo geocoding to get municipality from coordinates - if not municipality_name and centroid and canton: - try: - geocode_url = "https://api3.geo.admin.ch/rest/services/api/MapServer/identify" - params = { - "geometry": f"{centroid['x']},{centroid['y']}", - "geometryType": "esriGeometryPoint", - "layers": "all:ch.swisstopo.swissboundaries3d-gemeinde-flaeche.fill", - "tolerance": "0", - "returnGeometry": "false", - "sr": "2056", - "f": "json", - } - async with aiohttp.ClientSession() as session: - async with session.get(geocode_url, params=params) as resp: - if resp.status == 200: - data = await resp.json() - results = data.get("results", []) - if results: - attrs = results[0].get("attributes", {}) - geo_name = attrs.get("name") or attrs.get("gemeindename") or attrs.get("label") - if geo_name: - municipality_name = connector._clean_municipality_name(str(geo_name)) - logger.debug(f"Found municipality via Swiss Topo geocoding: {municipality_name}") - except Exception as e: - logger.debug(f"Error querying Swiss Topo geocoding: {e}") - - # Expanded common municipalities fallback - if not municipality_name and bfsnr: - common_municipalities = { - 261: "Zürich", 198: "Pfäffikon", 191: "Uster", 3203: "Winterthur", - 351: "Bern", 2701: "Basel", 6621: "Genève", 5586: "Lausanne", - 1061: "Luzern", 230: "St. Gallen", 5192: "Lugano", 1367: "Schwyz", - } - if bfsnr in common_municipalities: - municipality_name = common_municipalities[bfsnr] - logger.debug(f"Looked up municipality from common list: {municipality_name}") - elif canton and bfsnr: - municipality_name = f"{canton}-{bfsnr}" - logger.debug(f"Using fallback municipality: {municipality_name}") - - # Final validation: Don't use EGRID as address - if full_address and full_address.startswith("CH") and len(full_address) == 14 and full_address[2:].isdigit(): - # This is an EGRID, not an address - full_address = None - logger.debug("Removed EGRID from address field") - - # Query Bauzone (wohnzone) from ÖREB WFS when requested - bauzone = None - has_geometry = geometry and (geometry.get("rings") or geometry.get("coordinates")) - if include_bauzone and canton and has_geometry and centroid: - try: - logger.debug(f"Querying zone information for parcel {attributes.get('label')} in canton {canton}") - oereb_connector = OerebWfsConnector() - zone_results = await oereb_connector.query_zone_layer( - egrid=attributes.get("egris_egrid", "") or "", - x=centroid["x"], - y=centroid["y"], - canton=canton, - geometry=geometry, - ) - if zone_results and len(zone_results) > 0: - zone_attrs = zone_results[0].get("attributes", {}) - typ_gde_abkuerzung = zone_attrs.get("typ_gde_abkuerzung") - if typ_gde_abkuerzung: - bauzone = typ_gde_abkuerzung - logger.debug(f"Found bauzone: {bauzone} for parcel {attributes.get('label')}") - except Exception as e: - logger.warning(f"Error querying zone information: {e}", exc_info=True) - - # Build parcel info - parcel_info = { - "id": attributes.get("label") or attributes.get("number"), - "egrid": attributes.get("egris_egrid"), - "number": attributes.get("number"), - "name": attributes.get("name"), - "identnd": attributes.get("identnd"), - "canton": attributes.get("ak"), - "municipality_code": attributes.get("bfsnr"), - "municipality_name": municipality_name, - "address": full_address, - "plz": plz, - "perimeter": extracted_attributes.get("perimeter"), - "area_m2": area_m2, - "centroid": centroid, - "geoportal_url": attributes.get("geoportal_url"), - "realestate_type": attributes.get("realestate_type"), - "bauzone": bauzone, - } - - # Build map view info - bbox = parcel_data.get("bbox", []) - map_view = { - "center": centroid, - "zoom_bounds": { - "min_x": bbox[0] if len(bbox) >= 4 else None, - "min_y": bbox[1] if len(bbox) >= 4 else None, - "max_x": bbox[2] if len(bbox) >= 4 else None, - "max_y": bbox[3] if len(bbox) >= 4 else None - }, - "geometry_geojson": { - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": [ - [[p["x"], p["y"]] for p in extracted_attributes["perimeter"]["punkte"]] - ] if extracted_attributes.get("perimeter") else [] - }, - "properties": { - "id": parcel_info["id"], - "egrid": parcel_info["egrid"], - "number": parcel_info["number"] - } - } - } - - # Build response - response_data = { - "parcel": parcel_info, - "map_view": map_view - } - - # Fetch adjacent parcels if requested - if include_adjacent and parcel_data and parcel_data.get("geometry"): - try: - # Use the connector's method to find neighboring parcels by sampling along the boundary - # This ensures we find all parcels that actually touch the selected parcel - selected_parcel_id = parcel_info["id"] - adjacent_parcels_raw = await connector.find_neighboring_parcels( - parcel_data=parcel_data, - selected_parcel_id=selected_parcel_id, - sample_distance=20.0, # Sample every 20 meters (balanced for coverage and speed) - max_sample_points=30, # Allow up to 30 points to ensure all vertices are covered - max_neighbors=15, # Find up to 15 neighbors - max_concurrent=50 # Process up to 50 queries concurrently (maximum parallelization) - ) - - # Convert adjacent parcels to include GeoJSON geometry (optimized, minimal logging) - def convert_parcel_geometry(adj_parcel: Dict[str, Any]) -> Dict[str, Any]: - """Convert a single adjacent parcel to include GeoJSON geometry.""" - adj_parcel_with_geo = { - "id": adj_parcel["id"], - "egrid": adj_parcel.get("egrid"), - "number": adj_parcel.get("number"), - "perimeter": adj_parcel.get("perimeter") - } - - # Convert geometry to GeoJSON format if available - adj_geometry = adj_parcel.get("geometry") - adj_perimeter = adj_parcel.get("perimeter") - - if adj_geometry: - # Handle ESRI format (rings) - if "rings" in adj_geometry and adj_geometry["rings"]: - ring = adj_geometry["rings"][0] # Outer ring - coordinates = [[[p[0], p[1]] for p in ring]] - adj_parcel_with_geo["geometry_geojson"] = { - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": coordinates - }, - "properties": { - "id": adj_parcel["id"], - "egrid": adj_parcel.get("egrid"), - "number": adj_parcel.get("number") - } - } - # Handle GeoJSON format - elif adj_geometry.get("type") == "Polygon": - adj_parcel_with_geo["geometry_geojson"] = { - "type": "Feature", - "geometry": adj_geometry, - "properties": { - "id": adj_parcel["id"], - "egrid": adj_parcel.get("egrid"), - "number": adj_parcel.get("number") - } - } - - # If no geometry_geojson was created but we have perimeter, create it from perimeter - if "geometry_geojson" not in adj_parcel_with_geo and adj_perimeter and adj_perimeter.get("punkte"): - punkte = adj_perimeter["punkte"] - coordinates = [[[p["x"], p["y"]] for p in punkte]] - adj_parcel_with_geo["geometry_geojson"] = { - "type": "Feature", - "geometry": { - "type": "Polygon", - "coordinates": coordinates - }, - "properties": { - "id": adj_parcel["id"], - "egrid": adj_parcel.get("egrid"), - "number": adj_parcel.get("number") - } - } - - return adj_parcel_with_geo - - # Convert all parcels in parallel (using list comprehension for speed) - adjacent_parcels = [convert_parcel_geometry(adj_parcel) for adj_parcel in adjacent_parcels_raw] - - response_data["adjacent_parcels"] = adjacent_parcels - logger.info(f"Found {len(adjacent_parcels)} neighboring parcels for parcel {selected_parcel_id}") - - except Exception as e: - logger.warning(f"Error fetching adjacent parcels: {e}", exc_info=True) - response_data["adjacent_parcels"] = [] - - return response_data - + mandateId = str(context.mandateId) if context.mandateId else None + return await processParcelSearch(context.user, mandateId, location, include_bauzone, include_adjacent) except HTTPException: raise except Exception as e: @@ -1811,18 +861,6 @@ async def parcel_selection_summary( ) -def _build_geometry_geojson(extracted: Dict[str, Any], parcel_info: Dict[str, Any]) -> Dict[str, Any]: - """Build geometry_geojson from extracted perimeter for add-adjacent response.""" - coords = [] - if extracted.get("perimeter", {}).get("punkte"): - coords = [[[p["x"], p["y"]] for p in extracted["perimeter"]["punkte"]]] - return { - "type": "Feature", - "geometry": {"type": "Polygon", "coordinates": coords}, - "properties": {"id": parcel_info["id"], "egrid": parcel_info["egrid"], "number": parcel_info["number"]}, - } - - @router.post("/parcel/add-adjacent", response_model=Dict[str, Any]) @limiter.limit("60/minute") async def add_adjacent_parcel( @@ -1846,89 +884,7 @@ async def add_adjacent_parcel( selected_parcels = body.get("selected_parcels", []) if not location or "x" not in location or "y" not in location: raise HTTPException(status_code=400, detail=routeApiMsg("location with x,y required")) - loc_str = f"{location['x']},{location['y']}" - connector = SwissTopoMapServerConnector() - parcel_data = await connector.search_parcel(loc_str) - if not parcel_data: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=routeApiMsg("No parcel found at this location") - ) - extracted = connector.extract_parcel_attributes(parcel_data) - attributes = parcel_data.get("attributes", {}) - geometry = parcel_data.get("geometry", {}) - area_m2 = None - centroid = None - if extracted.get("perimeter"): - perimeter = extracted["perimeter"] - points = perimeter.get("punkte", []) - if len(points) >= 3: - area = 0 - for i in range(len(points)): - j = (i + 1) % len(points) - area += points[i]["x"] * points[j]["y"] - area -= points[j]["x"] * points[i]["y"] - area_m2 = abs(area / 2) - sum_x = sum(p["x"] for p in points) - sum_y = sum(p["y"] for p in points) - centroid = {"x": sum_x / len(points), "y": sum_y / len(points)} - parcel_info = { - "id": attributes.get("label") or attributes.get("number"), - "egrid": attributes.get("egris_egrid"), - "number": attributes.get("number"), - "name": attributes.get("name"), - "identnd": attributes.get("identnd"), - "canton": attributes.get("ak"), - "municipality_code": attributes.get("bfsnr"), - "municipality_name": None, - "address": None, - "plz": None, - "perimeter": extracted.get("perimeter"), - "area_m2": area_m2, - "centroid": centroid, - "geoportal_url": attributes.get("geoportal_url"), - "realestate_type": attributes.get("realestate_type"), - "bauzone": None, - } - map_view = { - "center": centroid, - "zoom_bounds": parcel_data.get("bbox", []) and { - "min_x": parcel_data["bbox"][0], - "min_y": parcel_data["bbox"][1], - "max_x": parcel_data["bbox"][2], - "max_y": parcel_data["bbox"][3], - } or None, - "geometry_geojson": _build_geometry_geojson(extracted, parcel_info), - } - new_parcel_response = {"parcel": parcel_info, "map_view": map_view} - if not is_parcel_adjacent_to_selection(new_parcel_response, selected_parcels): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=routeApiMsg("Nur angrenzende Parzellen können hinzugefügt werden") - ) - bbox = parcel_data.get("bbox", []) - map_view["zoom_bounds"] = { - "min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3] - } if len(bbox) >= 4 else None - geocoded_address = parcel_data.get("geocoded_address") - if geocoded_address: - parcel_info["municipality_name"] = geocoded_address.get("municipality") - parcel_info["address"] = geocoded_address.get("full_address") - parcel_info["plz"] = geocoded_address.get("plz") - if centroid and attributes.get("ak"): - try: - oereb = OerebWfsConnector() - zone_results = await oereb.query_zone_layer( - egrid=attributes.get("egris_egrid", "") or "", - x=centroid["x"], y=centroid["y"], - canton=attributes.get("ak"), - geometry=geometry, - ) - if zone_results and len(zone_results) > 0: - parcel_info["bauzone"] = zone_results[0].get("attributes", {}).get("typ_gde_abkuerzung") - except Exception as oe: - logger.debug(f"ÖREB zone query failed: {oe}") - return new_parcel_response + return await processAddAdjacentParcel(location, selected_parcels) except HTTPException: raise except Exception as e: @@ -1949,196 +905,13 @@ async def add_parcel_to_project( ) -> Dict[str, Any]: """ Add a parcel to an existing project. - - This endpoint can either: - 1. Link an existing Parzelle to the Projekt - 2. Create a new Parzelle from location data and link it - - Request Body: - Option 1 - Link existing parcel: - { - "parcelId": "existing-parcel-id" - } - - Option 2 - Create new parcel from location: - { - "location": "Hauptstrasse 42, 8000 Zuerich" - } - - Option 3 - Create new parcel with custom data: - { - "parcelData": { - "label": "Parzelle 123", - "strasseNr": "Hauptstrasse 42", - "plz": "8000", - "bauzone": "W3", - ... - } - } - - Headers: - - X-CSRF-Token: CSRF token (required for security) - - Returns: - { - "projekt": {...}, // Updated Projekt - "parzelle": {...} // Parcel that was added - } + Supports linking existing parcel, creating from location, or creating from custom data. """ try: - # Validate CSRF token - csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") - if not csrf_token: - logger.warning(f"CSRF token missing for POST /api/realestate/projekt/{projekt_id}/add-parcel from user {context.user.id}") - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("CSRF token missing. Please include X-CSRF-Token header.") - ) - - # Validate CSRF token format - if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - try: - int(csrf_token, 16) - except ValueError: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=routeApiMsg("Invalid CSRF token format") - ) - + _validateCsrfToken(request, f"POST /api/realestate/projekt/{projekt_id}/add-parcel", str(context.user.id)) logger.info(f"Adding parcel to project {projekt_id} for user {context.user.id} (mandate: {context.mandateId})") - - # Get interface - realEstateInterface = getRealEstateInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) - - # Fetch existing Projekt - use mandateId from context - recordFilter = {"id": projekt_id} - if context.mandateId: - recordFilter["mandateId"] = str(context.mandateId) - projekte = realEstateInterface.getProjekte(recordFilter=recordFilter) - if not projekte: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Projekt {projekt_id} not found" - ) - projekt = projekte[0] - - # Determine which option was used - parcel_id = body.get("parcelId") - location = body.get("location") - parcel_data_dict = body.get("parcelData") - - parzelle = None - - # Option 1: Link existing parcel - if parcel_id: - logger.info(f"Linking existing parcel {parcel_id}") - parcelFilter = {"id": parcel_id} - if context.mandateId: - parcelFilter["mandateId"] = str(context.mandateId) - parcels = realEstateInterface.getParzellen(recordFilter=parcelFilter) - if not parcels: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Parzelle {parcel_id} not found" - ) - parzelle = parcels[0] - - # Option 2: Create from location - elif location: - logger.info(f"Creating parcel from location: {location}") - - # Initialize connector and search for parcel - connector = SwissTopoMapServerConnector() - parcel_data = await connector.search_parcel(location) - - if not parcel_data: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"No parcel found at location: {location}" - ) - - # Extract attributes - extracted_attributes = connector.extract_parcel_attributes(parcel_data) - attributes = parcel_data.get("attributes", {}) - - # Create Parzelle with mandateId from context - parzelle_create_data = { - "mandateId": str(context.mandateId) if context.mandateId else None, - "label": extracted_attributes.get("label") or attributes.get("number") or "Unknown", - "parzellenAliasTags": [attributes.get("egris_egrid")] if attributes.get("egris_egrid") else [], - "eigentuemerschaft": None, - "strasseNr": location if not location.replace(",", "").replace(".", "").replace(" ", "").isdigit() else None, - "plz": None, - "perimeter": extracted_attributes.get("perimeter"), - "baulinie": None, - "kontextGemeinde": None, - "bauzone": None, - "az": None, - "bz": None, - "vollgeschossZahl": None, - "anrechenbarDachgeschoss": None, - "anrechenbarUntergeschoss": None, - "gebaeudehoeheMax": None, - "regelnGrenzabstand": [], - "regelnMehrlaengenzuschlag": [], - "regelnMehrhoehenzuschlag": [], - "parzelleBebaut": None, - "parzelleErschlossen": None, - "parzelleHanglage": None, - "laermschutzzone": None, - "hochwasserschutzzone": None, - "grundwasserschutzzone": None, - "parzellenNachbarschaft": [], - "dokumente": [], - "kontextInformationen": [ - Kontext( - thema="Swiss Topo Data", - inhalt=json.dumps({ - "egrid": attributes.get("egris_egrid"), - "identnd": attributes.get("identnd"), - "canton": attributes.get("ak"), - "municipality_code": attributes.get("bfsnr"), - "geoportal_url": attributes.get("geoportal_url") - }, ensure_ascii=False) - ) - ] - } - - parzelle_instance = Parzelle(**parzelle_create_data) - parzelle = realEstateInterface.createParzelle(parzelle_instance) - - # Option 3: Create from custom data - elif parcel_data_dict: - logger.info(f"Creating parcel from custom data") - parcel_data_dict["mandateId"] = str(context.mandateId) if context.mandateId else None - parzelle_instance = Parzelle(**parcel_data_dict) - parzelle = realEstateInterface.createParzelle(parzelle_instance) - - else: - raise ValueError("One of 'parcelId', 'location', or 'parcelData' is required") - - # Add parcel to project - if parzelle not in projekt.parzellen: - projekt.parzellen.append(parzelle) - - # Update projekt perimeter if needed (use first parcel's perimeter) - if not projekt.perimeter and parzelle.perimeter: - projekt.perimeter = parzelle.perimeter - - # Update Projekt - updated_projekt = realEstateInterface.updateProjekt(projekt) - - logger.info(f"Added Parzelle {parzelle.id} to Projekt {projekt_id}") - - return { - "projekt": updated_projekt.model_dump(), - "parzelle": parzelle.model_dump() - } - + mandateId = str(context.mandateId) if context.mandateId else None + return await processAddParcelToProject(context.user, mandateId, projekt_id, body) except ValueError as e: logger.error(f"Validation error in add_parcel_to_project: {str(e)}", exc_info=True) raise HTTPException( diff --git a/modules/features/realEstate/serviceAiIntent.py b/modules/features/realEstate/serviceAiIntent.py new file mode 100644 index 00000000..62efb1a0 --- /dev/null +++ b/modules/features/realEstate/serviceAiIntent.py @@ -0,0 +1,1087 @@ +""" +Real Estate feature — AI-based intent recognition and CRUD operations. + +Handles natural language processing, intent analysis, direct query execution, +and intent-based CRUD operations for the real estate domain. +""" + +import json +import re +import logging +from typing import Optional, Dict, Any, List + +from fastapi import HTTPException, status + +from modules.datamodels.datamodelUam import User +from .datamodelFeatureRealEstate import ( + Projekt, + Parzelle, + StatusProzess, + GeoPolylinie, + GeoPunkt, + Kontext, + Gemeinde, + Kanton, + Land, +) +from modules.serviceCenter.serviceHub import getInterface as getServices +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface +from .serviceGeometry import fetch_parcel_polygon_from_swisstopo + +logger = logging.getLogger(__name__) + + +async def executeDirectQuery( + currentUser: User, + mandateId: str, + queryText: str, + parameters: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Execute a database query directly without session management. + + Args: + currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) + queryText: SQL query text + parameters: Optional parameters for parameterized queries + + Returns: + Dictionary containing query result (rows, columns, rowCount) + + Note: + - No session or query history is saved + - Query is executed directly and result is returned + - For production, validate and sanitize queries before execution + """ + try: + logger.info(f"Executing direct query for user {currentUser.id} (mandate: {mandateId})") + logger.debug(f"Query text: {queryText}") + if parameters: + logger.debug(f"Query parameters: {parameters}") + + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) + result = realEstateInterface.executeQuery(queryText, parameters) + + logger.info( + f"Query executed successfully: {result['rowCount']} rows in {result.get('executionTime', 0):.3f}s" + ) + + return { + "status": "success", + "rows": result["rows"], + "columns": result["columns"], + "rowCount": result["rowCount"], + "executionTime": result.get("executionTime", 0), + } + + except Exception as e: + logger.error(f"Error executing query: {str(e)}", exc_info=True) + raise + + +def _formatEntitySummary(entity_type: str, items: List[Dict[str, Any]], filters: Dict[str, Any]) -> str: + """ + Format a human-readable summary of query results. + + Args: + entity_type: Type of entity (Projekt, Parzelle, etc.) + items: List of entity data dictionaries + filters: Filter parameters used in the query + + Returns: + Human-readable summary string + """ + if not items: + return f"Keine {entity_type} gefunden" + + count = len(items) + filter_desc = "" + if filters: + if "kontextGemeinde" in filters: + filter_desc = f" in {filters['kontextGemeinde']}" + elif "plz" in filters: + filter_desc = f" mit PLZ {filters['plz']}" + elif "location_filter" in filters: + filter_desc = f" in {filters['location_filter']}" + + summary = f"Gefunden: {count} {entity_type}{filter_desc}" + + if entity_type == "Parzelle": + summary += "\n\nDetails:" + for i, item in enumerate(items[:10], 1): + parts = [] + + if item.get("label"): + parts.append(f"Parzelle '{item['label']}'") + elif item.get("id"): + parts.append(f"Parzelle {item['id'][:8]}...") + + if item.get("strasseNr"): + parts.append(item["strasseNr"]) + + location_parts = [] + if item.get("plz"): + location_parts.append(item["plz"]) + if item.get("kontextGemeinde"): + location_parts.append(item["kontextGemeinde"]) + if location_parts: + parts.append(" ".join(location_parts)) + + if item.get("bauzone"): + parts.append(f"Bauzone: {item['bauzone']}") + + summary += f"\n{i}. {', '.join(parts)}" + + if count > 10: + summary += f"\n... und {count - 10} weitere" + + elif entity_type == "Projekt": + summary += "\n\nDetails:" + for i, item in enumerate(items[:10], 1): + parts = [] + + if item.get("label"): + parts.append(f"'{item['label']}'") + + if item.get("statusProzess"): + parts.append(f"Status: {item['statusProzess']}") + + parzellen = item.get("parzellen", []) + if parzellen: + parts.append(f"{len(parzellen)} Parzelle(n)") + + summary += f"\n{i}. {' - '.join(parts)}" + + if count > 10: + summary += f"\n... und {count - 10} weitere" + + elif entity_type == "Gemeinde": + summary += "\n\nDetails:" + for i, item in enumerate(items[:10], 1): + parts = [] + + if item.get("label"): + parts.append(item["label"]) + if item.get("plz"): + parts.append(f"PLZ: {item['plz']}") + if item.get("abk"): + parts.append(f"Abk: {item['abk']}") + + summary += f"\n{i}. {', '.join(parts)}" + + if count > 10: + summary += f"\n... und {count - 10} weitere" + + elif entity_type == "Dokument": + summary += "\n\nDetails:" + for i, item in enumerate(items[:10], 1): + parts = [] + + if item.get("label"): + parts.append(item["label"]) + if item.get("dokumentTyp"): + parts.append(f"Typ: {item['dokumentTyp']}") + if item.get("quelle"): + parts.append(f"Quelle: {item['quelle']}") + + summary += f"\n{i}. {', '.join(parts)}" + + if count > 10: + summary += f"\n... und {count - 10} weitere" + + else: + if count <= 5: + summary += "\n\nDetails:" + for i, item in enumerate(items, 1): + label = item.get("label") or item.get("id", "") + if label: + summary += f"\n{i}. {label}" + + return summary + + +async def processNaturalLanguageCommand( + currentUser: User, + mandateId: str, + userInput: str, +) -> Dict[str, Any]: + """ + Process natural language user input and execute corresponding CRUD operations. + + Uses AI to analyze user intent and extract parameters, then executes the appropriate + CRUD operation through the interface. Works stateless without session management. + + Args: + currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) + userInput: Natural language command from user + + Returns: + Dictionary containing operation result and metadata + + Example user inputs: + - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + - "Zeige mir alle Projekte in Zürich" + - "Aktualisiere Projekt XYZ mit Status 'Planung'" + - "Lösche Parzelle ABC" + - "SELECT * FROM Projekt WHERE plz = '8000'" + """ + try: + logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {mandateId})") + logger.debug(f"User input: {userInput}") + + services = getServices(currentUser, workflow=None, mandateId=mandateId) + aiService = services.ai + + intentAnalysis = await analyzeUserIntent(aiService, userInput) + + logger.info(f"Intent analysis result: intent={intentAnalysis.get('intent')}, entity={intentAnalysis.get('entity')}") + + result = await executeIntentBasedOperation( + currentUser=currentUser, + mandateId=mandateId, + intent=intentAnalysis["intent"], + entity=intentAnalysis.get("entity"), + parameters=intentAnalysis.get("parameters", {}), + ) + + response = { + "success": True, + "intent": intentAnalysis["intent"], + "entity": intentAnalysis.get("entity"), + "result": result, + } + + if intentAnalysis["intent"] == "CREATE" and isinstance(result, dict): + operation_result = result.get("result") + if isinstance(operation_result, dict): + entity_name = intentAnalysis.get('entity', 'Eintrag') + label = operation_result.get("label", operation_result.get("id", "")) + + msg_parts = [f"✅ {entity_name} '{label}' erfolgreich erstellt"] + + if entity_name == "Parzelle": + if operation_result.get("plz"): + msg_parts.append(f"PLZ: {operation_result['plz']}") + if operation_result.get("kontextGemeinde"): + msg_parts.append(f"Gemeinde: {operation_result['kontextGemeinde']}") + if operation_result.get("bauzone"): + msg_parts.append(f"Bauzone: {operation_result['bauzone']}") + + kontext_items = operation_result.get("kontextInformationen", []) + if kontext_items: + msg_parts.append(f"\n📋 {len(kontext_items)} Kontextinformationen gespeichert:") + for kontext in kontext_items[:5]: + thema = kontext.get("thema", "") + inhalt = kontext.get("inhalt", "") + if thema and inhalt: + msg_parts.append(f" • {thema}: {inhalt}") + if len(kontext_items) > 5: + msg_parts.append(f" • ... und {len(kontext_items) - 5} weitere") + + elif entity_name == "Projekt": + if operation_result.get("statusProzess"): + msg_parts.append(f"Status: {operation_result['statusProzess']}") + parzellen = operation_result.get("parzellen", []) + if parzellen: + msg_parts.append(f"{len(parzellen)} Parzelle(n)") + + response["message"] = "\n".join(msg_parts) + + elif intentAnalysis["intent"] == "READ" and isinstance(result, dict): + operation_result = result.get("result") + if isinstance(operation_result, list): + response["count"] = len(operation_result) + entity_name = intentAnalysis.get('entity', 'Einträge') + + if len(operation_result) == 0: + filter_info = intentAnalysis.get('parameters', {}) + if filter_info: + filter_desc = ", ".join([f"{k}={v}" for k, v in filter_info.items()]) + response["message"] = f"Keine {entity_name} gefunden mit Filter: {filter_desc}. Möglicherweise sind noch keine Daten vorhanden oder der Filter ist zu spezifisch." + else: + response["message"] = f"Keine {entity_name} vorhanden. Erstellen Sie zuerst neue Einträge." + else: + response["message"] = _formatEntitySummary( + entity_name, + operation_result, + intentAnalysis.get('parameters', {}) + ) + elif isinstance(operation_result, dict): + response["count"] = 1 + entity_name = intentAnalysis.get('entity', 'Eintrag') + response["message"] = _formatEntitySummary(entity_name, [operation_result], {}) + + return response + + except Exception as e: + logger.error(f"Error processing natural language command: {str(e)}", exc_info=True) + raise + + +async def analyzeUserIntent( + aiService, + userInput: str +) -> Dict[str, Any]: + """ + Use AI to analyze user input and extract intent, entity, and parameters. + + Args: + aiService: AI service instance + userInput: Natural language user input + + Returns: + Dictionary with 'intent', 'entity', and 'parameters' + """ + intentPrompt = f""" +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "{userInput}" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) +- kontextInformationen: List[Kontext] (metadata - each item has 'thema' and 'inhalt' fields only) + +**Kontext** (Context information for metadata): +- thema: string (topic/subject, e.g. "EGRID", "Fläche", "Zentrum") +- inhalt: string (content as text, e.g. "CH887199917793", "6514.99 m²", "X: 123, Y: 456") + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": {{ + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }}, + "confidence": 0.0-1.0 // Confidence score for the analysis +}} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {{"intent": "CREATE", "entity": "Projekt", "parameters": {{"label": "Hauptstrasse 42"}}, "confidence": 0.95}} + +- Input: "Erstelle eine Parzelle mit Label 123, PLZ 8000, Gemeinde Zürich, Bauzone W3" + Output: {{"intent": "CREATE", "entity": "Parzelle", "parameters": {{"label": "123", "plz": "8000", "kontextGemeinde": "Zürich", "bauzone": "W3"}}, "confidence": 0.95}} + +- Input: "Parzellen-Informationen: ID:AA1704, Nummer:AA1704, EGRID:CH887199917793, Kanton:ZH, Gemeinde:Zürich, Gemeinde-Code:261, Fläche:6514.99 m², Zentrum:2682951.44,1247622.91" + Output: {{ + "intent": "CREATE", + "entity": "Parzelle", + "parameters": {{ + "label": "AA1704", + "parzellenAliasTags": ["AA1704"], + "kontextGemeinde": "Zürich", + "kontextInformationen": [ + {{"thema": "EGRID", "inhalt": "CH887199917793"}}, + {{"thema": "Kanton", "inhalt": "ZH"}}, + {{"thema": "BFS-Nummer", "inhalt": "261"}}, + {{"thema": "Fläche", "inhalt": "6514.99 m²"}}, + {{"thema": "Zentrum (LV95)", "inhalt": "X: 2682951.44 m, Y: 1247622.91 m (EPSG:2056)"}} + ] + }}, + "confidence": 0.9 + }} + Note: Extract structured data from detailed input. Use kontextInformationen for metadata. Each item has 'thema' (topic) and 'inhalt' (content as text). + +- Input: "Zeige mir alle Projekte" + Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{}}, "confidence": 0.9}} + +- Input: "Zeige mir Projekte in Zürich" or "Wie viele Projekte in Zürich" + Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{"location_filter": "Zürich"}}, "confidence": 0.9}} + Note: For project location queries, use Projekt entity with location_filter parameter + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {{"intent": "READ", "entity": "Parzelle", "parameters": {{"plz": "8000"}}, "confidence": 0.95}} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {{"intent": "UPDATE", "entity": "Projekt", "parameters": {{"id": "XYZ", "statusProzess": "Planung"}}, "confidence": 0.85}} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {{"intent": "QUERY", "entity": null, "parameters": {{"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}}, "confidence": 1.0}} + +- Input: "Lösche Parzelle ABC" + Output: {{"intent": "DELETE", "entity": "Parzelle", "parameters": {{"id": "ABC"}}, "confidence": 0.9}} + +IMPORTANT EXTRACTION RULES: +1. For CREATE operations, extract ALL mentioned data fields from the user input +2. Use kontextInformationen array for metadata that doesn't have dedicated fields (EGRID, BFS numbers, area, coordinates, etc.) +3. Each kontextInformationen item MUST have exactly two fields: 'thema' (topic/subject) and 'inhalt' (content as text string) +4. Format kontextInformationen values as readable text strings, including units (e.g., "6514.99 m²", "X: 123, Y: 456") +5. Match field names EXACTLY to the entity definition above +6. Convert data types correctly (strings for text, numbers for numeric values) +7. Extract coordinates, areas, and other numeric values from text +8. When multiple values are mentioned for the same concept (ID, Nummer, Name), use the most relevant one for 'label' and put alternatives in parzellenAliasTags +""" + + try: + response = await aiService.callAiPlanning( + prompt=intentPrompt, + debugType="intentanalysis" + ) + + jsonStart = response.find('{') + jsonEnd = response.rfind('}') + 1 + + if jsonStart == -1 or jsonEnd == 0: + raise ValueError("No JSON found in AI response") + + jsonStr = response[jsonStart:jsonEnd] + + intentData = json.loads(jsonStr) + + if "intent" not in intentData: + raise ValueError("Invalid intent analysis response: missing 'intent' field") + + if "parameters" not in intentData: + intentData["parameters"] = {} + + logger.debug(f"Parsed intent analysis: {intentData}") + + return intentData + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse AI intent analysis response: {e}") + logger.error(f"Raw response: {response}") + raise ValueError(f"AI returned invalid JSON: {str(e)}") + except Exception as e: + logger.error(f"Error analyzing user intent: {str(e)}", exc_info=True) + raise + + +async def executeIntentBasedOperation( + currentUser: User, + mandateId: str, + intent: str, + entity: Optional[str], + parameters: Dict[str, Any], +) -> Dict[str, Any]: + """ + Execute CRUD operation based on analyzed intent. + + Args: + currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) + intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) + entity: Entity type from AI analysis + parameters: Extracted parameters from AI analysis + + Returns: + Operation result + + Note: + - Supports CREATE, READ, UPDATE, DELETE, QUERY intents + - Entity types: Projekt, Parzelle, Gemeinde, Kanton, Land, Dokument + """ + try: + logger.info(f"Executing intent-based operation: intent={intent}, entity={entity}") + logger.debug(f"Parameters: {parameters}") + + if intent == "QUERY": + queryText = parameters.get("queryText", "") + + if not queryText: + raise ValueError("QUERY intent requires queryText in parameters") + + result = await executeDirectQuery( + currentUser=currentUser, + mandateId=mandateId, + queryText=queryText, + parameters=parameters.get("queryParameters"), + ) + return result + + elif intent == "CREATE": + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) + + if entity == "Projekt": + projekt = Projekt( + mandateId=mandateId, + label=parameters.get("label", ""), + statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, + ) + created = realEstateInterface.createProjekt(projekt) + return { + "operation": "CREATE", + "entity": "Projekt", + "result": created.model_dump() + } + + elif entity == "Parzelle": + from modules.features.realestate.datamodelFeatureRealEstate import Kontext, GeoPolylinie + + parzelle_data = { + "mandateId": mandateId, + "label": parameters.get("label", ""), + } + + optional_fields = [ + "parzellenAliasTags", "eigentuemerschaft", "strasseNr", "plz", + "bauzone", "az", "bz", "vollgeschossZahl", "anrechenbarDachgeschoss", + "anrechenbarUntergeschoss", "gebaeudehoeheMax", "kontextGemeinde", + "regelnGrenzabstand", "regelnMehrlaengenzuschlag", "regelnMehrhoehenzuschlag", + "parzelleBebaut", "parzelleErschlossen", "parzelleHanglage", + "laermschutzzone", "hochwasserschutzzone", "grundwasserschutzzone" + ] + + for field in optional_fields: + if field in parameters and parameters[field] is not None: + parzelle_data[field] = parameters[field] + + if "perimeter" in parameters and parameters["perimeter"]: + parzelle_data["perimeter"] = GeoPolylinie(**parameters["perimeter"]) + elif "kontextGemeinde" in parameters and parameters.get("kontextGemeinde"): + gemeinde = parameters.get("kontextGemeinde") + parzellen_nr = parameters.get("label") or parameters.get("parzellen_nr") or parameters.get("parzellennummer") + + if gemeinde and parzellen_nr: + logger.info(f"Attempting to fetch polygon from Swisstopo for {gemeinde} {parzellen_nr}") + try: + gemeinde_name = gemeinde + if len(gemeinde) == 36: # UUID format + gemeinde_obj = realEstateInterface.getGemeinde(gemeinde) + if gemeinde_obj: + gemeinde_name = gemeinde_obj.label + + polygon_data = await fetch_parcel_polygon_from_swisstopo( + gemeinde=gemeinde_name, + parzellen_nr=str(parzellen_nr), + sr=2056 + ) + + if polygon_data: + parzelle_data["perimeter"] = GeoPolylinie(**polygon_data) + logger.info(f"Successfully fetched and set perimeter from Swisstopo") + else: + logger.warning(f"Could not fetch polygon from Swisstopo for {gemeinde_name} {parzellen_nr}") + except Exception as e: + logger.warning(f"Error fetching polygon from Swisstopo (continuing without): {e}") + + if "baulinie" in parameters and parameters["baulinie"]: + parzelle_data["baulinie"] = GeoPolylinie(**parameters["baulinie"]) + + if "kontextInformationen" in parameters and parameters["kontextInformationen"]: + kontext_list = [] + for kontext_data in parameters["kontextInformationen"]: + if isinstance(kontext_data, dict): + kontext_obj = Kontext( + thema=kontext_data.get("thema", ""), + inhalt=kontext_data.get("inhalt", "") + ) + kontext_list.append(kontext_obj) + else: + kontext_list.append(kontext_data) + parzelle_data["kontextInformationen"] = kontext_list + + parzelle = Parzelle(**parzelle_data) + created = realEstateInterface.createParzelle(parzelle) + + logger.info(f"Created Parzelle '{created.label}' with {len(created.kontextInformationen)} context items") + + return { + "operation": "CREATE", + "entity": "Parzelle", + "result": created.model_dump() + } + elif entity == "Gemeinde": + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde + gemeinde = Gemeinde( + mandateId=mandateId, + label=parameters.get("label", ""), + id_kanton=parameters.get("id_kanton"), + plz=parameters.get("plz"), + ) + created = realEstateInterface.createGemeinde(gemeinde) + return { + "operation": "CREATE", + "entity": "Gemeinde", + "result": created.model_dump() + } + elif entity == "Kanton": + from modules.features.realestate.datamodelFeatureRealEstate import Kanton + kanton = Kanton( + mandateId=mandateId, + label=parameters.get("label", ""), + id_land=parameters.get("id_land"), + abk=parameters.get("abk"), + ) + created = realEstateInterface.createKanton(kanton) + return { + "operation": "CREATE", + "entity": "Kanton", + "result": created.model_dump() + } + elif entity == "Land": + from modules.features.realestate.datamodelFeatureRealEstate import Land + land = Land( + mandateId=mandateId, + label=parameters.get("label", ""), + abk=parameters.get("abk"), + ) + created = realEstateInterface.createLand(land) + return { + "operation": "CREATE", + "entity": "Land", + "result": created.model_dump() + } + elif entity == "Dokument": + from modules.features.realestate.datamodelFeatureRealEstate import Dokument + dokument = Dokument( + mandateId=mandateId, + label=parameters.get("label", ""), + dokumentReferenz=parameters.get("dokumentReferenz", ""), + versionsbezeichnung=parameters.get("versionsbezeichnung"), + dokumentTyp=parameters.get("dokumentTyp"), + quelle=parameters.get("quelle"), + mimeType=parameters.get("mimeType"), + ) + created = realEstateInterface.createDokument(dokument) + return { + "operation": "CREATE", + "entity": "Dokument", + "result": created.model_dump() + } + else: + raise ValueError(f"CREATE operation not supported for entity: {entity}") + + elif intent == "READ": + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if projektId: + projekt = realEstateInterface.getProjekt(projektId) + if not projekt: + raise ValueError(f"Projekt {projektId} not found") + return { + "operation": "READ", + "entity": "Projekt", + "result": projekt.model_dump() + } + else: + validProjektFields = {"id", "mandateId", "label", "statusProzess"} + recordFilter = { + k: v for k, v in parameters.items() + if k != "id" and k in validProjektFields + } + + location_filter = parameters.get("location_filter") + + projekte = realEstateInterface.getProjekte(recordFilter=recordFilter if recordFilter else None) + + if location_filter: + logger.info(f"Filtering projects by location: {location_filter}") + + location_id = None + try: + uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE) + if not uuid_pattern.match(location_filter): + gemeinde_records = realEstateInterface.getGemeinden(recordFilter={"label": location_filter}) + if gemeinde_records: + location_id = gemeinde_records[0].id + logger.debug(f"Resolved location '{location_filter}' to ID '{location_id}'") + except Exception as e: + logger.debug(f"Could not resolve location filter: {e}") + + filtered_projekte = [] + + for projekt in projekte: + for parzelle in projekt.parzellen: + location_lower = location_filter.lower() + matches = False + + if parzelle.kontextGemeinde: + if (parzelle.kontextGemeinde == location_id or + parzelle.kontextGemeinde == location_filter or + location_lower in parzelle.kontextGemeinde.lower()): + matches = True + + if not matches and ( + (parzelle.plz and location_lower in parzelle.plz) or + (parzelle.strasseNr and location_lower in parzelle.strasseNr.lower()) + ): + matches = True + + if matches: + filtered_projekte.append(projekt) + break + + projekte = filtered_projekte + logger.info(f"Found {len(projekte)} projects in location '{location_filter}'") + + return { + "operation": "READ", + "entity": "Projekt", + "result": [p.model_dump() for p in projekte], + "count": len(projekte) + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if parzelleId: + parzelle = realEstateInterface.getParzelle(parzelleId) + if not parzelle: + raise ValueError(f"Parzelle {parzelleId} not found") + return { + "operation": "READ", + "entity": "Parzelle", + "result": parzelle.model_dump() + } + else: + validParzelleFields = { + "id", "mandateId", "label", "strasseNr", "plz", + "kontextGemeinde", + "bauzone", "az", "bz", "vollgeschossZahl", "gebaeudehoeheMax", + "laermschutzzone", "hochwasserschutzzone", "grundwasserschutzzone", + "parzelleBebaut", "parzelleErschlossen", "parzelleHanglage" + } + recordFilter = { + k: v for k, v in parameters.items() + if k != "id" and k in validParzelleFields + } + invalidFields = {k: v for k, v in parameters.items() if k not in validParzelleFields and k != "id"} + if invalidFields: + logger.warning(f"Invalid filter fields for Parzelle ignored: {list(invalidFields.keys())}") + + parzellen = realEstateInterface.getParzellen(recordFilter=recordFilter if recordFilter else None) + + if not parzellen and recordFilter: + logger.info(f"No Parzellen found matching filter: {recordFilter}") + all_parzellen = realEstateInterface.getParzellen(recordFilter=None) + logger.info(f"Total Parzellen in database: {len(all_parzellen)}") + if all_parzellen: + sample_gemeinden = set() + for p in all_parzellen[:10]: + if p.kontextGemeinde: + sample_gemeinden.add(p.kontextGemeinde) + logger.info(f"Sample kontextGemeinde values in database: {sample_gemeinden}") + + return { + "operation": "READ", + "entity": "Parzelle", + "result": [p.model_dump() for p in parzellen], + "count": len(parzellen) + } + elif entity == "Gemeinde": + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if gemeindeId: + gemeinde = realEstateInterface.getGemeinde(gemeindeId) + if not gemeinde: + raise ValueError(f"Gemeinde {gemeindeId} not found") + return { + "operation": "READ", + "entity": "Gemeinde", + "result": gemeinde.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + gemeinden = realEstateInterface.getGemeinden(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Gemeinde", + "result": [g.model_dump() for g in gemeinden], + "count": len(gemeinden) + } + elif entity == "Kanton": + from modules.features.realestate.datamodelFeatureRealEstate import Kanton + kantonId = parameters.get("id") + if kantonId: + kanton = realEstateInterface.getKanton(kantonId) + if not kanton: + raise ValueError(f"Kanton {kantonId} not found") + return { + "operation": "READ", + "entity": "Kanton", + "result": kanton.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + kantone = realEstateInterface.getKantone(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Kanton", + "result": [k.model_dump() for k in kantone], + "count": len(kantone) + } + elif entity == "Land": + from modules.features.realestate.datamodelFeatureRealEstate import Land + landId = parameters.get("id") + if landId: + land = realEstateInterface.getLand(landId) + if not land: + raise ValueError(f"Land {landId} not found") + return { + "operation": "READ", + "entity": "Land", + "result": land.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + laender = realEstateInterface.getLaender(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Land", + "result": [l.model_dump() for l in laender], + "count": len(laender) + } + elif entity == "Dokument": + from modules.features.realestate.datamodelFeatureRealEstate import Dokument + dokumentId = parameters.get("id") + if dokumentId: + dokument = realEstateInterface.getDokument(dokumentId) + if not dokument: + raise ValueError(f"Dokument {dokumentId} not found") + return { + "operation": "READ", + "entity": "Dokument", + "result": dokument.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + dokumente = realEstateInterface.getDokumente(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Dokument", + "result": [d.model_dump() for d in dokumente], + "count": len(dokumente) + } + else: + raise ValueError(f"READ operation not supported for entity: {entity}") + + elif intent == "UPDATE": + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("UPDATE operation requires entity ID") + + projekt = realEstateInterface.getProjekt(projektId) + if not projekt: + raise ValueError(f"Projekt {projektId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateProjekt(projektId, updateData) + return { + "operation": "UPDATE", + "entity": "Projekt", + "result": updated.model_dump() + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if not parzelleId: + raise ValueError("UPDATE operation requires entity ID") + + parzelle = realEstateInterface.getParzelle(parzelleId) + if not parzelle: + raise ValueError(f"Parzelle {parzelleId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateParzelle(parzelleId, updateData) + return { + "operation": "UPDATE", + "entity": "Parzelle", + "result": updated.model_dump() + } + elif entity == "Gemeinde": + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if not gemeindeId: + raise ValueError("UPDATE operation requires entity ID") + + gemeinde = realEstateInterface.getGemeinde(gemeindeId) + if not gemeinde: + raise ValueError(f"Gemeinde {gemeindeId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateGemeinde(gemeindeId, updateData) + return { + "operation": "UPDATE", + "entity": "Gemeinde", + "result": updated.model_dump() + } + elif entity == "Kanton": + from modules.features.realestate.datamodelFeatureRealEstate import Kanton + kantonId = parameters.get("id") + if not kantonId: + raise ValueError("UPDATE operation requires entity ID") + + kanton = realEstateInterface.getKanton(kantonId) + if not kanton: + raise ValueError(f"Kanton {kantonId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateKanton(kantonId, updateData) + return { + "operation": "UPDATE", + "entity": "Kanton", + "result": updated.model_dump() + } + elif entity == "Land": + from modules.features.realestate.datamodelFeatureRealEstate import Land + landId = parameters.get("id") + if not landId: + raise ValueError("UPDATE operation requires entity ID") + + land = realEstateInterface.getLand(landId) + if not land: + raise ValueError(f"Land {landId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateLand(landId, updateData) + return { + "operation": "UPDATE", + "entity": "Land", + "result": updated.model_dump() + } + elif entity == "Dokument": + from modules.features.realestate.datamodelFeatureRealEstate import Dokument + dokumentId = parameters.get("id") + if not dokumentId: + raise ValueError("UPDATE operation requires entity ID") + + dokument = realEstateInterface.getDokument(dokumentId) + if not dokument: + raise ValueError(f"Dokument {dokumentId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateDokument(dokumentId, updateData) + return { + "operation": "UPDATE", + "entity": "Dokument", + "result": updated.model_dump() + } + else: + raise ValueError(f"UPDATE operation not supported for entity: {entity}") + + elif intent == "DELETE": + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteProjekt(projektId) + return { + "operation": "DELETE", + "entity": "Projekt", + "success": success + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if not parzelleId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteParzelle(parzelleId) + return { + "operation": "DELETE", + "entity": "Parzelle", + "success": success + } + elif entity == "Gemeinde": + from modules.features.realestate.datamodelFeatureRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if not gemeindeId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteGemeinde(gemeindeId) + return { + "operation": "DELETE", + "entity": "Gemeinde", + "success": success + } + elif entity == "Kanton": + from modules.features.realestate.datamodelFeatureRealEstate import Kanton + kantonId = parameters.get("id") + if not kantonId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteKanton(kantonId) + return { + "operation": "DELETE", + "entity": "Kanton", + "success": success + } + elif entity == "Land": + from modules.features.realestate.datamodelFeatureRealEstate import Land + landId = parameters.get("id") + if not landId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteLand(landId) + return { + "operation": "DELETE", + "entity": "Land", + "success": success + } + elif entity == "Dokument": + from modules.features.realestate.datamodelFeatureRealEstate import Dokument + dokumentId = parameters.get("id") + if not dokumentId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteDokument(dokumentId) + return { + "operation": "DELETE", + "entity": "Dokument", + "success": success + } + else: + raise ValueError(f"DELETE operation not supported for entity: {entity}") + + else: + raise ValueError(f"Unknown intent: {intent}") + + except Exception as e: + logger.error(f"Error executing intent-based operation: {str(e)}", exc_info=True) + raise diff --git a/modules/features/realEstate/serviceBzo.py b/modules/features/realEstate/serviceBzo.py new file mode 100644 index 00000000..c7510fb3 --- /dev/null +++ b/modules/features/realEstate/serviceBzo.py @@ -0,0 +1,725 @@ +""" +Real Estate feature — BZO (Bau- und Zonenordnung) information extraction. + +Handles extraction of BZO information from PDF documents, filtering rules/zones/articles +by Bauzone, and generating AI summaries for building zone regulations. +""" + +import logging +from typing import Optional, Dict, Any, List + +from fastapi import HTTPException, status + +from modules.datamodels.datamodelUam import User +from .datamodelFeatureRealEstate import DokumentTyp +from modules.serviceCenter.serviceHub import getInterface as getServices +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface +from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface +from modules.features.realEstate.bzoDocumentRetriever import BZODocumentRetriever +from modules.features.realEstate.bzoExtraction import run_extraction, run_bzo_params_extraction +from modules.features.realEstate.parcelSelectionService import compute_selection_summary +from modules.features.realEstate.realEstateGemeindeService import ( + ensure_single_gemeinde, + fetch_bzo_for_gemeinde, +) + +logger = logging.getLogger(__name__) + + +async def extract_bzo_information( + currentUser: User, + gemeinde: str, + bauzone: str, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, + total_area_m2: Optional[float] = None, + parcels: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """ + Extract BZO information from PDF documents for a specific Bauzone in a Gemeinde. + + Retrieves BZO documents for the specified Gemeinde, extracts content using + the BZO extraction pipeline, filters by Bauzone, and uses AI to find relevant information. + When total_area_m2 or parcels are provided, runs Machbarkeitsstudie for structured output. + + Args: + currentUser: Current authenticated user + gemeinde: Gemeinde name (e.g., "Zürich") or ID + bauzone: Bauzone code (e.g., "W3", "W2/30") + mandateId: Optional mandate ID for instance-scoped data (defaults to currentUser.mandateId) + featureInstanceId: Optional feature instance ID for instance-scoped data + total_area_m2: Optional total parcel area (m²) for Machbarkeitsstudie + parcels: Optional list of parcel dicts; total area computed via compute_selection_summary if not total_area_m2 + + Returns: + Dictionary containing: + - bauzone, gemeinde, extracted_content, ai_summary, relevant_rules, documents_processed + - machbarkeitsstudie: Structured Machbarkeitsstudie output when total_area_m2/parcels provided + """ + try: + _mandateId = mandateId or (str(currentUser.mandateId) if currentUser.mandateId else None) + logger.info(f"Extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}' (user: {currentUser.id}, mandate: {_mandateId})") + + realEstateInterface = getRealEstateInterface( + currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId + ) + componentInterface = getComponentInterface( + currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId + ) + + logger.debug(f"Attempting to retrieve Gemeinde '{gemeinde}' for mandate {_mandateId}") + gemeinde_obj = realEstateInterface.getGemeinde(gemeinde) + + if not gemeinde_obj: + logger.debug(f"Gemeinde not found by ID, trying to search by label: {gemeinde}") + record_filter = {"label": gemeinde} + if _mandateId: + record_filter["mandateId"] = _mandateId + gemeinden_by_label = realEstateInterface.getGemeinden( + recordFilter=record_filter + ) + if gemeinden_by_label and len(gemeinden_by_label) > 0: + gemeinde_obj = gemeinden_by_label[0] + logger.info(f"Found Gemeinde by label '{gemeinde}' with ID: {gemeinde_obj.id}") + + if not gemeinde_obj and _mandateId and featureInstanceId: + logger.info(f"Gemeinde '{gemeinde}' not in DB - fetching from Swiss Topo (this Gemeinde only)") + gemeinde_obj = await ensure_single_gemeinde( + realEstateInterface, _mandateId, featureInstanceId, gemeinde_name=gemeinde + ) + + if not gemeinde_obj: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Gemeinde '{gemeinde}' not found or not accessible" + ) + + gemeinde_id = gemeinde_obj.id + + bzo_documents = [] + if gemeinde_obj.dokumente: + for doc in gemeinde_obj.dokumente: + if isinstance(doc, dict): + doc_id = doc.get("id") + doc_typ = doc.get("dokumentTyp") + else: + doc_id = doc.id if hasattr(doc, "id") else None + doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None + + if doc_typ: + if isinstance(doc_typ, DokumentTyp): + is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] + elif isinstance(doc_typ, str): + is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] + else: + doc_typ_str = str(doc_typ) + is_bzo = doc_typ_str in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] + + if is_bzo: + if doc_id: + full_doc = realEstateInterface.getDokument(doc_id) + if full_doc: + bzo_documents.append(full_doc) + else: + logger.warning(f"Document {doc_id} referenced in Gemeinde but not found in database") + + if not bzo_documents and _mandateId and featureInstanceId: + logger.info(f"No BZO documents for Gemeinde '{gemeinde_obj.label}' - fetching from web") + fetched = await fetch_bzo_for_gemeinde( + realEstateInterface, componentInterface, gemeinde_obj, _mandateId, featureInstanceId + ) + if fetched: + gemeinde_obj = realEstateInterface.getGemeinde(gemeinde_obj.id) + bzo_documents = [] + if gemeinde_obj and gemeinde_obj.dokumente: + for doc in gemeinde_obj.dokumente: + if isinstance(doc, dict): + doc_id = doc.get("id") + doc_typ = doc.get("dokumentTyp") + else: + doc_id = doc.id if hasattr(doc, "id") else None + doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None + if doc_typ: + if isinstance(doc_typ, DokumentTyp): + is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] + elif isinstance(doc_typ, str): + is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] + else: + is_bzo = str(doc_typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"] + if is_bzo and doc_id: + full_doc = realEstateInterface.getDokument(doc_id) + if full_doc: + bzo_documents.append(full_doc) + + if not bzo_documents: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"No BZO documents found for Gemeinde '{gemeinde_obj.label}'" + ) + + logger.info(f"Found {len(bzo_documents)} BZO document(s) for Gemeinde '{gemeinde_obj.label}'") + + document_retriever = BZODocumentRetriever(realEstateInterface, componentInterface) + + all_extracted_content = { + "articles": [], + "zones": [], + "rules": [], + "zone_parameter_tables": [], + "errors": [], + "warnings": [] + } + documents_processed = [] + + for dokument in bzo_documents: + try: + logger.info(f"Processing document {dokument.id}: {dokument.label}") + + pdf_bytes = document_retriever.retrieve_pdf_content(dokument) + if not pdf_bytes: + logger.warning(f"Could not retrieve PDF content for dokument {dokument.id}") + all_extracted_content["warnings"].append( + f"Could not retrieve PDF content for document '{dokument.label}'" + ) + continue + + extraction_result = run_extraction( + pdf_bytes=pdf_bytes, + pdf_id=dokument.dokumentReferenz or f"dok_{dokument.id}", + dokument_id=dokument.id + ) + + all_extracted_content["articles"].extend(extraction_result.get("articles", [])) + all_extracted_content["zones"].extend(extraction_result.get("zones", [])) + all_extracted_content["rules"].extend(extraction_result.get("rules", [])) + all_extracted_content["zone_parameter_tables"].extend(extraction_result.get("zone_parameter_tables", [])) + all_extracted_content["errors"].extend(extraction_result.get("errors", [])) + all_extracted_content["warnings"].extend(extraction_result.get("warnings", [])) + + documents_processed.append({ + "id": dokument.id, + "label": dokument.label, + "dokumentTyp": dokument.dokumentTyp.value if dokument.dokumentTyp else None + }) + + except Exception as e: + logger.error(f"Error processing document {dokument.id}: {str(e)}", exc_info=True) + all_extracted_content["errors"].append( + f"Error processing document '{dokument.label}': {str(e)}" + ) + continue + + relevant_rules = filter_rules_by_bauzone( + all_extracted_content["rules"], + bauzone + ) + logger.info(f"Extracting for Bauzone {bauzone}: {len(relevant_rules)} zone-specific rules, " + f"{len([t for t in all_extracted_content.get('zone_parameter_tables', []) if bauzone.upper() in str(t.get('zones', [])).upper()])} tables with zone data") + + relevant_zones = filter_zones_by_bauzone( + all_extracted_content["zones"], + bauzone + ) + + relevant_articles = filter_articles_by_bauzone( + all_extracted_content.get("articles", []), + bauzone + ) + + _total_area_m2 = total_area_m2 + if _total_area_m2 is None and parcels: + selection_summary = compute_selection_summary(parcels) + _total_area_m2 = selection_summary.get("total_area_m2") or 0.0 + + bzo_params_result = None + try: + services = getServices( + currentUser, workflow=None, mandateId=_mandateId, featureInstanceId=featureInstanceId + ) + ai_service = services.ai + bzo_params_result = await run_bzo_params_extraction( + extracted_content=all_extracted_content, + bauzone=bauzone, + ai_service=ai_service, + gemeinde=gemeinde_obj.label, + relevant_rules=relevant_rules, + relevant_articles=relevant_articles, + total_area_m2=_total_area_m2, + ) + except Exception as me: + logger.warning(f"BZO parameter extraction failed: {me}", exc_info=True) + all_extracted_content["warnings"] = all_extracted_content.get("warnings", []) + [ + f"BZO-Parameter konnten nicht extrahiert werden: {str(me)}" + ] + + ai_summary = await generate_bauzone_ai_summary( + currentUser=currentUser, + bauzone=bauzone, + gemeinde=gemeinde_obj.label, + extracted_content=all_extracted_content, + relevant_rules=relevant_rules, + relevant_zones=relevant_zones, + mandateId=_mandateId, + featureInstanceId=featureInstanceId, + ) + + unified_summary = ai_summary + + summary_lower = unified_summary.lower() + + zones_mentioned = any(zone.get("zone_code", "").upper() in summary_lower for zone in relevant_zones) + if not zones_mentioned and relevant_zones: + unified_summary += "\n\n=== ZONENDEFINITIONEN ===\n" + for zone in relevant_zones: + zone_code = zone.get("zone_code", "") + zone_name = zone.get("zone_name", "") + zone_category = zone.get("zone_category", "") + geschosszahl = zone.get("geschosszahl") + gewerbeerleichterung = zone.get("gewerbeerleichterung", False) + page_num = zone.get("page", 0) + source_article = zone.get("source_article", "") + + zone_info = f"{zone_code}: {zone_name}" + if zone_category: + zone_info += f"\nKategorie: {zone_category}" + if geschosszahl: + zone_info += f"\nGeschosszahl: {geschosszahl}" + if gewerbeerleichterung: + zone_info += "\nGewerbeerleichterung: Ja" + if source_article: + zone_info += f"\nQuelle: {source_article} (Seite {page_num})" + unified_summary += zone_info + "\n\n" + + articles_mentioned = any(article.get("article_label", "") in summary_lower for article in relevant_articles) + if not articles_mentioned and relevant_articles: + unified_summary += "\n\n=== RELEVANTE ARTIKEL ===\n" + for article in relevant_articles: + article_label = article.get("article_label", "") + article_title = article.get("article_title", "") + article_text = article.get("text", "") + page_start = article.get("page_start", 0) + page_end = article.get("page_end", 0) + page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}" + + unified_summary += f"{article_label}" + if article_title: + unified_summary += f": {article_title}" + unified_summary += f" ({page_range})\n" + if article_text: + preview = article_text[:500] + "..." if len(article_text) > 500 else article_text + unified_summary += f"{preview}\n\n" + + return { + "bauzone": bauzone, + "gemeinde": { + "id": gemeinde_obj.id, + "label": gemeinde_obj.label, + "plz": gemeinde_obj.plz + }, + "extracted_content": { + "zones": relevant_zones, + "rules": relevant_rules, + "articles": relevant_articles, + "zone_parameter_tables": _filter_tables_by_bauzone( + all_extracted_content.get("zone_parameter_tables", []), + bauzone + ), + "total_zones": len(all_extracted_content.get("zones", [])), + "total_rules": len(all_extracted_content.get("rules", [])), + "total_articles": len(all_extracted_content.get("articles", [])), + "total_tables": len(all_extracted_content.get("zone_parameter_tables", [])) + }, + "ai_summary": unified_summary, + "relevant_rules": relevant_rules, + "documents_processed": documents_processed, + "errors": all_extracted_content.get("errors", []), + "warnings": all_extracted_content.get("warnings", []), + "machbarkeitsstudie": bzo_params_result, + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}': {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error extracting BZO information: {str(e)}" + ) + + +def filter_rules_by_bauzone(rules: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: + """ + Filter rules by Bauzone code. Only keeps rules from SINGLE-zone articles to avoid + wrong values (e.g. article with W2,W3,W5 has different values per zone - we cannot + associate a rule value with a specific zone from article text alone). + """ + relevant_rules = [] + bauzone_upper = bauzone.upper() + + def _zone_matches(z: str) -> bool: + zu = (z or "").upper().strip() + if not zu: + return False + if bauzone_upper in zu: + return True + if zu in bauzone_upper and len(zu) >= 2: + return True + return False + + for rule in rules: + table_zones = rule.get("table_zones", []) or [] + zone_raw = rule.get("zone_raw") + + has_zone = bool(zone_raw) or bool(table_zones) + if not has_zone: + continue + + if len(table_zones) > 1: + matches_all = all(_zone_matches(str(z)) for z in table_zones) + if not matches_all: + continue + + matches = False + if zone_raw and _zone_matches(zone_raw): + matches = True + if not matches and table_zones: + for tz in table_zones: + if _zone_matches(str(tz)): + matches = True + break + if not matches: + ts = (rule.get("text_snippet") or "").upper() + if bauzone_upper in ts and len(table_zones) <= 1: + matches = True + + if matches: + relevant_rules.append(rule) + + logger.info(f"Filtered {len(relevant_rules)} rules for Bauzone {bauzone} from {len(rules)} total (multi-zone articles excluded)") + return relevant_rules + + +def filter_zones_by_bauzone(zones: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: + """ + Filter zones by Bauzone code. + + Args: + zones: List of zone dictionaries from extraction + bauzone: Bauzone code to filter by + + Returns: + Filtered list of zones that match the Bauzone + """ + relevant_zones = [] + bauzone_upper = bauzone.upper() + + for zone in zones: + zone_code = zone.get("zone_code", "") + if bauzone_upper in zone_code.upper(): + relevant_zones.append(zone) + + logger.info(f"Filtered {len(relevant_zones)} zones for Bauzone {bauzone} from {len(zones)} total zones") + return relevant_zones + + +def filter_articles_by_bauzone(articles: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: + """ + Filter articles that mention the Bauzone. + + Args: + articles: List of article dictionaries from extraction + bauzone: Bauzone code to filter by + + Returns: + Filtered list of articles that mention the Bauzone + """ + relevant_articles = [] + bauzone_upper = bauzone.upper() + + for article in articles: + text = article.get("text", "") + zone_raw = article.get("zone_raw") + + text_matches = bauzone_upper in text.upper() if text else False + zone_matches = bauzone_upper in zone_raw.upper() if zone_raw else False + + if text_matches or zone_matches: + relevant_articles.append(article) + + logger.info(f"Filtered {len(relevant_articles)} articles for Bauzone {bauzone} from {len(articles)} total articles") + return relevant_articles + + +def _filter_tables_by_bauzone(tables: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: + """ + Filter zone-parameter tables to include only those containing the specified Bauzone. + + Args: + tables: List of zone-parameter table dictionaries + bauzone: Bauzone code to filter by + + Returns: + Filtered list of tables containing the Bauzone + """ + relevant_tables = [] + bauzone_upper = bauzone.upper() + + for table in tables: + zones = table.get("zones", []) + matching_zones = [z for z in zones if bauzone_upper in str(z).upper()] + + if matching_zones: + filtered_table = { + "page": table.get("page"), + "zones": matching_zones, + "parameters": [] + } + + for param in table.get("parameters", []): + values_by_zone = param.get("values_by_zone", {}) + filtered_values = { + zone: values_by_zone[zone] + for zone in matching_zones + if zone in values_by_zone + } + + if filtered_values: + filtered_table["parameters"].append({ + "parameter": param.get("parameter"), + "values_by_zone": filtered_values + }) + + if filtered_table["parameters"]: + relevant_tables.append(filtered_table) + + logger.info(f"Filtered {len(relevant_tables)} tables for Bauzone {bauzone} from {len(tables)} total tables") + return relevant_tables + + +async def generate_bauzone_ai_summary( + currentUser: User, + bauzone: str, + gemeinde: str, + extracted_content: Dict[str, Any], + relevant_rules: List[Dict[str, Any]], + relevant_zones: List[Dict[str, Any]], + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, +) -> str: + """ + Use AI to generate a summary of relevant information for a Bauzone. + + Args: + currentUser: Current authenticated user + bauzone: Bauzone code + gemeinde: Gemeinde name + extracted_content: All extracted content from PDFs + relevant_rules: Rules filtered by Bauzone + relevant_zones: Zones filtered by Bauzone + + Returns: + AI-generated summary string + """ + try: + services = getServices( + currentUser, workflow=None, mandateId=mandateId, featureInstanceId=featureInstanceId + ) + aiService = services.ai + + context_parts = [] + + zone_parameter_tables = extracted_content.get("zone_parameter_tables", []) + table_values_for_bauzone = [] + + if zone_parameter_tables: + context_parts.append("=== BUILDING REGULATIONS TABLE VALUES FOR BAUZONE (INCLUDE THESE EXACT VALUES IN YOUR SUMMARY) ===") + for table in zone_parameter_tables: + page_num = table.get("page", 0) + article_ref = table.get("article", "Unknown article") + zones_in_table = table.get("zones", []) + + matching_zones = [z for z in zones_in_table if bauzone.upper() in str(z).upper()] + + if matching_zones: + context_parts.append(f"\nTabelle aus {article_ref} (Seite {page_num}):") + + for param in table.get("parameters", []): + param_name = param.get("parameter", "") + values_by_zone = param.get("values_by_zone", {}) + + for zone, values in values_by_zone.items(): + if bauzone.upper() in zone.upper(): + if isinstance(values, list) and len(values) > 0: + val_entry = values[0] + value = val_entry.get("value", "") + unit = val_entry.get("unit", "") + unit_str = f" {unit}" if unit else "" + + formatted_param = param_name + if "Ausnützungsziffer" in param_name or "ausnützungsziffer" in param_name.lower(): + formatted_param = "Ausnützungsziffer max." + elif "Vollgeschosse" in param_name or "vollgeschosse" in param_name.lower(): + formatted_param = "Vollgeschosse max." + elif "Gebäudelänge" in param_name or "gebäudelänge" in param_name.lower(): + formatted_param = "Gebäudelänge max." + elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Grundabstand" in param_name or "grundabstand" in param_name.lower()): + formatted_param = "Grenzabstand - Grundabstand min." + elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Mehrlängen" in param_name or "mehrlängen" in param_name.lower()): + formatted_param = "Grenzabstand - Mehrlängen-zuschlag" + elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Höchstmass" in param_name or "höchstmass" in param_name.lower() or "Höchstmaß" in param_name): + formatted_param = "Grenzabstand - Höchstmass max." + elif "Fassadenhöhen" in param_name or "fassadenhöhen" in param_name.lower(): + formatted_param = "Fassadenhöhen max." + elif "Dachgeschosse" in param_name or "dachgeschosse" in param_name.lower(): + formatted_param = "anrechenbare Dachgeschosse max." + elif "Attikageschoss" in param_name or "attikageschoss" in param_name.lower(): + formatted_param = "anrechenbares Attikageschoss max." + elif "Untergeschoss" in param_name or "untergeschoss" in param_name.lower(): + formatted_param = "anrechenbares Untergeschoss max." + + table_values_for_bauzone.append({ + "parameter": formatted_param, + "value": value, + "unit": unit_str, + "article": article_ref, + "page": page_num + }) + context_parts.append(f" • {formatted_param}: {value}{unit_str} (Quelle: {article_ref}, Seite {page_num})") + + if len(values) > 1: + for idx, val_entry in enumerate(values[1:], 1): + value_extra = val_entry.get("value", "") + unit_extra = val_entry.get("unit", "") + unit_str_extra = f" {unit_extra}" if unit_extra else "" + context_parts.append(f" (Alternative: {value_extra}{unit_str_extra})") + + if relevant_zones: + context_parts.append("\n=== ZONE DEFINITIONS ===") + for zone in relevant_zones: + zone_code = zone.get("zone_code", "") + zone_name = zone.get("zone_name", "") + zone_category = zone.get("zone_category", "") + geschosszahl = zone.get("geschosszahl") + gewerbeerleichterung = zone.get("gewerbeerleichterung", False) + page_num = zone.get("page", 0) + source_article = zone.get("source_article", "") + + zone_info = f"- {zone_code}: {zone_name}" + if zone_category: + zone_info += f" (Kategorie: {zone_category})" + if geschosszahl: + zone_info += f", Geschosszahl: {geschosszahl}" + if gewerbeerleichterung: + zone_info += ", Gewerbeerleichterung: Ja" + if source_article: + zone_info += f" - Quelle: {source_article} (Seite {page_num})" + context_parts.append(zone_info) + + relevant_articles = filter_articles_by_bauzone(extracted_content.get("articles", []), bauzone) + if relevant_articles: + context_parts.append("\n=== RELEVANT ARTICLES (full content) ===") + for article in relevant_articles: + article_label = article.get("article_label", "") + article_title = article.get("article_title", "") + article_text = article.get("text", "") + page_start = article.get("page_start", 0) + page_end = article.get("page_end", 0) + page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}" + + context_parts.append(f"\n{article_label}: {article_title or 'Kein Titel'}") + context_parts.append(f"Lage: {page_range}") + if len(article_text) > 1000: + context_parts.append(f"Inhalt: {article_text[:1000]}...") + else: + context_parts.append(f"Inhalt: {article_text}") + + if relevant_rules: + table_parameter_names = set() + for table in zone_parameter_tables: + for param in table.get("parameters", []): + param_name = param.get("parameter", "").lower() + table_parameter_names.add(param_name) + + unique_rules = [] + for rule in relevant_rules[:15]: + rule_type = rule.get("rule_type", "").lower() + if not any(tp in rule_type for tp in table_parameter_names): + unique_rules.append(rule) + + if unique_rules: + context_parts.append("\n=== ADDITIONAL BUILDING REGULATIONS (from text) ===") + for rule in unique_rules[:8]: + rule_type = rule.get("rule_type", "") + value_numeric = rule.get("value_numeric") + value_text = rule.get("value_text", "") + unit = rule.get("unit", "") + page_num = rule.get("page", 0) + + rule_desc = f"- {rule_type}: " + if value_numeric is not None: + rule_desc += f"{value_numeric}" + if unit: + rule_desc += f" {unit}" + else: + rule_desc += value_text + rule_desc += f" (Seite {page_num})" + + context_parts.append(rule_desc) + + context = "\n".join(context_parts) + + prompt = f""" +Analyze the following building zone (Bauzone) information extracted from BZO (Bau- und Zonenordnung) documents for {gemeinde}, specifically for Bauzone {bauzone}. + +Extracted Content: +{context} + +CRITICAL INSTRUCTIONS: +1. You MUST include ALL actual values from the tables in your summary - do NOT just say "see tables on page X" +2. List ALL parameters with their actual values: Ausnützungsziffer, Vollgeschosse, Gebäudelänge, Grenzabstand (Grundabstand, Mehrlängen-zuschlag, Höchstmass), Fassadenhöhen, etc. +3. Integrate zone definitions and article information INTO the summary text - do NOT create separate sections +4. Always cite WHERE each piece of information was found (article number and page number) +5. Combine everything into ONE unified, flowing summary - no separate sections for zones/articles +6. Be comprehensive - include all relevant details from zones, articles, and tables +7. Format as a single, well-structured German text document + +Please provide a comprehensive, unified summary that includes: + +1. General description of Bauzone {bauzone}: + - Zone category (Wohnzonen, Zentrumszonen, etc.) + - Geschosszahl (number of full storeys) + - Gewerbeerleichterung status (Ja/Nein) + - Where defined (article and page number) + +2. ALL building regulations with ACTUAL VALUES from tables (you MUST include the exact values): + - Ausnützungsziffer max.: [ACTUAL PERCENTAGE VALUE]% (from article, page) + - Vollgeschosse max.: [ACTUAL NUMBER] (from article, page) + - anrechenbare Dachgeschosse max.: [ACTUAL NUMBER] (from article, page) + - anrechenbares Attikageschoss max.: [ACTUAL NUMBER] (from article, page) + - anrechenbares Untergeschoss max.: [ACTUAL NUMBER] (from article, page) + - Gebäudelänge max.: [ACTUAL VALUE] m (from article, page) + - Grenzabstand - Grundabstand min.: [ACTUAL VALUE] m (from article, page) + - Grenzabstand - Mehrlängen-zuschlag: [ACTUAL FRACTION] (from article, page) + - Grenzabstand - Höchstmass max.: [ACTUAL VALUE] m (from article, page) + - Fassadenhöhen max.: [ACTUAL VALUE] m (from article, page, include footnote values if present) + +3. Zone definitions: Integrate information about where this zone is defined (which articles mention it, with page numbers) + +4. Relevant articles: Integrate key content from relevant articles naturally into the summary, citing article numbers and page numbers + +5. Special conditions: Any special requirements or exceptions mentioned in articles + +CRITICAL: You MUST include the actual numeric values from the tables in your summary. Do NOT say "see tables" - list the actual values. Format everything as ONE unified, flowing German text document without separate sections. Integrate zones and articles naturally into the narrative. +""" + + logger.info(f"Generating AI summary for Bauzone {bauzone} in {gemeinde}") + ai_response = await aiService.callAiPlanning( + prompt=prompt, + debugType="bzo_summary" + ) + + return ai_response.strip() + + except Exception as e: + logger.error(f"Error generating AI summary: {str(e)}", exc_info=True) + return f"Summary generation failed: {str(e)}. Found {len(relevant_rules)} relevant rules and {len(relevant_zones)} zones for Bauzone {bauzone}." diff --git a/modules/features/realEstate/serviceGeometry.py b/modules/features/realEstate/serviceGeometry.py new file mode 100644 index 00000000..c8021701 --- /dev/null +++ b/modules/features/realEstate/serviceGeometry.py @@ -0,0 +1,817 @@ +""" +Real Estate feature — Geometry utilities. + +Handles conversion between GeoPolylinie and Shapely polygons, combining +parcel geometries, filtering neighbor parcels, fetching parcel polygons +from Swisstopo, creating projects with parcel data, and GeoJSON conversion. +""" + +import logging +from typing import Optional, Dict, Any, List + +from shapely.geometry import Polygon +from shapely.ops import unary_union + +from .datamodelFeatureRealEstate import ( + Projekt, + Parzelle, + StatusProzess, + GeoPolylinie, + GeoPunkt, + Kontext, + Gemeinde, + Kanton, + Land, +) +from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector +from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface +from modules.datamodels.datamodelUam import User +from fastapi import HTTPException, status + +logger = logging.getLogger(__name__) + + +def geopolylinie_to_shapely_polygon(geopolylinie: GeoPolylinie) -> Polygon: + """ + Convert GeoPolylinie to Shapely Polygon. + + Args: + geopolylinie: GeoPolylinie instance with punkte list + + Returns: + Shapely Polygon object + """ + if not geopolylinie or not geopolylinie.punkte: + raise ValueError("GeoPolylinie must have at least one point") + + coordinates = [] + for punkt in geopolylinie.punkte: + coordinates.append((punkt.x, punkt.y)) + + if len(coordinates) < 3: + raise ValueError("Polygon must have at least 3 points") + + if coordinates[0] != coordinates[-1]: + coordinates.append(coordinates[0]) + + return Polygon(coordinates) + + +def shapely_polygon_to_geopolylinie(polygon: Polygon) -> GeoPolylinie: + """ + Convert Shapely Polygon to GeoPolylinie. + + Args: + polygon: Shapely Polygon object + + Returns: + GeoPolylinie instance with LV95 coordinate system + """ + if not polygon or polygon.is_empty: + raise ValueError("Polygon must not be empty") + + exterior_coords = list(polygon.exterior.coords) + + if len(exterior_coords) > 1 and exterior_coords[0] == exterior_coords[-1]: + exterior_coords = exterior_coords[:-1] + + punkte = [] + for coord in exterior_coords: + punkt = GeoPunkt( + koordinatensystem="LV95", + x=float(coord[0]), + y=float(coord[1]), + z=None + ) + punkte.append(punkt) + + return GeoPolylinie( + closed=True, + punkte=punkte + ) + + +def combine_parcel_geometries(geometries: List[GeoPolylinie]) -> GeoPolylinie: + """ + Combine multiple parcel geometries into a single outer outline. + + Uses Shapely union operation to merge polygons and automatically + removes internal edges. The result is a clean outer boundary. + + Args: + geometries: List of GeoPolylinie instances to combine + + Returns: + Combined GeoPolylinie representing the outer outline + + Raises: + ValueError: If geometries list is empty or invalid + """ + if not geometries or len(geometries) == 0: + raise ValueError("At least one geometry is required") + + if len(geometries) == 1: + return geometries[0] + + shapely_polygons = [] + for geo in geometries: + try: + polygon = geopolylinie_to_shapely_polygon(geo) + if not polygon.is_empty: + shapely_polygons.append(polygon) + except Exception as e: + logger.warning(f"Error converting geometry to Shapely Polygon: {e}") + continue + + if not shapely_polygons: + raise ValueError("No valid geometries to combine") + + if len(shapely_polygons) == 1: + return shapely_polygon_to_geopolylinie(shapely_polygons[0]) + + try: + combined = unary_union(shapely_polygons) + + if hasattr(combined, 'geoms'): + largest = max(combined.geoms, key=lambda p: p.area) + combined = largest + + if combined.is_empty: + raise ValueError("Union resulted in empty geometry") + + result = shapely_polygon_to_geopolylinie(combined) + logger.info(f"Combined {len(geometries)} geometries into single outline with {len(result.punkte)} points") + return result + + except Exception as e: + logger.error(f"Error combining geometries: {e}", exc_info=True) + raise ValueError(f"Failed to combine geometries: {str(e)}") + + +def filter_neighbor_parcels( + neighbors: List[Dict[str, Any]], + selected_geometries: List[GeoPolylinie] +) -> List[Dict[str, Any]]: + """ + Filter neighbor parcels to exclude those that are part of the selected parcels. + + Uses geometric comparison to check if neighbor parcels intersect or touch + any of the selected parcel geometries. + + Args: + neighbors: List of neighbor parcel dictionaries (must have 'perimeter' or 'geometry_geojson') + selected_geometries: List of GeoPolylinie instances representing selected parcels + + Returns: + Filtered list of neighbor parcels (excluding selected ones) + """ + if not neighbors or not selected_geometries: + return neighbors + + selected_polygons = [] + for geo in selected_geometries: + try: + polygon = geopolylinie_to_shapely_polygon(geo) + if not polygon.is_empty: + selected_polygons.append(polygon) + except Exception as e: + logger.warning(f"Error converting selected geometry for filtering: {e}") + continue + + if not selected_polygons: + return neighbors + + filtered_neighbors = [] + for neighbor in neighbors: + try: + neighbor_geometry = None + + if neighbor.get("perimeter"): + perimeter = neighbor["perimeter"] + if isinstance(perimeter, dict) and perimeter.get("punkte"): + punkte = [] + for p in perimeter["punkte"]: + punkt = GeoPunkt( + koordinatensystem=p.get("koordinatensystem", "LV95"), + x=float(p.get("x", 0)), + y=float(p.get("y", 0)), + z=p.get("z") + ) + punkte.append(punkt) + neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte) + + elif neighbor.get("geometry_geojson"): + geo_json = neighbor["geometry_geojson"] + geometry = geo_json.get("geometry") if isinstance(geo_json, dict) else geo_json + + if geometry and geometry.get("type") == "Polygon": + coordinates = geometry.get("coordinates", []) + if coordinates and len(coordinates) > 0: + ring = coordinates[0] + punkte = [] + for coord in ring: + if len(coord) >= 2: + punkt = GeoPunkt( + koordinatensystem="LV95", + x=float(coord[0]), + y=float(coord[1]), + z=float(coord[2]) if len(coord) > 2 else None + ) + punkte.append(punkt) + neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte) + + if not neighbor_geometry: + filtered_neighbors.append(neighbor) + continue + + neighbor_polygon = geopolylinie_to_shapely_polygon(neighbor_geometry) + + is_selected = False + for selected_polygon in selected_polygons: + if neighbor_polygon.intersects(selected_polygon) or neighbor_polygon.touches(selected_polygon): + area_diff = abs(neighbor_polygon.area - selected_polygon.area) + if area_diff < 1.0: + is_selected = True + break + if neighbor_polygon.contains(selected_polygon) or selected_polygon.contains(neighbor_polygon): + is_selected = True + break + + if not is_selected: + filtered_neighbors.append(neighbor) + else: + logger.debug(f"Filtered out neighbor parcel {neighbor.get('id')} - part of selected parcels") + + except Exception as e: + logger.warning(f"Error filtering neighbor parcel {neighbor.get('id')}: {e}") + filtered_neighbors.append(neighbor) + + logger.info(f"Filtered {len(neighbors)} neighbors to {len(filtered_neighbors)} (removed {len(neighbors) - len(filtered_neighbors)} selected parcels)") + return filtered_neighbors + + +async def fetch_parcel_polygon_from_swisstopo( + gemeinde: str, + parzellen_nr: str, + sr: int = 2056 +) -> Optional[Dict[str, Any]]: + """ + Holt die vollständige Polygon-Geometrie einer Parzelle von Swisstopo API. + + Args: + gemeinde: Name der Gemeinde (z.B. "Bern") + parzellen_nr: Parzellennummer (z.B. "1234") + sr: Koordinatensystem (2056=LV95, 4326=WGS84) + + Returns: + Dictionary mit GeoPolylinie-Format für perimeter-Feld, oder None wenn nicht gefunden + Format: {"closed": True, "punkte": [{"koordinatensystem": "LV95", "x": ..., "y": ..., "z": None}, ...]} + """ + try: + connector = SwissTopoMapServerConnector() + + feature = await connector.get_parcel_polygon(gemeinde, parzellen_nr, sr) + + if not feature: + logger.warning(f"Parzelle {gemeinde} {parzellen_nr} nicht gefunden in Swisstopo") + return None + + geometry = feature.get("geometry", {}) + if geometry.get("type") == "Polygon": + coordinates = geometry.get("coordinates", []) + if coordinates and len(coordinates) > 0: + ring = coordinates[0] + + punkte = [] + for coord in ring: + if len(coord) >= 2: + punkt = { + "koordinatensystem": "LV95" if sr == 2056 else "WGS84", + "x": coord[0], + "y": coord[1], + "z": coord[2] if len(coord) > 2 else None + } + punkte.append(punkt) + + logger.info(f"Successfully fetched polygon with {len(punkte)} points for {gemeinde} {parzellen_nr}") + + return { + "closed": True, + "punkte": punkte + } + + logger.warning(f"Unexpected geometry type in Swisstopo response: {geometry.get('type')}") + return None + + except Exception as e: + logger.error(f"Error fetching parcel polygon from Swisstopo: {e}", exc_info=True) + return None + + +def convert_geojson_to_geopolylinie(geometry_data: Dict[str, Any]) -> Optional[GeoPolylinie]: + """Convert GeoJSON geometry to GeoPolylinie format.""" + if not geometry_data: + return None + + if "geometry" in geometry_data: + geometry_data = geometry_data["geometry"] + + geometry_type = geometry_data.get("type") + coordinates = geometry_data.get("coordinates") + + if not coordinates or geometry_type != "Polygon": + return None + + if not coordinates or len(coordinates) == 0: + return None + + ring = coordinates[0] + + punkte = [] + for coord in ring: + if len(coord) >= 2: + punkt = GeoPunkt( + koordinatensystem="LV95", + x=float(coord[0]), + y=float(coord[1]), + z=float(coord[2]) if len(coord) > 2 else None + ) + punkte.append(punkt) + + if not punkte: + return None + + return GeoPolylinie( + closed=True, + punkte=punkte + ) + + +async def create_project_with_parcel_data( + currentUser: User, + mandateId: str, + projekt_label: str, + parzellen_data: List[Dict[str, Any]], + status_prozess: Optional[str] = None, +) -> Dict[str, Any]: + """ + Create a Projekt with one or more Parzellen from provided parcel data. + + Args: + currentUser: Current authenticated user + mandateId: Mandate context (from RequestContext / X-Mandate-Id header) + projekt_label: Label for the Projekt + parzellen_data: List of dictionaries containing parcel information from request + status_prozess: Optional project status (defaults to "Eingang") + + Returns: + Dictionary containing created Projekt and list of Parzellen + + Raises: + HTTPException: If Gemeinde or Kanton not found, or validation fails + """ + try: + logger.info(f"Creating project '{projekt_label}' with {len(parzellen_data)} parcel(s) for user {currentUser.id}") + + realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId) + + if not projekt_label: + raise ValueError("Projekt label is required") + + if not parzellen_data or len(parzellen_data) == 0: + raise ValueError("At least one Parzelle data is required") + + for idx, parzelle_data in enumerate(parzellen_data): + if not parzelle_data.get("perimeter"): + raise ValueError(f"Parzelle {idx + 1} perimeter is required") + + # First pass: Collect all parcel geometries for neighbor filtering + all_parcel_geometries = [] + for parzelle_data in parzellen_data: + perimeter = parzelle_data.get("perimeter") + if perimeter: + if isinstance(perimeter, dict): + if "punkte" in perimeter and "closed" in perimeter: + try: + geo_perimeter = GeoPolylinie(**perimeter) + all_parcel_geometries.append(geo_perimeter) + except Exception as e: + logger.warning(f"Error converting perimeter to GeoPolylinie: {e}") + else: + converted = convert_geojson_to_geopolylinie(perimeter) + if converted: + all_parcel_geometries.append(converted) + elif isinstance(perimeter, GeoPolylinie): + all_parcel_geometries.append(perimeter) + + created_parzellen = [] + parcel_perimeters = [] + + for idx, parzelle_data in enumerate(parzellen_data): + logger.info(f"Processing Parzelle {idx + 1}/{len(parzellen_data)}") + + parcel_label = parzelle_data.get("id") or parzelle_data.get("number") or parzelle_data.get("label") or "Unknown" + + existing_parzellen = realEstateInterface.getParzellen( + recordFilter={"label": parcel_label, "mandateId": mandateId} + ) + + if existing_parzellen and len(existing_parzellen) > 0: + existing_parzelle = existing_parzellen[0] + logger.info(f"Parzelle with label '{parcel_label}' already exists (ID: {existing_parzelle.id}), reusing it") + + if existing_parzelle.perimeter: + parcel_perimeters.append(existing_parzelle.perimeter) + + created_parzellen.append(existing_parzelle) + continue + + logger.info(f"Parzelle with label '{parcel_label}' does not exist, creating new one") + + gemeinde_id = None + canton_abk = parzelle_data.get("canton") + municipality_name = parzelle_data.get("municipality_name") + + logger.debug(f"Resolving Gemeinde/Kanton: canton='{canton_abk}', municipality='{municipality_name}'") + + if municipality_name and canton_abk: + canton_names = { + "ZH": "Zürich", "BE": "Bern", "LU": "Luzern", "UR": "Uri", "SZ": "Schwyz", + "OW": "Obwalden", "NW": "Nidwalden", "GL": "Glarus", "ZG": "Zug", "FR": "Freiburg", + "SO": "Solothurn", "BS": "Basel-Stadt", "BL": "Basel-Landschaft", "SH": "Schaffhausen", + "AR": "Appenzell Ausserrhoden", "AI": "Appenzell Innerrhoden", "SG": "St. Gallen", + "GR": "Graubünden", "AG": "Aargau", "TG": "Thurgau", "TI": "Tessin", + "VD": "Waadt", "VS": "Wallis", "NE": "Neuenburg", "GE": "Genf", "JU": "Jura" + } + + logger.debug("Ensuring Land 'Schweiz' exists") + laender = realEstateInterface.getLaender(recordFilter={"label": "Schweiz"}) + if not laender: + logger.info("Creating Land 'Schweiz'") + land = Land( + mandateId=mandateId, + label="Schweiz", + abk="CH" + ) + land = realEstateInterface.createLand(land) + logger.info(f"Created Land 'Schweiz' with ID: {land.id}") + else: + land = laender[0] + logger.debug(f"Found Land 'Schweiz' with ID: {land.id}") + + logger.debug(f"Looking up Kanton with abk='{canton_abk}'") + kantone = realEstateInterface.getKantone(recordFilter={"abk": canton_abk}) + logger.debug(f"Found {len(kantone)} Kanton(e) with abk='{canton_abk}'") + if not kantone: + logger.info(f"Kanton '{canton_abk}' not found, creating it") + kanton_label = canton_names.get(canton_abk, canton_abk) + kanton = Kanton( + mandateId=mandateId, + label=kanton_label, + abk=canton_abk, + id_land=land.id + ) + kanton = realEstateInterface.createKanton(kanton) + logger.info(f"Created Kanton '{kanton_label}' ({canton_abk}) with ID: {kanton.id}") + else: + kanton = kantone[0] + logger.debug(f"Found Kanton: ID={kanton.id}, Label={kanton.label}, abk={kanton.abk}") + + logger.debug(f"Looking up Gemeinde with label='{municipality_name}' and id_kanton='{kanton.id}'") + gemeinden = realEstateInterface.getGemeinden( + recordFilter={"label": municipality_name, "id_kanton": kanton.id} + ) + logger.debug(f"Found {len(gemeinden)} Gemeinde(n) with label='{municipality_name}' and id_kanton='{kanton.id}'") + if not gemeinden: + logger.info(f"Gemeinde '{municipality_name}' in Kanton '{canton_abk}' not found, creating it") + gemeinde = Gemeinde( + mandateId=mandateId, + label=municipality_name, + id_kanton=kanton.id, + plz=parzelle_data.get("plz") + ) + gemeinde = realEstateInterface.createGemeinde(gemeinde) + logger.info(f"Created Gemeinde '{municipality_name}' with ID: {gemeinde.id}") + else: + gemeinde = gemeinden[0] + logger.debug(f"Found Gemeinde: ID={gemeinde.id}, Label={gemeinde.label}") + + gemeinde_id = gemeinde.id + logger.info(f"Resolved Gemeinde '{municipality_name}' to ID '{gemeinde_id}'") + else: + logger.warning(f"Missing Gemeinde/Kanton data: municipality_name={municipality_name}, canton={canton_abk}") + + alias_tags = [] + if parzelle_data.get("egrid"): + alias_tags.append(parzelle_data["egrid"]) + if parzelle_data.get("number") and parzelle_data["number"] != parzelle_data.get("id"): + alias_tags.append(parzelle_data["number"]) + + strasse_nr = None + plz = None + + address = parzelle_data.get("address") + if address: + parts = address.split(",") + if len(parts) >= 1: + strasse_nr = parts[0].strip() + plz = parzelle_data.get("plz") + + logger.debug(f"Parzelle {idx + 1} address data: strasse_nr='{strasse_nr}', plz='{plz}', full_address='{address}'") + + if not strasse_nr and not plz: + logger.warning(f"No address data found for Parzelle {idx + 1} (label: {parcel_label})") + + kontext_items = [] + + if parzelle_data.get("egrid"): + kontext_items.append(Kontext( + thema="EGRID", + inhalt=parzelle_data["egrid"] + )) + + if parzelle_data.get("identnd"): + kontext_items.append(Kontext( + thema="IdentND", + inhalt=parzelle_data["identnd"] + )) + + if parzelle_data.get("area_m2"): + kontext_items.append(Kontext( + thema="Fläche", + inhalt=f"{parzelle_data['area_m2']} m²" + )) + + if parzelle_data.get("centroid"): + centroid = parzelle_data["centroid"] + kontext_items.append(Kontext( + thema="Zentrum (LV95)", + inhalt=f"X: {centroid.get('x')} m, Y: {centroid.get('y')} m (EPSG:2056)" + )) + + if parzelle_data.get("geoportal_url"): + kontext_items.append(Kontext( + thema="Geoportal URL", + inhalt=parzelle_data["geoportal_url"] + )) + + if parzelle_data.get("municipality_code"): + kontext_items.append(Kontext( + thema="BFS-Nummer", + inhalt=str(parzelle_data["municipality_code"]) + )) + + adjacent_parcel_refs = [] + if parzelle_data.get("adjacent_parcels"): + neighbors_to_filter = [] + for adj_parcel in parzelle_data["adjacent_parcels"]: + if isinstance(adj_parcel, dict): + neighbors_to_filter.append(adj_parcel) + elif isinstance(adj_parcel, str): + neighbors_to_filter.append({"id": adj_parcel}) + + if all_parcel_geometries and neighbors_to_filter: + try: + filtered_neighbors = filter_neighbor_parcels( + neighbors_to_filter, + all_parcel_geometries + ) + for filtered_neighbor in filtered_neighbors: + adj_id = filtered_neighbor.get("id") + if adj_id: + adjacent_parcel_refs.append({"id": adj_id}) + except Exception as e: + logger.warning(f"Error filtering neighbor parcels: {e}, including all neighbors") + for adj_parcel in parzelle_data["adjacent_parcels"]: + if isinstance(adj_parcel, dict): + adj_id = adj_parcel.get("id") + if adj_id: + adjacent_parcel_refs.append({"id": adj_id}) + elif isinstance(adj_parcel, str): + adjacent_parcel_refs.append({"id": adj_parcel}) + else: + for adj_parcel in parzelle_data["adjacent_parcels"]: + if isinstance(adj_parcel, dict): + adj_id = adj_parcel.get("id") + if adj_id: + adjacent_parcel_refs.append({"id": adj_id}) + elif isinstance(adj_parcel, str): + adjacent_parcel_refs.append({"id": adj_parcel}) + + perimeter = parzelle_data.get("perimeter") + if isinstance(perimeter, dict): + if "punkte" in perimeter and "closed" in perimeter: + try: + perimeter = GeoPolylinie(**perimeter) + except Exception as e: + raise ValueError(f"Invalid perimeter format: {str(e)}") + else: + converted = convert_geojson_to_geopolylinie(perimeter) + if converted: + perimeter = converted + else: + raise ValueError("Invalid perimeter format: cannot convert to GeoPolylinie") + elif isinstance(perimeter, GeoPolylinie): + pass + else: + raise ValueError("Invalid perimeter type: must be dict or GeoPolylinie") + + baulinie = None + geometry = parzelle_data.get("geometry") + logger.debug(f"Geometry present: {geometry is not None}") + if geometry: + logger.debug(f"Geometry type: {type(geometry)}, keys: {list(geometry.keys()) if isinstance(geometry, dict) else 'not a dict'}") + baulinie = convert_geojson_to_geopolylinie(geometry) + if baulinie: + logger.info(f"Extracted baulinie from geometry with {len(baulinie.punkte)} points") + else: + logger.warning("Failed to extract baulinie from geometry") + else: + logger.warning("No geometry found in parzelle_data") + + parzelle_create_data = { + "mandateId": mandateId, + "label": parcel_label, + "parzellenAliasTags": alias_tags, + "eigentuemerschaft": None, + "strasseNr": strasse_nr, + "plz": plz, + "perimeter": perimeter, + "baulinie": baulinie, + "kontextGemeinde": gemeinde_id, + "bauzone": None, + "az": None, + "bz": None, + "vollgeschossZahl": None, + "anrechenbarDachgeschoss": None, + "anrechenbarUntergeschoss": None, + "gebaeudehoeheMax": None, + "regelnGrenzabstand": [], + "regelnMehrlaengenzuschlag": [], + "regelnMehrhoehenzuschlag": [], + "parzelleBebaut": None, + "parzelleErschlossen": None, + "parzelleHanglage": None, + "laermschutzzone": None, + "hochwasserschutzzone": None, + "grundwasserschutzzone": None, + "parzellenNachbarschaft": adjacent_parcel_refs, + "dokumente": [], + "kontextInformationen": kontext_items, + } + + logger.debug(f"Creating Parzelle with label: {parzelle_create_data.get('label')}") + logger.debug(f"Parzelle mandateId: {parzelle_create_data.get('mandateId')}") + logger.debug(f"Parzelle perimeter present: {parzelle_create_data.get('perimeter') is not None}") + + try: + parzelle_instance = Parzelle(**parzelle_create_data) + logger.debug(f"Parzelle instance created successfully with ID: {parzelle_instance.id}") + except Exception as e: + logger.error(f"Error creating Parzelle instance: {str(e)}", exc_info=True) + raise + + try: + logger.info(f"Calling createParzelle for Parzelle '{parzelle_instance.label}' (ID: {parzelle_instance.id})") + logger.debug(f"Parzelle instance before createParzelle: {parzelle_instance.model_dump(mode='json', exclude={'perimeter', 'baulinie', 'kontextInformationen'})}") + + parzelle_dict = parzelle_instance.model_dump(mode='json') + logger.debug(f"Parzelle dict keys: {list(parzelle_dict.keys())}") + + created_parzelle = realEstateInterface.createParzelle(parzelle_instance) + + logger.info(f"createParzelle returned: ID={created_parzelle.id if created_parzelle else 'None'}, Label={created_parzelle.label if created_parzelle else 'None'}") + + if not created_parzelle: + raise ValueError("Failed to create Parzelle - createParzelle returned None") + + if not created_parzelle.id: + raise ValueError("Failed to create Parzelle - no ID returned") + + logger.info(f"Parzelle created with ID: {created_parzelle.id}") + + logger.debug(f"Verifying Parzelle {created_parzelle.id} exists in database...") + verify_parzelle = realEstateInterface.getParzelle(created_parzelle.id) + if not verify_parzelle: + logger.error(f"Parzelle {created_parzelle.id} was not found in database after creation") + all_parzellen = realEstateInterface.getParzellen(recordFilter=None) + logger.error(f"Total Parzellen in database: {len(all_parzellen)}") + if all_parzellen: + logger.error(f"Sample Parzelle IDs: {[p.id for p in all_parzellen[:5]]}") + raise ValueError(f"Parzelle {created_parzelle.id} was not found in database after creation") + + logger.info(f"Verified Parzelle {created_parzelle.id} exists in database") + created_parzelle = verify_parzelle + + if created_parzelle.perimeter: + parcel_perimeters.append(created_parzelle.perimeter) + + created_parzellen.append(created_parzelle) + + except Exception as e: + logger.error(f"Error creating Parzelle {idx + 1}: {str(e)}", exc_info=True) + raise + + if not created_parzellen: + raise ValueError("No Parzellen were successfully created") + + logger.info(f"Successfully created {len(created_parzellen)} Parzelle(n)") + + project_baulinie = None + if len(parcel_perimeters) > 0: + try: + if len(parcel_perimeters) == 1: + project_baulinie = parcel_perimeters[0] + logger.info("Using single parcel perimeter as baulinie") + else: + logger.info(f"Combining {len(parcel_perimeters)} parcel geometries to create baulinie") + project_baulinie = combine_parcel_geometries(parcel_perimeters) + logger.info(f"Created combined baulinie with {len(project_baulinie.punkte)} points") + except Exception as e: + logger.error(f"Error combining parcel geometries for baulinie: {e}", exc_info=True) + if parcel_perimeters: + project_baulinie = parcel_perimeters[0] + logger.warning("Using first parcel perimeter as fallback baulinie") + + status_prozess_enum = None + if status_prozess: + try: + if isinstance(status_prozess, str): + status_prozess_enum = StatusProzess(status_prozess) + elif isinstance(status_prozess, StatusProzess): + status_prozess_enum = status_prozess + except (ValueError, KeyError): + logger.warning(f"Invalid statusProzess '{status_prozess}', using default 'Eingang'") + status_prozess_enum = StatusProzess.EINGANG + else: + status_prozess_enum = StatusProzess.EINGANG + + logger.debug(f"Preparing Projekt creation with baulinie: {project_baulinie is not None}") + if project_baulinie: + logger.debug(f"Baulinie has {len(project_baulinie.punkte)} points") + + project_perimeter = created_parzellen[0].perimeter if created_parzellen else None + + projekt_create_data = { + "mandateId": mandateId, + "label": projekt_label, + "statusProzess": status_prozess_enum, + "perimeter": project_perimeter, + "baulinie": project_baulinie, + "parzellen": created_parzellen, + "dokumente": [], + "kontextInformationen": [], + } + + logger.debug(f"Projekt data prepared: label={projekt_label}, parzellen_count={len(projekt_create_data['parzellen'])}, baulinie={'present' if project_baulinie else 'None'}") + + try: + projekt_instance = Projekt(**projekt_create_data) + logger.debug(f"Projekt instance created successfully with ID: {projekt_instance.id}") + except Exception as e: + logger.error(f"Error creating Projekt instance: {str(e)}", exc_info=True) + raise + + logger.debug(f"Creating Projekt with {len(projekt_instance.parzellen)} Parzelle(n)") + if projekt_instance.parzellen: + for idx, p in enumerate(projekt_instance.parzellen): + logger.debug(f" Parzelle {idx}: ID={p.id}, Label={p.label}") + + logger.debug(f"Projekt baulinie before save: {projekt_instance.baulinie is not None}") + if projekt_instance.baulinie: + logger.debug(f"Projekt baulinie has {len(projekt_instance.baulinie.punkte)} points") + + try: + created_projekt = realEstateInterface.createProjekt(projekt_instance) + logger.info(f"Created Projekt '{created_projekt.label}' (ID: {created_projekt.id})") + logger.debug(f"Created Projekt baulinie: {created_projekt.baulinie is not None}") + except Exception as e: + logger.error(f"Error calling createProjekt: {str(e)}", exc_info=True) + raise + + if not created_projekt or not created_projekt.id: + raise ValueError("Failed to create Projekt - no ID returned") + + if not created_projekt.parzellen or len(created_projekt.parzellen) == 0: + logger.warning(f"Projekt {created_projekt.id} created but no Parzellen linked") + verify_projekt = realEstateInterface.getProjekt(created_projekt.id) + if verify_projekt and verify_projekt.parzellen: + logger.info(f"Parzellen found when fetching Projekt from database: {len(verify_projekt.parzellen)}") + created_projekt = verify_projekt + else: + raise ValueError(f"Projekt {created_projekt.id} has no Parzellen linked after creation") + else: + logger.info(f"Projekt {created_projekt.id} successfully linked to {len(created_projekt.parzellen)} Parzelle(n)") + for idx, p in enumerate(created_projekt.parzellen): + logger.debug(f" Linked Parzelle {idx}: ID={p.id if hasattr(p, 'id') else 'NO ID'}, Label={p.label if hasattr(p, 'label') else 'NO LABEL'}") + + return { + "projekt": created_projekt.model_dump(), + "parzellen": [p.model_dump() for p in created_parzellen], + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating project with parcel data: {str(e)}", exc_info=True) + raise diff --git a/modules/features/teamsbot/service.py b/modules/features/teamsbot/service.py index edf5af94..2bafd0e2 100644 --- a/modules/features/teamsbot/service.py +++ b/modules/features/teamsbot/service.py @@ -44,31 +44,19 @@ from .browserBotConnector import BrowserBotConnector logger = logging.getLogger(__name__) # Agent run limits for director prompts / speech escalation (meeting context). -# Higher than default workspace agent: Teams research + tool chains need depth. TEAMSBOT_AGENT_MAX_ROUNDS = 8 TEAMSBOT_AGENT_MAX_COST_CHF = 0.12 -# How many recent director-prompt briefings (one-shot + persistent) we keep in -# session memory so SPEECH_TEAMS triggers and speech escalation can still see -# the operator's attached files + analysis after the prompt itself was consumed. +# How many recent director-prompt briefings we keep in session memory. _RECENT_DIRECTOR_BRIEFINGS_MAX = 6 -# Quick-ack ("Moment...") UX: fire a SHORT TTS the moment the bot's name is -# detected so the speaker hears within ~1s that the bot reacted, instead of -# waiting for the full debounce + SPEECH_TEAMS + agent pipeline (~5-30s). -# Throttled per session to avoid acking every fragment of a long utterance. +# Quick-ack throttle interval. _QUICK_ACK_MIN_INTERVAL_SEC = 25.0 -# Number of phrase variants we generate per kind (rotated round-robin so back- -# to-back acks/notices don't sound identical). +# Number of phrase variants we generate per kind. _EPHEMERAL_PHRASE_VARIANTS = 4 -# Localisation INTENTS for ephemeral phrases. Each kind describes WHAT the -# phrase should express; the actual wording is produced at runtime by the AI -# in the bot's configured language + persona. The intent text below is the -# instruction passed to the LLM (English, since it's a model directive — the -# OUTPUT will be in the configured spoken language). Add new ephemeral phrase -# kinds here, never inline string literals at the call site. +# Localisation INTENTS for ephemeral phrases. _EPHEMERAL_PHRASE_INTENTS: Dict[str, str] = { "quickAck": ( "Very short verbal acknowledgment (1 to 4 words) the assistant says " @@ -91,6 +79,10 @@ _EPHEMERAL_PHRASE_INTENTS: Dict[str, str] = { } +# ========================================================================= +# Module-level utility functions (used by extracted modules too) +# ========================================================================= + def _voiceLineLooksLikeBillingOrMeta(line: str) -> bool: """Heuristic: trailing lines that are separators or billing/usage footers.""" s = line.strip() @@ -119,45 +111,22 @@ def _voiceLineLooksLikeBillingOrMeta(line: str) -> bool: _EMOJI_PATTERN = re.compile( "[" - "\U0001F300-\U0001FAFF" # symbols & pictographs, emoticons, transport, supplemental - "\U00002600-\U000027BF" # misc symbols + dingbats (incl. ⚙ 🔐 🔌 ✓ ✗) - "\U0001F1E6-\U0001F1FF" # regional indicator (flags) - "\U00002B00-\U00002BFF" # arrows, geometric - "\U0001F900-\U0001F9FF" # supplemental symbols (incl. 🤖 🧠) - "\U0000FE0F" # variation selector-16 (emoji presentation) + "\U0001F300-\U0001FAFF" + "\U00002600-\U000027BF" + "\U0001F1E6-\U0001F1FF" + "\U00002B00-\U00002BFF" + "\U0001F900-\U0001F9FF" + "\U0000FE0F" "]+", flags=re.UNICODE, ) def _voiceFriendlyMeetingText(raw: str) -> str: - """Sanitise a chat/markdown response so it can be SPOKEN naturally. - - Aggressive cleanup — when a TTS engine reads raw markdown out loud the - listener hears "hash hash hash Zusammenfassung pipe pipe pipe", which - is unbearable in a meeting. The chat / DB / UI keep the original text; - only the audio path goes through this sanitiser. - - What we strip: - * Code fences and inline code - * Markdown emphasis (**bold**, *italic*, __bold__, _italic_) - * Markdown links → keep label - * Headings (# .. ######) - * Markdown tables (any line with two or more pipes is dropped wholesale) - * Horizontal rules (---, ***, ___ on their own line) - * Bullet markers (-, *, •, ·) and numbered list markers (1., 2)) at line start - * Emojis (full Unicode pictograph ranges + variation selector) - * Decorative trailing colons on bullet headings - * Stray pipes left over from inline tables - * Trailing billing / "maximum rounds reached" / "budget exceeded" footers - - Whitespace is then collapsed to single spaces. - """ + """Sanitise a chat/markdown response so it can be SPOKEN naturally.""" if not raw: return "" - # Trim trailing operator/billing footers BEFORE any structural rewrite - # so we don't waste effort sanitising a footer that gets dropped. low = raw.lower() if "maximum rounds reached" in low: m = re.search(r"(?is)maximum\s+rounds\s+reached", raw) @@ -179,13 +148,9 @@ def _voiceFriendlyMeetingText(raw: str) -> str: if not t: t = raw.strip() - # 1) Strip code blocks (multi-line first, then inline) t = re.sub(r"```[\s\S]*?```", " ", t) t = re.sub(r"`([^`]+)`", r"\1", t) - # 2) Drop markdown table rows (any line with two or more pipes) and the - # separator lines they come with (|---|---|). A paragraph that just - # happens to contain ONE pipe survives. cleanedLines: List[str] = [] for ln in t.split("\n"): stripped = ln.strip() @@ -196,84 +161,31 @@ def _voiceFriendlyMeetingText(raw: str) -> str: cleanedLines.append(ln) t = "\n".join(cleanedLines) - # 3) Drop horizontal rule lines (---, ***, ___, with optional spaces) t = re.sub(r"(?m)^\s*([-*_])\s*\1\s*\1[\s\1]*$", "", t) - - # 4) Headings: drop the leading hashes t = re.sub(r"(?m)^\s*#{1,6}\s+", "", t) - - # 5) Bullet markers at line start — keep the content, drop the bullet t = re.sub(r"(?m)^\s*[-*•·]\s+", "", t) - # 6) Numbered list markers at line start ("1.", "2)", "3 -") t = re.sub(r"(?m)^\s*\d+[\.\)]\s+", "", t) - - # 7) Emphasis markers (after bullets so a "**Bold:**" heading is handled) t = re.sub(r"\*\*([^*]+)\*\*", r"\1", t) t = re.sub(r"\*([^*\n]+)\*", r"\1", t) t = re.sub(r"__([^_]+)__", r"\1", t) t = re.sub(r"(?` `{` `}` `[` `]` `(` `)` - # `_` `&` `@` `$` `%` `` -- replaced with a space so word - # boundaries are preserved. t = re.sub(r"[*#~^=+|\\<>{}\[\]()_&@$%`]+", " ", t) - - # 10e) Drop ASCII double-quote (single quotes are legitimate apostrophes - # in contractions like "don't" / "geht's", so we keep U+0027). t = t.replace('"', "") - - # 10f) Slash between letters/digits — TTS reads "slash". Replace with - # " or " for readability when it separates words like "und/oder". t = re.sub(r"(?<=\w)\s*/\s*(?=\w)", " oder ", t) - # Any remaining stray slash is just whitespace. t = t.replace("/", " ") - - # 10g) Trim multiple punctuation runs ("...!!!" → "..." / "!" / etc.) t = re.sub(r"([\.,;:!\?])\1{1,}", r"\1", t) - # Remove orphan punctuation directly preceded by whitespace - # (common after symbol stripping: " , ", " . "). t = re.sub(r"\s+([\.,;:!\?])", r"\1", t) - # Collapse trailing colon at end of meaningful phrase to a period for - # nicer cadence ("Was ist PowerOn:" → "Was ist PowerOn."). t = re.sub(r":\s*$", ".", t.rstrip()) - # 10h) Collapse " :" tail of MULTI-LINE blocks the same way. t = re.sub(r"\s+:\s*$", ":", t, flags=re.MULTILINE) - # 11) Collapse whitespace to single spaces; protect sentence breaks by - # turning paragraph blanks into a period if the previous chunk - # didn't already terminate. paragraphs = [p.strip() for p in re.split(r"\n\s*\n", t) if p.strip()] rebuilt: List[str] = [] for p in paragraphs: @@ -285,28 +197,14 @@ def _voiceFriendlyMeetingText(raw: str) -> str: rebuilt.append(p) t = " ".join(rebuilt) t = re.sub(r"\s+", " ", t).strip() - - # If we sanitised away everything (e.g. the input was *only* a markdown - # table or a wall of pictographs) return empty — the caller (TTS / voice - # summary) treats empty as "nothing to say", which is the safe default. - # Falling back to raw markdown here would leak the very symbols we just - # spent ten passes removing. return t -# Google Cloud TTS rejects single sentences that exceed ~5000 bytes. The Chirp3 -# voices are stricter: long, comma-heavy sentences (no terminating punctuation) -# also fail with "Sentence ... is too long". We chunk well below the documented -# limit AND inject sentence terminators so the synthesizer accepts every chunk. _TTS_MAX_CHUNK_CHARS = 800 def _splitTextForTts(text: str, maxChars: int = _TTS_MAX_CHUNK_CHARS) -> List[str]: - """Split a long voice line into TTS-safe chunks at sentence/paragraph boundaries. - - The result preserves order and contains no empty strings. A single - sentence longer than ``maxChars`` is hard-cut at word boundaries. - """ + """Split a long voice line into TTS-safe chunks at sentence/paragraph boundaries.""" cleaned = (text or "").strip() if not cleaned: return [] @@ -369,19 +267,7 @@ async def _speakTextChunked( voiceName: Optional[str], isCancelled: Optional[Callable[[], bool]] = None, ) -> Dict[str, Any]: - """Run TTS in chunks and dispatch each ``playAudio`` over the websocket. - - Returns ``{"success": bool, "chunks": int, "played": int, "error": Optional[str], "cancelled": bool}``. - Failure for one chunk does NOT abort the rest; partial playback still - counts as ``success=True`` so the caller can decide whether to add a chat - fallback for the missing parts. - - ``isCancelled`` is an optional zero-arg predicate the caller passes in to - signal "abort the remaining chunks". It is checked BEFORE each Google - TTS round-trip and again BEFORE each websocket send, so a stop word in - the meeting can interrupt a multi-chunk dispatch within at most one - chunk boundary instead of waiting for the whole answer to finish. - """ + """Run TTS in chunks and dispatch each ``playAudio`` over the websocket.""" chunks = _splitTextForTts(voiceText) result: Dict[str, Any] = {"success": False, "chunks": len(chunks), "played": 0, "error": None, "cancelled": False} if not chunks: @@ -406,7 +292,7 @@ async def _speakTextChunked( languageCode=languageCode, voiceName=voiceName, ) - except Exception as ttsErr: # pragma: no cover - network/runtime errors + except Exception as ttsErr: lastError = f"chunk {idx}/{len(chunks)} raised: {ttsErr}" logger.warning(f"Session {sessionId}: TTS {lastError}") continue @@ -447,7 +333,7 @@ async def _speakTextChunked( }, })) result["played"] += 1 - except Exception as wsErr: # pragma: no cover - websocket failures + except Exception as wsErr: lastError = f"chunk {idx}/{len(chunks)} websocket send failed: {wsErr}" logger.warning(f"Session {sessionId}: TTS {lastError}") break @@ -459,8 +345,7 @@ async def _speakTextChunked( def _coercePersistedDetectedIntent(raw: Optional[str]) -> tuple: - """Map free-form intent labels (e.g. agent:directorPrompt) to TeamsbotDetectedIntent - for DB persistence; return (enum, meta_suffix_or_None for reasoning).""" + """Map free-form intent labels to TeamsbotDetectedIntent for DB persistence.""" if not raw or not str(raw).strip(): return TeamsbotDetectedIntent.NONE, None s = str(raw).strip().lower() @@ -472,11 +357,6 @@ def _coercePersistedDetectedIntent(raw: Optional[str]) -> tuple: return TeamsbotDetectedIntent.NONE, str(raw).strip()[:120] -# Director prompts are PRIVATE operator instructions — they must NOT be echoed -# verbatim into the meeting. The agent is asked to start its FINAL answer with -# either ``MEETING_REPLY:`` (followed by the text actually meant for the meeting) -# or ``SILENT:`` / ``INTERNAL_ONLY:`` (followed by an internal note for the -# operator UI). Anything else → treat as silent (safe default). _DIRECTOR_REPLY_PATTERN = re.compile( r"^\s*(MEETING_REPLY|MEETING|REPLY|SAY|SPEAK)\s*:\s*", re.IGNORECASE, @@ -488,12 +368,7 @@ _DIRECTOR_SILENT_PATTERN = re.compile( def _parseDirectorPromptFinal(finalText: str) -> Dict[str, Any]: - """Parse the agent's final answer for a director prompt. - - Returns ``{"kind": "meeting"|"silent", "meetingText": str, "internalNote": str}``. - - Default is ``silent`` so unmarked replies are NOT broadcast into the meeting. - """ + """Parse the agent's final answer for a director prompt.""" text = (finalText or "").strip() if not text: return {"kind": "silent", "meetingText": "", "internalNote": ""} @@ -508,18 +383,11 @@ def _parseDirectorPromptFinal(finalText: str) -> Dict[str, Any]: body = text[silentMatch.end():].strip() return {"kind": "silent", "meetingText": "", "internalNote": body} - # No marker → safe default: do NOT spam the meeting with the agent's - # internal reasoning. Keep the full text as an internal note for the - # operator UI so nothing is lost. return {"kind": "silent", "meetingText": "", "internalNote": text} # ========================================================================= # Active Service Registry (sessionId -> running TeamsbotService instance) -# -# Required so HTTP endpoints (e.g. director-prompt POST) can reach the -# TeamsbotService instance currently holding the live websocket + voice -# interface for that session, without going through the websocket loop. # ========================================================================= _activeServices: Dict[str, "TeamsbotService"] = {} @@ -530,7 +398,7 @@ def getActiveService(sessionId: str) -> Optional["TeamsbotService"]: # ========================================================================= -# AI Service Factory (for billing-aware AI calls) +# AI Service Factory # ========================================================================= def createAiService(user, mandateId, featureInstanceId=None): @@ -551,19 +419,14 @@ sessionEvents: Dict[str, asyncio.Queue] = {} async def _emitSessionEvent(sessionId: str, eventType: str, data: Any): - """Emit an event to the session's SSE stream. - Creates the queue on-demand so events are never silently dropped.""" + """Emit an event to the session's SSE stream.""" if sessionId not in sessionEvents: sessionEvents[sessionId] = asyncio.Queue() await sessionEvents[sessionId].put({"type": eventType, "data": data, "timestamp": getUtcTimestamp()}) def _normalizeGatewayHostForBotWs(host: str) -> str: - """Use IPv4 loopback for local dev WebSocket URLs passed to the Node browser-bot. - - Node on Windows often resolves ``localhost`` to ``::1`` first; Uvicorn bound to - ``0.0.0.0`` typically accepts IPv4 only, so the bot gets ``ECONNREFUSED ::1``. - """ + """Use IPv4 loopback for local dev WebSocket URLs passed to the Node browser-bot.""" h = host.strip() lower = h.lower() if lower == "localhost": @@ -577,6 +440,10 @@ def _normalizeGatewayHostForBotWs(host: str) -> str: return h +# ========================================================================= +# TeamsbotService Class +# ========================================================================= + class TeamsbotService: """ Pipeline Orchestrator for Teams Bot sessions. @@ -594,8 +461,8 @@ class TeamsbotService: self._lastAiCallTime: float = 0.0 self._aiAnalysisInProgress: bool = False self._contextBuffer: List[Dict[str, Any]] = [] - self._sessionContext: Optional[str] = None # User-provided background context - self._contextSummary: Optional[str] = None # AI-generated summary of long context + self._sessionContext: Optional[str] = None + self._contextSummary: Optional[str] = None # Differential transcript tracking self._lastTranscriptSpeaker: Optional[str] = None @@ -603,8 +470,7 @@ class TeamsbotService: self._lastTranscriptId: Optional[str] = None self._lastSttTime: float = 0.0 - # Audio chunk aggregation: collect chunks and send to STT only - # after a speech pause or when the buffer reaches a target duration. + # Audio chunk aggregation self._audioBuffer: bytes = b"" self._audioBufferStartTime: float = 0.0 self._audioBufferLastChunkTime: float = 0.0 @@ -612,82 +478,37 @@ class TeamsbotService: self._lastBotResponseText: Optional[str] = None self._lastBotResponseTs: float = 0.0 - # Speaker attribution: simple last-caption-speaker model + # Speaker attribution self._lastCaptionSpeaker: Optional[str] = None self._unattributedTranscriptIds: List[str] = [] self._knownSpeakers: set = set() - # Debounced name trigger: wait for speaker to finish before AI analysis + # Debounced name trigger self._pendingNameTrigger: Optional[Dict[str, Any]] = None self._followUpWindowEnd: float = 0.0 - # Quick-ack throttle (timestamp of the last short "Moment..." ack we - # spoke into the meeting). Without this guard a long sentence with - # multiple name mentions would trigger several acks in a row. + # Quick-ack throttle self._lastQuickAckTs: float = 0.0 - # Session-scoped phrase pool for SHORT ephemeral utterances (quick - # acks, "checking..." notices, per-round progress). Lazily populated - # by the AI in the bot's configured language + persona — no hardcoded - # strings or hardcoded language branching anywhere downstream. Keyed - # by the kinds defined in ``_EPHEMERAL_PHRASE_INTENTS``. - # * ``self._phrasePool[kind]`` -> list of variants for that kind - # * ``self._phrasePoolIdx[kind]`` -> round-robin pointer - # Concurrent generation calls for the same kind are serialised by the - # lock so we don't spawn duplicate AI requests on a burst. + # Ephemeral phrase pool self._phrasePool: Dict[str, List[str]] = {} self._phrasePoolIdx: Dict[str, int] = {} self._phrasePoolLock: asyncio.Lock = asyncio.Lock() - # Voice pipeline: a single per-session lock that serialises every TTS - # dispatch into the meeting. Without it three independent code paths - # (SPEECH_TEAMS direct answer, agent escalation final answer, and - # operator-driven director prompt) can all reach - # ``websocket.send_text({"type": "playAudio", ...})`` at the same time - # and the browser bot then plays interleaved chunks — i.e. "two bots - # talking over each other" exactly as the operator suspects. Chat - # (text) sends are NOT locked: they're cheap and can interleave fine. + # Voice pipeline serialisation self._meetingTtsLock: asyncio.Lock = asyncio.Lock() - # Generation counter incremented every time we begin producing a NEW - # meeting answer OR every time the user issues a hard stop. Any TTS - # chunk loop captures the counter value at start; before sending - # each chunk to the bot it re-checks the counter and bails out if - # it has moved on. This is what makes "Stopp" actually feel - # instantaneous: the in-flight TTS dispatch loop drops itself the - # moment the next chunk would have been sent, without waiting for - # any AI round-trip or extra Google TTS call to come back. self._answerGenerationCounter: int = 0 - # Tracking handles for cancellable background tasks. Keeping a - # reference lets ``_cancelInFlightSpeech`` actually call - # ``task.cancel()`` instead of just hoping the task notices the - # generation counter has moved on. Cleared in the task's own - # ``finally`` block. self._currentEscalationTask: Optional[asyncio.Task] = None self._currentQuickAckTask: Optional[asyncio.Task] = None - # Whether an agent escalation task is in flight. Kept separate from - # ``_aiAnalysisInProgress`` (which only covers the SPEECH_TEAMS phase) - # so a new speech trigger that arrives WHILE the agent is still - # researching does not start a parallel SPEECH_TEAMS that would then - # answer at the same time as the agent. self._agentEscalationInFlight: bool = False - # Live transport handles for out-of-band actions (director prompts, agent escalation). - # Set in handleBotWebSocket once the bot connects; cleared on disconnect. + # Live transport handles self._activeSessionId: Optional[str] = None self._websocket: Optional[WebSocket] = None self._voiceInterface = None - # Persistent director prompts kept in memory for context injection across triggers. - # Loaded from DB on (re)connect; mutated by submit/delete director prompt routes. + # Director prompts self._activePersistentPrompts: List[Dict[str, Any]] = [] - - # Recent director-prompt briefings (one-shot AND persistent) — keeps the - # operator's attached files and the agent's internal analysis available - # for later SPEECH_TEAMS triggers, even after a one-shot prompt has been - # consumed. Without this pool, the bot "forgets" attached docs as soon - # as the director prompt finished, and answers later meeting questions - # ("summarize the doc") with general babble instead of the file content. - # Capped by ``_RECENT_DIRECTOR_BRIEFINGS_MAX`` to bound prompt size. self._recentDirectorBriefings: List[Dict[str, Any]] = [] # ========================================================================= @@ -703,36 +524,22 @@ class TeamsbotService: botAccountEmail: Optional[str] = None, botAccountPassword: Optional[str] = None, ): - """Send join command to the Browser Bot service. - - The browser bot will: - 1. Launch browser (headful if credentials provided, headless otherwise) - 2. Navigate to Teams web app - 3. Authenticate if credentials provided, otherwise join as anonymous guest - 4. Enable captions/audio capture and start scraping - 5. Connect back via WebSocket to send transcripts - """ + """Send join command to the Browser Bot service.""" from . import interfaceFeatureTeamsbot as interfaceDb interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) - # Reuse existing SSE event queue if a consumer (SSE generator) already - # holds a reference; replacing it would strand the SSE stream. if sessionId not in sessionEvents: sessionEvents[sessionId] = asyncio.Queue() try: - # Update status to JOINING interface.updateSession(sessionId, {"status": TeamsbotSessionStatus.JOINING.value}) await _emitSessionEvent(sessionId, "statusChange", {"status": "joining"}) - # Send join command to browser bot session = interface.getSession(sessionId) if not session: raise ValueError(f"Session {sessionId} not found") - # Build the full WebSocket URL for the bot to connect back to this gateway instance - # gatewayBaseUrl is passed from the route handler (derived from request.base_url) wsScheme = "wss" if gatewayBaseUrl.startswith("https") else "ws" gatewayHost = gatewayBaseUrl.replace("https://", "").replace("http://", "").rstrip("/") gatewayHost = _normalizeGatewayHostForBotWs(gatewayHost) @@ -764,7 +571,7 @@ class TeamsbotService: if result.get("success"): interface.updateSession(sessionId, { - "status": TeamsbotSessionStatus.JOINING.value, # Will become ACTIVE when bot connects via WS + "status": TeamsbotSessionStatus.JOINING.value, }) logger.info(f"Browser bot deployment started for session {sessionId}") else: @@ -794,7 +601,7 @@ class TeamsbotService: return getattr(self.config, "avatarFileId", None) def _loadAvatarFileData(self, fileId, _teamsbotInterface): - """Load avatar file as base64 data + mime type. Returns (data, mimeType) or (None, None).""" + """Load avatar file as base64 data + mime type.""" from modules.interfaces import interfaceDbManagement try: mgmt = interfaceDbManagement.getInterface(self.currentUser, self.mandateId, featureInstanceId=self.instanceId) @@ -832,7 +639,6 @@ class TeamsbotService: }) await _emitSessionEvent(sessionId, "statusChange", {"status": "ended"}) - # Generate meeting summary in background asyncio.create_task(self._generateMeetingSummary(sessionId)) logger.info(f"Bot left meeting for session {sessionId}") @@ -845,504 +651,95 @@ class TeamsbotService: "endedAt": getUtcTimestamp(), }) - # Cleanup event queue sessionEvents.pop(sessionId, None) # ========================================================================= - # Browser Bot WebSocket Communication + # WebSocket — delegates to serviceWebSocket module # ========================================================================= async def handleBotWebSocket(self, websocket: WebSocket, sessionId: str): - """ - Main WebSocket handler for Browser Bot communication. - - Receives: - - transcript: Caption text scraped from Teams - - status: Bot state changes (joined, in_lobby, left, error) - - Sends: - - playAudio: TTS audio for the bot to play in the meeting - """ - from . import interfaceFeatureTeamsbot as interfaceDb - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + """Main WebSocket handler — delegates to serviceWebSocket.""" + from .serviceWebSocket import handleBotWebSocket as _handleBotWebSocket + await _handleBotWebSocket(self, websocket, sessionId) - interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) - voiceInterface = getVoiceInterface(self.currentUser, self.mandateId) + # ========================================================================= + # Conversation — delegates to serviceConversation module + # ========================================================================= - # Load session context (user-provided background knowledge) - # If the context is long (>500 chars), summarize it to reduce token usage - session = interface.getSession(sessionId) - if session: - rawContext = session.get("sessionContext") - if rawContext and len(rawContext) > 500: - logger.info(f"Session {sessionId}: Summarizing long session context ({len(rawContext)} chars)...") - self._sessionContext = await self._summarizeSessionContext(sessionId, rawContext) - elif rawContext: - self._sessionContext = rawContext - if self._sessionContext: - logger.info(f"Session {sessionId}: Session context ready ({len(self._sessionContext)} chars)") + async def _processTranscript(self, *args, **kwargs): + from .serviceConversation import _processTranscript + await _processTranscript(self, *args, **kwargs) - # Resolve system bot email for speaker detection (prevents bot from triggering AI on own speech) - try: - systemBot = interface.getActiveSystemBot(self.mandateId) - self._botAccountEmail = systemBot.get("email") if systemBot else None - if self._botAccountEmail: - logger.info(f"Session {sessionId}: Bot account email resolved: {self._botAccountEmail}") - except Exception: - self._botAccountEmail = None + async def _analyzeAndRespond(self, *args, **kwargs): + from .serviceConversation import _analyzeAndRespond + await _analyzeAndRespond(self, *args, **kwargs) - # Register the live service so out-of-band callers (director prompts, - # agent escalation) can deliver text/audio through this same websocket. - self._activeSessionId = sessionId - self._websocket = websocket - self._voiceInterface = voiceInterface - _activeServices[sessionId] = self + async def _summarizeForVoice(self, sessionId: str, rawAnswer: str) -> str: + from .serviceConversation import _summarizeForVoice + return await _summarizeForVoice(self, sessionId, rawAnswer) - # Notify the operator UI that the bot's WebSocket is now live so the - # director-prompt panel can enable its submit button. - try: - await _emitSessionEvent(sessionId, "botConnectionState", { - "connected": True, - "timestamp": getUtcTimestamp(), - }) - except Exception: - pass + async def _pickQuickAckText(self) -> Optional[str]: + from .serviceConversation import _pickQuickAckText + return await _pickQuickAckText(self) - # Restore active persistent director prompts from DB (survives reconnects). - try: - self._activePersistentPrompts = interface.getActivePersistentPrompts(sessionId) or [] - if self._activePersistentPrompts: - logger.info( - f"Session {sessionId}: Loaded {len(self._activePersistentPrompts)} active persistent director prompt(s)" - ) - except Exception as restoreErr: - logger.warning(f"Session {sessionId}: Could not restore persistent director prompts: {restoreErr}") - self._activePersistentPrompts = [] + async def _pickEphemeralPhrase(self, kind: str, substitutions=None) -> Optional[str]: + from .serviceConversation import _pickEphemeralPhrase + return await _pickEphemeralPhrase(self, kind, substitutions) - # Pre-warm the ephemeral phrase pool in the background so the first - # quick-ack ("Moment...") and interim agent notice don't have to wait - # for the AI round-trip. Best-effort: if generation fails, the - # corresponding ephemeral cue is silently skipped at runtime — never - # falls back to hardcoded language strings. - asyncio.create_task(self._warmEphemeralPhrasePool(sessionId)) + async def _getEphemeralPhrases(self, kind: str) -> List[str]: + from .serviceConversation import _getEphemeralPhrases + return await _getEphemeralPhrases(self, kind) - logger.info(f"[WS] Handler started for session {sessionId}") + async def _generateEphemeralPhrases(self, kind: str, count: int) -> List[str]: + from .serviceConversation import _generateEphemeralPhrases + return await _generateEphemeralPhrases(self, kind, count) - try: - msgCount = 0 - while True: - data = await websocket.receive() - msgCount += 1 + async def _runQuickAck(self, sessionId: str) -> None: + from .serviceConversation import _runQuickAck + await _runQuickAck(self, sessionId) - if "text" not in data: - logger.debug(f"[WS] session={sessionId} msg #{msgCount}: non-text data (keys: {list(data.keys())})") - continue + async def _checkPendingNameTrigger(self, delaySec: float = 3.0): + from .serviceConversation import _checkPendingNameTrigger + await _checkPendingNameTrigger(self, delaySec) - message = json.loads(data["text"]) - msgType = message.get("type") - if msgType not in ("audioChunk", "ping"): - logger.info(f"[WS] session={sessionId} msg #{msgCount}: type={msgType}") + async def _warmEphemeralPhrasePool(self, sessionId: str) -> None: + from .serviceConversation import _warmEphemeralPhrasePool + await _warmEphemeralPhrasePool(self, sessionId) - if msgType == "transcript": - transcript = message.get("transcript", {}) - source = transcript.get("source", "caption") - speaker = transcript.get("speaker", "Unknown") - textPreview = (transcript.get("text", "") or "")[:60] - # Caption/speakerHint: name resolution only; transcript comes from STT - logger.info(f"[WS] Transcript (source={source}, speaker={speaker}): {textPreview}...") - await self._processTranscript( - sessionId=sessionId, - speaker=transcript.get("speaker", "Unknown"), - text=transcript.get("text", ""), - isFinal=transcript.get("isFinal", True), - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - source=source, - ) + async def _runEscalationAndRelease(self, *args, **kwargs) -> None: + from .serviceConversation import _runEscalationAndRelease + await _runEscalationAndRelease(self, *args, **kwargs) - elif msgType == "chatMessage": - chat = message.get("chat", {}) - isHistory = chat.get("isHistory", False) - source = "chatHistory" if isHistory else "chat" - logger.info( - f"[WS] Chat{'[HISTORY]' if isHistory else ''}: " - f"speaker={chat.get('speaker')}, text={chat.get('text', '')[:60]}..." - ) - await self._processTranscript( - sessionId=sessionId, - speaker=chat.get("speaker", "Unknown"), - text=chat.get("text", ""), - isFinal=True, - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - source=source, - ) + # ========================================================================= + # Commands — delegates to serviceCommands module + # ========================================================================= - elif msgType == "status": - status = message.get("status") - errorMessage = message.get("message") - logger.info(f"[WS] Status: status={status}, message={errorMessage}") - await self._handleBotStatus(sessionId, status, errorMessage, interface) + async def _executeCommands(self, sessionId, commands, voiceInterface, websocket): + from .serviceCommands import _executeCommands + await _executeCommands(self, sessionId, commands, voiceInterface, websocket) - elif msgType == "audioChunk": - audioData = message.get("audio", {}) - audioBase64 = audioData.get("data", "") - sampleRate = audioData.get("sampleRate", 16000) - captureDiagnostics = audioData.get("captureDiagnostics") or {} - if audioBase64: - await self._processAudioChunk( - sessionId=sessionId, - audioBase64=audioBase64, - sampleRate=sampleRate, - captureDiagnostics=captureDiagnostics, - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - ) + # ========================================================================= + # WebSocket helpers (kept on class — used by serviceWebSocket) + # ========================================================================= - elif msgType == "voiceGreeting": - # Legacy path: older bot images send a pre-built greeting - # text. New bots use ``requestGreeting`` and let the - # Gateway own greeting generation. - greetingText = message.get("text", "") - greetingLang = message.get("language", self.config.language) - logger.info( - f"[WS] Voice greeting (legacy): text={greetingText[:60]}..., language={greetingLang}" - ) - if greetingText and voiceInterface: - await self._dispatchGreetingToMeeting( - sessionId=sessionId, - greetingText=greetingText, - greetingLang=greetingLang, - sendToChat=False, - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - ) + async def _handleBotStatus(self, sessionId, status, errorMessage, interface): + from .serviceWebSocket import _handleBotStatus + await _handleBotStatus(self, sessionId, status, errorMessage, interface) - elif msgType == "requestGreeting": - # New path: bot just signals "I have joined" — Gateway - # generates the greeting text via AI in the configured - # language + persona, then dispatches it to BOTH the - # meeting chat (sendChatMessage command) and TTS. No - # hardcoded language strings on the bot side. - requestedLang = ( - message.get("language") or self.config.language or "" - ).strip() or "en-US" - botNameHint = ( - message.get("botName") or self.config.botName or "" - ).strip() or self.config.botName - logger.info( - f"[WS] Greeting request from bot: language={requestedLang}, name={botNameHint}" - ) - if voiceInterface: - try: - greetingText = await self._generateGreetingText( - requestedLang - ) - except Exception as genErr: - logger.warning( - f"Greeting generation failed for session {sessionId}: {genErr}" - ) - greetingText = "" - if greetingText: - await self._dispatchGreetingToMeeting( - sessionId=sessionId, - greetingText=greetingText, - greetingLang=requestedLang, - sendToChat=True, - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - ) - else: - logger.warning( - f"Session {sessionId}: Skipping greeting — AI generation produced no text" - ) + async def _processAudioChunk(self, *args, **kwargs): + from .serviceWebSocket import _processAudioChunk + await _processAudioChunk(self, *args, **kwargs) - elif msgType == "ping": - await websocket.send_text(json.dumps({"type": "pong"})) + async def _cancelInFlightSpeech(self, sessionId, websocket, reason): + from .serviceWebSocket import _cancelInFlightSpeech + await _cancelInFlightSpeech(self, sessionId, websocket, reason) - elif msgType == "ttsPlaybackAck": - playback = message.get("playback", {}) or {} - status = playback.get("status", "unknown") - ackMessage = playback.get("message") or "Bot playback status update" - logger.info( - f"[WS] TTS playback ack: status={status}, format={playback.get('format')}, " - f"bytesBase64={playback.get('bytesBase64')}" - ) - await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { - "status": f"playback_{status}", - "hasWebSocket": True, - "message": ackMessage, - "timestamp": playback.get("timestamp") or getUtcTimestamp(), - "format": playback.get("format"), - "bytesBase64": playback.get("bytesBase64"), - }) - - elif msgType == "mfaChallenge": - mfaData = message.get("mfa", {}) - mfaType = mfaData.get("type", "unknown") - displayNumber = mfaData.get("displayNumber") - prompt = mfaData.get("prompt", "") - logger.info(f"[WS] MFA challenge: type={mfaType}, number={displayNumber}, prompt={prompt[:60]}") - - await _emitSessionEvent(sessionId, "mfaChallenge", { - "mfaType": mfaType, - "displayNumber": displayNumber, - "prompt": prompt, - "timestamp": getUtcTimestamp(), - }) - - from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks - mfaQueue = asyncio.Queue() - mfaCodeQueues[sessionId] = mfaQueue - - async def _waitAndForwardMfa(sid, queue, ws): - try: - mfaResponse = await asyncio.wait_for(queue.get(), timeout=120.0) - logger.info(f"[WS] MFA response received for session {sid}: action={mfaResponse.get('action')}") - await ws.send_text(json.dumps({ - "type": "mfaResponse", - "sessionId": sid, - "mfa": mfaResponse, - })) - except asyncio.TimeoutError: - logger.warning(f"[WS] MFA response timeout for session {sid}") - await ws.send_text(json.dumps({ - "type": "mfaResponse", - "sessionId": sid, - "mfa": {"action": "timeout"}, - })) - await _emitSessionEvent(sid, "mfaChallenge", { - "mfaType": "timeout", - "prompt": "MFA-Zeitlimit ueberschritten. Bitte erneut versuchen.", - }) - except asyncio.CancelledError: - logger.info(f"[WS] MFA wait cancelled for session {sid} (resolved via page)") - finally: - mfaCodeQueues.pop(sid, None) - mfaWaitTasks.pop(sid, None) - - mfaWaitTasks[sessionId] = asyncio.create_task( - _waitAndForwardMfa(sessionId, mfaQueue, websocket) - ) - - elif msgType == "chatSendFailed": - errorData = message.get("error", {}) - reason = errorData.get("reason", "unknown") - failedText = errorData.get("text", "") - logger.warning( - f"[WS] Chat send failed for session {sessionId}: " - f"reason={reason}, text={failedText[:60]}" - ) - await _emitSessionEvent(sessionId, "chatSendFailed", { - "reason": reason, - "message": errorData.get("message", "Chat message could not be sent"), - "text": failedText, - "timestamp": getUtcTimestamp(), - }) - - elif msgType == "mfaResolved": - success = message.get("success", False) - logger.info(f"[WS] MFA resolved: success={success}") - from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks - task = mfaWaitTasks.pop(sessionId, None) - if task and not task.done(): - task.cancel() - mfaCodeQueues.pop(sessionId, None) - await _emitSessionEvent(sessionId, "mfaResolved", { - "success": success, - "timestamp": getUtcTimestamp(), - }) - - except Exception as e: - if "disconnect" not in str(e).lower(): - logger.error(f"[WS] Error for session {sessionId}: {type(e).__name__}: {e}") - finally: - if _activeServices.get(sessionId) is self: - _activeServices.pop(sessionId, None) - self._websocket = None - self._voiceInterface = None - self._activeSessionId = None - try: - await _emitSessionEvent(sessionId, "botConnectionState", { - "connected": False, - "timestamp": getUtcTimestamp(), - }) - except Exception: - pass - - logger.info(f"[WS] Handler ended for session {sessionId} after {msgCount} messages") - - async def _handleBotStatus( - self, - sessionId: str, - status: str, - errorMessage: Optional[str], - interface, - ): - """Handle status updates from the browser bot.""" - logger.info(f"Bot status update for session {sessionId}: {status}") - - statusMap = { - "connecting": TeamsbotSessionStatus.JOINING.value, - "launching": TeamsbotSessionStatus.JOINING.value, - "navigating": TeamsbotSessionStatus.JOINING.value, - "in_lobby": TeamsbotSessionStatus.JOINING.value, - "joined": TeamsbotSessionStatus.ACTIVE.value, - "in_meeting": TeamsbotSessionStatus.ACTIVE.value, - "left": TeamsbotSessionStatus.ENDED.value, - "error": TeamsbotSessionStatus.ERROR.value, - } - - dbStatus = statusMap.get(status, TeamsbotSessionStatus.ACTIVE.value) - - updates = {"status": dbStatus} - if errorMessage: - updates["errorMessage"] = errorMessage - if dbStatus == TeamsbotSessionStatus.ACTIVE.value: - updates["startedAt"] = getUtcTimestamp() - elif dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]: - updates["endedAt"] = getUtcTimestamp() - - interface.updateSession(sessionId, updates) - await _emitSessionEvent(sessionId, "statusChange", {"status": status, "errorMessage": errorMessage}) - - # Flush remaining audio buffer before generating summary - if dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]: - if self._audioBuffer: - logger.info(f"[AudioChunk] Flushing remaining buffer on session end ({len(self._audioBuffer)} bytes)") - self._audioBuffer = b"" - self._audioBufferStartTime = 0.0 - self._audioBufferLastChunkTime = 0.0 - - # Generate summary when session ends - if dbStatus == TeamsbotSessionStatus.ENDED.value: - asyncio.create_task(self._generateMeetingSummary(sessionId)) - - async def _processAudioChunk( - self, - sessionId: str, - audioBase64: str, - sampleRate: int, - captureDiagnostics: Optional[Dict[str, Any]], - interface, - voiceInterface, - websocket: WebSocket, - ): - """Process an audio chunk from WebRTC capture. The bot-side VAD - (AudioWorklet / ScriptProcessor) already segments speech into 1-8s - voiced chunks. Here we apply a minimum-duration safety net: very short - chunks (<1s) are buffered until they reach 1s; everything else goes - straight to STT. A wall-clock timeout flushes stale buffers.""" - _MIN_CHUNK_SEC = 1.0 - _STALE_TIMEOUT_SEC = 3.0 - - try: - audioBytes = base64.b64decode(audioBase64) - if len(audioBytes) < 500: - return - - if captureDiagnostics: - trackId = captureDiagnostics.get("trackId") - readyState = captureDiagnostics.get("readyState") - rms = captureDiagnostics.get("rms") - nativeSampleRate = captureDiagnostics.get("nativeSampleRate") - logger.debug( - f"[AudioChunk] diagnostics: track={trackId}, readyState={readyState}, " - f"rms={rms}, nativeRate={nativeSampleRate}, bytes={len(audioBytes)}" - ) - - isSilent = False - if captureDiagnostics and captureDiagnostics.get("rms") is not None: - try: - rmsVal = float(captureDiagnostics.get("rms")) - if rmsVal < 0.0003: - isSilent = True - except Exception: - pass - - if not voiceInterface: - logger.warning(f"[AudioChunk] No voice interface available for session {sessionId}") - return - - now = time.time() - effectiveRate = sampleRate if sampleRate and sampleRate > 0 else 16000 - - if not isSilent: - if not self._audioBuffer: - self._audioBufferStartTime = now - self._audioBuffer += audioBytes - self._audioBufferLastChunkTime = now - self._audioBufferSampleRate = effectiveRate - - bufferDuration = len(self._audioBuffer) / (effectiveRate * 2) if self._audioBuffer else 0.0 - bufferAge = (now - self._audioBufferStartTime) if self._audioBuffer else 0.0 - - shouldFlush = ( - self._audioBuffer - and ( - bufferDuration >= _MIN_CHUNK_SEC - or (bufferAge >= _STALE_TIMEOUT_SEC and bufferDuration > 0.3) - ) - ) - - if not shouldFlush: - return - - flushBytes = self._audioBuffer - flushRate = self._audioBufferSampleRate - self._audioBuffer = b"" - self._audioBufferStartTime = 0.0 - self._audioBufferLastChunkTime = 0.0 - - flushDuration = len(flushBytes) / (flushRate * 2) - logger.info(f"[AudioChunk] Flushing buffer: {len(flushBytes)} bytes, {flushDuration:.1f}s, {flushRate}Hz") - - phraseHints = list(self._knownSpeakers) - if self.config.botName: - phraseHints.append(self.config.botName) - - sttResult = await voiceInterface.speechToText( - audioContent=flushBytes, - language=self.config.language or "de-DE", - sampleRate=flushRate, - channels=1, - skipFallbacks=True, - phraseHints=phraseHints if phraseHints else None, - audioFormat="linear16", - ) - - if sttResult and sttResult.get("success") and sttResult.get("text"): - text = sttResult["text"].strip() - if text: - resolvedSpeaker = self._resolveSpeakerForAudioCapture() - fromCaption = resolvedSpeaker.get("speakerResolvedFromHint", False) - logger.info( - f"[AudioChunk] STT result: speaker={resolvedSpeaker.get('speaker', 'Meeting Audio')} " - f"(fromCaption={fromCaption}), text={text[:80]}..." - ) - await self._processTranscript( - sessionId=sessionId, - speaker=resolvedSpeaker["speaker"], - text=text, - isFinal=True, - interface=interface, - voiceInterface=voiceInterface, - websocket=websocket, - source="audioCapture", - speakerResolvedFromHint=resolvedSpeaker["speakerResolvedFromHint"], - ) - except Exception as e: - logger.error(f"[AudioChunk] STT error for session {sessionId}: {type(e).__name__}: {e}") + # ========================================================================= + # Speaker & Trigger detection (kept on class — small, stateful) + # ========================================================================= def _registerSpeakerHint(self, speaker: str, text: str, sessionId: str = ""): - """Track current speaker from captions for STT attribution. - Retroactively attributes any unattributed STT segments whenever a - new non-bot caption speaker arrives (not just the first time).""" + """Track current speaker from captions for STT attribution.""" if not speaker: return normalizedSpeaker = speaker.strip() @@ -1377,313 +774,42 @@ class TeamsbotService: return {"speaker": self._lastCaptionSpeaker, "speakerResolvedFromHint": True} return {"speaker": "Unknown", "speakerResolvedFromHint": False} - async def _processTranscript( - self, - sessionId: str, - speaker: str, - text: str, - isFinal: bool, - interface, - voiceInterface, - websocket: WebSocket, - source: str = "caption", - speakerResolvedFromHint: Optional[bool] = None, - ): - """Process a transcript segment from captions or chat messages. - - Differential writing: When the same speaker continues (text grows - incrementally as captions stream), we UPDATE the existing DB record - instead of creating a cascade of near-duplicate rows. A new record - is only created when the speaker changes or the text is not a - continuation of the previous segment. - """ - - text = text.strip() - if not text: - return - - # Captions are used ONLY for speaker name resolution (never as transcript). - # Transcript text comes exclusively from audio STT or chat. - # Address detection (bot name in caption) still triggers AI analysis - # using existing audio-based context — but caption text itself is NOT - # added to the context buffer. - if source in ("caption", "speakerHint"): - self._registerSpeakerHint(speaker, text, sessionId) - - if ( - source == "speakerHint" - and isFinal - and not self._isBotSpeaker(speaker) - and self.config.responseMode != TeamsbotResponseMode.TRANSCRIBE_ONLY - and self._detectBotName(text) - ): - triggerTranscript = {"id": None, "speaker": speaker, "text": text, "source": source} - isNew = self._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, triggerTranscript) - if isNew: - logger.info(f"Session {sessionId}: Bot name in caption, debounce trigger started") - asyncio.create_task(self._checkPendingNameTrigger()) - # Fire a short audible "Moment..." in parallel so the - # speaker hears the bot react immediately, instead of - # waiting for debounce + SPEECH_TEAMS + agent (~5-30s). - self._currentQuickAckTask = asyncio.create_task( - self._runQuickAck(sessionId) - ) - return - - # Chat history: messages sent before the bot joined the meeting. - # Stored in DB for reference but NOT added to the AI context buffer, - # because old messages (e.g. "nyla, summarize the protocol") would - # be treated as current requests when AI analysis is triggered. - if source == "chatHistory": - transcriptData = TeamsbotTranscript( - sessionId=sessionId, - speaker=speaker, - text=text, - timestamp=getUtcTimestamp(), - confidence=1.0, - language=self.config.language, - isFinal=True, - source="chatHistory", - ).model_dump() - createdTranscript = interface.createTranscript(transcriptData) - - await _emitSessionEvent(sessionId, "transcript", { - "id": createdTranscript.get("id"), - "speaker": speaker, - "text": text, - "confidence": 1.0, - "timestamp": getUtcTimestamp(), - "isContinuation": False, - "source": "chatHistory", - "isHistory": True, - }) - logger.debug(f"Session {sessionId}: Chat history stored (no AI trigger): [{speaker}] {text[:60]}") - return - - # Filter out the bot's own speech (caption/audioCapture) — garbled text - # pollutes context. Chat from the bot is clean text and must appear in - # the transcript for all participants. - isBotSpeaker = self._isBotSpeaker(speaker) - if isBotSpeaker and source != "chat": - logger.debug(f"Session {sessionId}: Ignoring own bot caption from: [{speaker}] {text[:80]}...") - return - - # Differential transcript writing: - # audioCapture from same speaker → append text (merge STT chunks into one block) - # Start a new block after a pause (>5s gap between STT results) - sttPauseThreshold = 5.0 - isMerge = ( - source == "audioCapture" - and self._lastTranscriptSpeaker == speaker - and self._lastTranscriptText is not None - and self._lastTranscriptId is not None - and (time.time() - self._lastSttTime) < sttPauseThreshold - ) - - if isMerge: - mergedText = f"{self._lastTranscriptText} {text}" - interface.updateTranscript(self._lastTranscriptId, { - "text": mergedText, - "isFinal": isFinal, - }) - self._lastTranscriptText = mergedText - createdTranscript = {"id": self._lastTranscriptId} - - if self._contextBuffer and self._contextBuffer[-1].get("speaker") == speaker: - self._contextBuffer[-1]["text"] = mergedText - else: - transcriptData = TeamsbotTranscript( - sessionId=sessionId, - speaker=speaker, - text=text, - timestamp=getUtcTimestamp(), - confidence=1.0, - language=self.config.language, - isFinal=isFinal, - source=source, - ).model_dump() - - createdTranscript = interface.createTranscript(transcriptData) - - self._lastTranscriptSpeaker = speaker - self._lastTranscriptText = text - self._lastTranscriptId = createdTranscript.get("id") - - if source == "audioCapture" and speaker == "Unknown": - self._unattributedTranscriptIds.append(createdTranscript.get("id")) - - self._contextBuffer.append({ - "speaker": speaker or "Unknown", - "text": text, - "timestamp": getUtcTimestamp(), - "source": source, - }) - - maxSegments = self.config.contextWindowSegments - if len(self._contextBuffer) > maxSegments: - if not self._contextSummary and len(self._contextBuffer) > maxSegments * 1.5: - asyncio.create_task(self._summarizeContextBuffer(sessionId)) - self._contextBuffer = self._contextBuffer[-maxSegments:] - - session = interface.getSession(sessionId) - if session: - count = session.get("transcriptSegmentCount", 0) + 1 - interface.updateSession(sessionId, {"transcriptSegmentCount": count}) - - if source == "audioCapture": - self._lastSttTime = time.time() - - displayText = self._lastTranscriptText if isMerge else text - await _emitSessionEvent(sessionId, "transcript", { - "id": createdTranscript.get("id"), - "speaker": speaker, - "text": displayText, - "confidence": 1.0, - "timestamp": getUtcTimestamp(), - "isContinuation": isMerge, - "source": source, - "speakerResolvedFromHint": ( - speakerResolvedFromHint - if speakerResolvedFromHint is not None - else False - ), - }) - - if not isFinal: - return - - if self.config.responseMode == TeamsbotResponseMode.TRANSCRIBE_ONLY: - return - - # Bot's own chat: stored for display only, never trigger AI - if source == "chat" and isBotSpeaker: - return - - # Stop phrases: HARD STOP, no AI round-trip. We previously routed - # this through ``_analyzeAndRespond`` which spent 1-2 seconds in - # the speech LLM just to classify the intent, during which the - # current TTS kept playing — and the LLM round-trip would also - # produce yet another response that joined the queue. The new - # path goes straight to the browser bot's audio cancel and - # invalidates everything else in flight. - if self._isStopPhrase(text): - logger.info( - f"Session {sessionId}: Stop phrase detected ('{text.strip()[:60]}'), " - f"hard-cancelling in-flight speech immediately" - ) - await self._cancelInFlightSpeech( - sessionId=sessionId, - websocket=websocket, - reason="userStopPhrase", - ) - return - - # Update activity for any pending debounced trigger - if self._pendingNameTrigger: - self._pendingNameTrigger["lastActivity"] = time.time() - - # Bot name detection → debounced trigger (wait for speaker to finish) - if self._detectBotName(text): - isNew = self._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript) - if isNew: - asyncio.create_task(self._checkPendingNameTrigger()) - # Audible early-feedback ack ("Moment...") in parallel — runs - # while we still wait the debounce window and SPEECH_TEAMS - # decides what to actually answer. - self._currentQuickAckTask = asyncio.create_task( - self._runQuickAck(sessionId) - ) - return - - # Follow-up window: after a bot response, trigger AI for any human speech - # without requiring the bot name — the AI decides via shouldRespond - if ( - source == "audioCapture" - and not self._isBotSpeaker(speaker) - and time.time() < self._followUpWindowEnd - and not self._pendingNameTrigger - ): - isNew = self._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript) - if isNew: - logger.info(f"Session {sessionId}: Follow-up window trigger (no name needed)") - asyncio.create_task(self._checkPendingNameTrigger()) - return - - # Periodic trigger (only when no debounce pending) - if not self._pendingNameTrigger: - shouldTrigger = self._shouldTriggerAnalysis(text) - if shouldTrigger: - logger.info(f"Session {sessionId}: Periodic trigger (buffer: {len(self._contextBuffer)} segments)") - await self._analyzeAndRespond(sessionId, interface, voiceInterface, websocket, createdTranscript) - def _isBotSpeaker(self, speaker: str) -> bool: - """Check if a transcript speaker is the bot itself. - - Teams captions show the bot as e.g. "BotName (Unverified)" or - "Nyla Larsson" depending on auth/anonymous join. We match against: - - The configured/derived bot name - - The bot account display name if authenticated - """ + """Check if a transcript speaker is the bot itself.""" if not speaker: return False - speakerLower = speaker.lower().strip() - - # Match against configured bot name botName = self.config.botName.lower().strip() if botName and botName in speakerLower: return True - - # Match against bot account email prefix (e.g. "nyla.larsson" from "nyla.larsson@poweron.swiss") botAccountEmail = getattr(self, '_botAccountEmail', None) or getattr(self.config, 'botAccountEmail', None) if botAccountEmail: emailPrefix = botAccountEmail.split("@")[0].lower().replace(".", " ") if emailPrefix in speakerLower: return True - return False def _shouldTriggerAnalysis(self, transcriptText: str, allowPeriodic: bool = True) -> bool: - """ - Decide whether to trigger AI analysis based on the latest transcript. - Bot name detection is handled separately via debounce. - This method only checks periodic/cooldown triggers. - """ + """Decide whether to trigger AI analysis based on the latest transcript.""" now = time.time() timeSinceLastCall = now - self._lastAiCallTime - if timeSinceLastCall < self.config.triggerCooldownSeconds: return False - if allowPeriodic and timeSinceLastCall >= self.config.triggerIntervalSeconds: logger.info(f"Trigger: Periodic interval ({self.config.triggerIntervalSeconds}s) elapsed ({timeSinceLastCall:.1f}s)") return True - return False def _isStopPhrase(self, text: str) -> bool: - """Check if text is an immediate-cancel command from the meeting. - - Recognised intents (any language we hear in practice): - * Hard stop: stop / stopp / halt / ruhe / stille / arrete / quiet / shut - * Pause / wait: warte / wait / moment / pause / hold (hold on) - * Silence: sei still / be quiet / shut up / aufhoeren / aufhören / silence - Hits trigger the direct stop pipeline in ``_cancelInFlightSpeech``: - kill TTS, invalidate pending generations, clear name-trigger debounce. - Critically: NO new AI call is fired — the user explicitly asked the - bot to be quiet, so the worst thing we could do is generate yet - another response on top of the one we just cancelled. - """ + """Check if text is an immediate-cancel command from the meeting.""" if not text or len(text.strip()) < 2: return False t = text.strip().lower() words = [w.strip(".,!?:;\"'()[]") for w in t.split() if w.strip()] wordSet = set(words) stopWords = { - # Hard-stop verbs "stop", "stopp", "halt", "ruhe", "stille", "schweig", "arrete", "quiet", "shut", "silence", - # Pause / wait verbs (still "be quiet now" semantics) "warte", "wait", "moment", "pause", } if wordSet & stopWords: @@ -1700,98 +826,16 @@ class TeamsbotService: return False def _makeAnswerCancelHook(self) -> Callable[[], bool]: - """Capture the current ``_answerGenerationCounter`` and return a - zero-arg predicate that returns ``True`` once a hard stop (or any - future "supersede this answer" event) has bumped the counter. - - Pass the returned predicate as ``isCancelled`` into - ``_speakTextChunked`` so a multi-chunk dispatch can bail out - between chunks instead of speaking a 30-second answer to the end. - """ + """Capture the current generation counter and return a cancel predicate.""" snapshot = self._answerGenerationCounter return lambda: self._answerGenerationCounter != snapshot - async def _cancelInFlightSpeech( - self, - sessionId: str, - websocket: Optional[WebSocket], - reason: str, - ) -> None: - """Hard stop everything the bot is currently doing in the meeting. - - Pipeline (ALL synchronous from the caller's point of view, no AI - round-trips): - - 1. Bump ``_answerGenerationCounter`` so any in-flight TTS chunk - loop, agent escalation or quick-ack drops its remaining work - the moment it next checks the counter. - 2. Clear ``_pendingNameTrigger`` so a debounced "speaker just said - the bot name" trigger that was queued before the stop word - cannot wake up 3 seconds later and answer anyway. - 3. Cancel tracked background tasks (escalation, quick-ack). The - tasks themselves swallow ``CancelledError`` in their finally - block. - 4. Send ``{"type":"stopAudio"}`` to the browser bot — it stops the - current playback in the AudioContext and clears its play queue - so nothing buffered comes through afterwards. - - Deliberately does NOT generate a new response. The user just told - the bot to be quiet; producing a "Okay, ich bin still" reply on - top would be the exact opposite of what was asked for. - """ - self._answerGenerationCounter += 1 - gen = self._answerGenerationCounter - logger.info( - f"Session {sessionId}: Cancelling in-flight speech " - f"(reason={reason}, gen={gen})" - ) - - if self._pendingNameTrigger: - logger.info( - f"Session {sessionId}: Dropping pending debounced name " - f"trigger (was queued before stop)" - ) - self._pendingNameTrigger = None - - for taskAttr in ("_currentEscalationTask", "_currentQuickAckTask"): - task = getattr(self, taskAttr, None) - if task is not None and not task.done(): - logger.info( - f"Session {sessionId}: Cancelling background task " - f"{taskAttr}" - ) - task.cancel() - - if websocket is not None: - try: - await websocket.send_text(json.dumps({ - "type": "stopAudio", - "sessionId": sessionId, - "reason": reason, - })) - except Exception as stopErr: - logger.warning( - f"Session {sessionId}: Failed to send stopAudio to " - f"browser bot: {stopErr}" - ) - - try: - await _emitSessionEvent(sessionId, "speechCancelled", { - "reason": reason, - "generation": gen, - "timestamp": getUtcTimestamp(), - }) - except Exception: - pass - def _detectBotName(self, text: str) -> bool: """Check if text contains the bot's name (exact or phonetically similar).""" botNameLower = self.config.botName.lower() textLower = text.lower() - if botNameLower in textLower: return True - botFirstName = botNameLower.split()[0] if " " in botNameLower else botNameLower if len(botFirstName) >= 3: for word in textLower.split(): @@ -1823,314 +867,13 @@ class TeamsbotService: } return True - async def _warmEphemeralPhrasePool(self, sessionId: str) -> None: - """Fire-and-forget background task: generate the ephemeral phrase - pool for every kind defined in ``_EPHEMERAL_PHRASE_INTENTS`` so the - first quick-ack / interim notice doesn't pay the AI round-trip - latency at runtime. Failures are logged but never raised — the - runtime selectors handle empty pools by silently skipping the cue.""" - try: - for kind in _EPHEMERAL_PHRASE_INTENTS: - try: - await self._getEphemeralPhrases(kind) - except Exception as innerErr: - logger.warning( - f"Session {sessionId}: Phrase pool warmup failed for " - f"kind={kind}: {innerErr}" - ) - except Exception as warmErr: - logger.warning( - f"Session {sessionId}: Phrase pool warmup task crashed: {warmErr}" - ) - - # ---------------------------------------------------------------- Voice - # When the bot's full answer is a long structured chat post (markdown - # tables, bullet lists, headings, multi-paragraph) we MUST NOT read it - # out verbatim into the meeting — even after sanitisation it sounds - # like a wall of text and easily takes 5+ minutes. The chat keeps the - # full answer; the audio path goes through ``_summarizeForVoice`` which - # asks the AI for a 1-3 sentence spoken paraphrase in the configured - # bot persona / language. - - # Threshold: anything longer than this many characters (after sanitise) - # OR any answer whose source contains markdown structure (tables / - # multiple bullets / multiple headings) gets condensed before TTS. - _VOICE_DIRECT_MAX_CHARS = 600 - _VOICE_SUMMARY_MAX_CHARS = 350 - - @staticmethod - def _looksLikeStructuredText(raw: str) -> bool: - """Heuristic: does the original answer have markdown structure that - would be miserable to listen to verbatim? Used to trigger the - AI summary path even when the sanitised text is short enough.""" - if not raw: - return False - if raw.count("|") >= 4: # at least one markdown table row - return True - if raw.count("\n#") >= 1: # at least one heading after newline - return True - if raw.count("\n- ") + raw.count("\n* ") + raw.count("\n• ") >= 3: - return True # 3+ bullets → list-like - if re.search(r"\n\d+[\.\)]\s", raw): # numbered list - count = len(re.findall(r"(?m)^\s*\d+[\.\)]\s", raw)) - if count >= 3: - return True - return False - - async def _summarizeForVoice( - self, - sessionId: str, - rawAnswer: str, - ) -> str: - """Return a SHORT, naturally-spoken paraphrase of ``rawAnswer`` for - TTS playback. Falls back to the sanitised + truncated original if - the AI call fails — never blocks the response. - - The chat / DB / UI keep the original ``rawAnswer`` untouched. Only - the voice channel goes through this condensation. - """ - if not rawAnswer or not rawAnswer.strip(): - return "" - - sanitised = _voiceFriendlyMeetingText(rawAnswer) - # Short + unstructured → speak as-is, no AI round-trip - if ( - len(sanitised) <= self._VOICE_DIRECT_MAX_CHARS - and not self._looksLikeStructuredText(rawAnswer) - ): - return sanitised - - targetLang = (self.config.language or "de-DE").strip() - botName = (self.config.botName or "").strip() or "the assistant" - persona = (self.config.aiSystemPrompt or "").strip() - personaBlock = ( - f"\n\nBOT PERSONA / TONE:\n{persona}\n" - if persona else "" - ) - - prompt = ( - f"You are condensing a long written answer into a SHORT spoken " - f"paraphrase that the assistant '{botName}' will say out loud " - f"into a Microsoft Teams meeting. The full written answer is " - f"already in the meeting chat — your job is to summarise it for " - f"the EAR, not the eye.\n\n" - f"STRICT REQUIREMENTS:\n" - f"1. Output language: BCP-47 '{targetLang}'. No other language.\n" - f"2. 1 to 3 sentences, max ~{self._VOICE_SUMMARY_MAX_CHARS} characters total.\n" - f"3. Natural spoken style — no headings, no bullet points, no " - f"tables, no markdown, no emojis, no enumerations like 'Erstens... " - f"Zweitens...' unless that genuinely flows in speech.\n" - f"4. Capture the essence and the most important conclusion. Do " - f"NOT try to fit every detail. Listeners can read the chat for " - f"the full version.\n" - f"5. End by gently pointing the audience to the chat for details, " - f"e.g. 'Details stehen im Chat.' (adapted to the target language).\n" - f"6. Output ONLY the spoken text. No JSON, no quotes around it, " - f"no preamble like 'Here is the summary:'.\n" - f"{personaBlock}\n" - f"FULL WRITTEN ANSWER (markdown-formatted, sometimes long):\n" - f"---\n{rawAnswer.strip()[:6000]}\n---\n" - ) - - try: - aiService = createAiService( - self.currentUser, self.mandateId, self.instanceId - ) - await aiService.ensureAiObjectsInitialized() - request = AiCallRequest( - prompt=prompt, - context="", - options=AiCallOptions( - operationType=OperationTypeEnum.DATA_ANALYSE, - priority=PriorityEnum.SPEED, - ), - ) - response = await aiService.callAi(request) - except Exception as aiErr: - logger.warning( - f"Session {sessionId}: Voice summary AI call failed: {aiErr}" - ) - return sanitised[: self._VOICE_DIRECT_MAX_CHARS] - - if not response or response.errorCount != 0 or not response.content: - logger.warning( - f"Session {sessionId}: Voice summary returned empty/error" - ) - return sanitised[: self._VOICE_DIRECT_MAX_CHARS] - - spoken = response.content.strip() - # Defensive sanitiser pass — the model usually obeys the - # "no markdown" instruction but not always. - spoken = _voiceFriendlyMeetingText(spoken) - if not spoken: - return sanitised[: self._VOICE_DIRECT_MAX_CHARS] - - logger.info( - f"Session {sessionId}: Voice summary generated " - f"(orig={len(rawAnswer)} chars, sanitised={len(sanitised)}, " - f"spoken={len(spoken)})" - ) - return spoken - - async def _pickQuickAckText(self) -> Optional[str]: - """Return a short ack text in the bot's configured language. The - actual phrases are AI-generated once per session (cached) and rotated - round-robin so consecutive acks don't sound identical. Returns - ``None`` only if AI generation completely failed and no fallback - variant could be produced — in that case the caller silently skips - the ack.""" - return await self._pickEphemeralPhrase("quickAck") - - async def _pickEphemeralPhrase( - self, - kind: str, - substitutions: Optional[Dict[str, Any]] = None, - ) -> Optional[str]: - """Round-robin selector over the cached phrase pool for ``kind``. - Lazily generates the pool on first use. ``substitutions`` is applied - to the chosen phrase via ``str.format(**substitutions)`` so kinds - like ``agentRound`` can render ``{round}`` / ``{maxRounds}``. - Returns ``None`` if no phrases are available.""" - variants = await self._getEphemeralPhrases(kind) - if not variants: - return None - idx = self._phrasePoolIdx.get(kind, 0) % len(variants) - self._phrasePoolIdx[kind] = (idx + 1) % len(variants) - chosen = variants[idx] - if substitutions: - try: - chosen = chosen.format(**substitutions) - except (KeyError, IndexError, ValueError) as fmtErr: - # The AI didn't include the expected placeholder — return the - # raw phrase rather than crash. The user still hears something - # in the right language; only the numeric hint is missing. - logger.debug( - f"Ephemeral phrase substitution failed for kind={kind}: {fmtErr}" - ) - return chosen - - async def _getEphemeralPhrases(self, kind: str) -> List[str]: - """Return the cached pool of AI-generated variants for ``kind``, - generating it on first request. Subsequent calls hit the in-memory - cache. Concurrent first-time callers are serialised by the pool lock - so only ONE AI request is fired per kind per session.""" - cached = self._phrasePool.get(kind) - if cached: - return cached - async with self._phrasePoolLock: - cached = self._phrasePool.get(kind) - if cached: - return cached - phrases = await self._generateEphemeralPhrases( - kind, _EPHEMERAL_PHRASE_VARIANTS - ) - if phrases: - self._phrasePool[kind] = phrases - return phrases - - async def _generateEphemeralPhrases( - self, kind: str, count: int - ) -> List[str]: - """Ask the AI to produce ``count`` short utterances for ``kind`` in - the bot's configured language and persona. Returns ``[]`` on any - failure — callers must treat empty as 'silently skip this ephemeral - cue', NEVER fall back to a hardcoded localized string.""" - intent = _EPHEMERAL_PHRASE_INTENTS.get(kind) - if not intent: - logger.warning(f"Unknown ephemeral phrase kind requested: {kind}") - return [] - - targetLang = (self.config.language or "").strip() or "en-US" - botName = (self.config.botName or "the assistant").strip() - persona = (self.config.aiSystemPrompt or "").strip() - - # The prompt is in English on purpose — these are instructions to the - # LLM, not user-facing text. The OUTPUT is required to be in - # ``targetLang``. We ask for a strict JSON array so parsing is robust. - prompt = ( - f"You are localizing short SPOKEN-LANGUAGE utterances for a " - f"meeting assistant named '{botName}'.\n\n" - f"Persona / style guide for the assistant:\n" - f"{persona or '(no persona configured — use a neutral, polite, professional tone)'}\n\n" - f"Target spoken language (BCP-47 code): {targetLang}\n\n" - f"Utterance intent:\n{intent}\n\n" - f"Generate {count} DIFFERENT variants matching this intent, in " - f"the target language. Variants should feel natural when spoken " - f"aloud, not robotic. Do NOT include the assistant's name in " - f"the variants.\n\n" - f"Output STRICTLY a JSON array of {count} plain-text strings, " - f"with no markdown fences, no commentary, no surrounding " - f"quotation marks beyond the JSON syntax itself. Example " - f"format: [\"...\", \"...\", \"...\", \"...\"]" - ) - - try: - aiService = createAiService( - self.currentUser, self.mandateId, self.instanceId - ) - await aiService.ensureAiObjectsInitialized() - request = AiCallRequest( - prompt=prompt, - context="", - options=AiCallOptions( - operationType=OperationTypeEnum.DATA_ANALYSE, - priority=PriorityEnum.SPEED, - ), - ) - response = await aiService.callAi(request) - except Exception as aiErr: - logger.warning( - f"Ephemeral phrase generation failed (kind={kind}, lang={targetLang}): {aiErr}" - ) - return [] - - if not response or response.errorCount != 0 or not response.content: - logger.warning( - f"Ephemeral phrase generation returned empty/error " - f"(kind={kind}, lang={targetLang})" - ) - return [] - - raw = response.content.strip() - # Strip optional ```json ... ``` fences before parsing. - raw = re.sub(r"^```(?:json)?\s*", "", raw) - raw = re.sub(r"\s*```\s*$", "", raw) - try: - arr = json.loads(raw) - except json.JSONDecodeError as parseErr: - logger.warning( - f"Ephemeral phrase generation: could not parse JSON " - f"(kind={kind}, lang={targetLang}): {parseErr} " - f"raw={raw[:200]}" - ) - return [] - if not isinstance(arr, list): - return [] - cleaned = [ - str(v).strip() - for v in arr - if isinstance(v, str) and str(v).strip() - ] - cleaned = cleaned[:count] - if cleaned: - logger.info( - f"Ephemeral phrase pool generated (kind={kind}, " - f"lang={targetLang}, count={len(cleaned)})" - ) - return cleaned - def _shouldFireQuickAck(self) -> bool: - """Centralized gate so the call sites stay short and consistent.""" + """Centralized gate for quick-ack firing.""" now = time.time() if (now - self._lastQuickAckTs) < _QUICK_ACK_MIN_INTERVAL_SEC: return False - # If we are already producing a real response, the ack would step on - # the actual answer's TTS — skip it. Same for an in-flight agent - # escalation: the agent will deliver its own answer (and we already - # spoke an interim "moment please" when it started). if self._aiAnalysisInProgress or self._agentEscalationInFlight: return False - # Voice channel must be active. Chat-only mode would just spam "...". channelRaw = self.config.responseChannel channelStr = ( channelRaw.value if hasattr(channelRaw, "value") else str(channelRaw) @@ -2144,852 +887,36 @@ class TeamsbotService: return False return True - async def _runQuickAck(self, sessionId: str) -> None: - """Background task: speak the short ack into the meeting via TTS. - - Designed to be fired as ``asyncio.create_task(self._runQuickAck(...))`` - the moment the bot's name is detected — does not block the regular - debounced analysis pipeline. Persists nothing to the DB and emits no - botResponse event; this is purely an audio cue ("Moment...") so the - speaker hears within ~1s that the bot is reacting. - """ - websocket = self._websocket - voiceInterface = self._voiceInterface - if websocket is None or voiceInterface is None: - return - if not self._shouldFireQuickAck(): - return - ackText = await self._pickQuickAckText() - if not ackText: - return - # Mark the throttle BEFORE TTS so two near-simultaneous detections - # don't both fire (TTS dispatch can take a few hundred ms). - self._lastQuickAckTs = time.time() - try: - await _emitSessionEvent(sessionId, "quickAck", { - "text": ackText, - "timestamp": getUtcTimestamp(), - }) - cancelHook = self._makeAnswerCancelHook() - async with self._meetingTtsLock: - outcome = await _speakTextChunked( - websocket=websocket, - voiceInterface=voiceInterface, - sessionId=sessionId, - voiceText=ackText, - languageCode=self.config.language, - voiceName=self.config.voiceId, - isCancelled=cancelHook, - ) - if not outcome.get("success"): - logger.info( - f"Session {sessionId}: Quick ack TTS failed silently " - f"({outcome.get('error')}) — main response will still go through" - ) - except asyncio.CancelledError: - logger.info(f"Session {sessionId}: Quick ack cancelled by stop signal") - except Exception as ackErr: - logger.warning(f"Session {sessionId}: Quick ack failed: {ackErr}") - finally: - self._currentQuickAckTask = None - - async def _checkPendingNameTrigger(self, delaySec: float = 3.0): - """Async loop: fire the pending name trigger once the speaker is quiet.""" - await asyncio.sleep(delaySec) - if not self._pendingNameTrigger: - return - - now = time.time() - lastActivity = self._pendingNameTrigger.get("lastActivity", 0) - detectedAt = self._pendingNameTrigger.get("detectedAt", 0) - quietSec = now - lastActivity - totalWaitSec = now - detectedAt - - if quietSec >= 3.0 or totalWaitSec >= 15.0: - trigger = self._pendingNameTrigger - self._pendingNameTrigger = None - logger.info( - f"Session {trigger['sessionId']}: Debounced name trigger fires " - f"(quiet={quietSec:.1f}s, totalWait={totalWaitSec:.1f}s)" - ) - await self._analyzeAndRespond( - trigger["sessionId"], - trigger["interface"], - trigger["voiceInterface"], - trigger["websocket"], - trigger["triggerTranscript"], - ) - else: - remaining = max(0.5, 3.0 - quietSec) - asyncio.create_task(self._checkPendingNameTrigger(remaining)) - - async def _analyzeAndRespond( - self, - sessionId: str, - interface, - voiceInterface, - websocket: WebSocket, - triggerTranscript: Dict[str, Any], - ): - """Run SPEECH_TEAMS AI analysis and respond if needed.""" - if self._aiAnalysisInProgress: - logger.info(f"Session {sessionId}: AI analysis already in progress, skipping duplicate trigger") - return - # An agent escalation from a previous trigger may still be researching - # (it lives in its own task, ``_aiAnalysisInProgress`` was already - # released when SPEECH_TEAMS returned). If we let a fresh SPEECH_TEAMS - # run now, both pipelines would race to the meeting voice channel and - # the operator would hear "two bots talking". Skip until the agent - # finishes; the speaker can re-trigger by saying the bot name again - # if they have a new question. - if self._agentEscalationInFlight: - logger.info( - f"Session {sessionId}: Agent escalation still in flight — " - f"skipping new SPEECH_TEAMS trigger to prevent overlapping replies" - ) - return - self._aiAnalysisInProgress = True - self._lastAiCallTime = time.time() - - # Build transcript context from buffer. - # Mark bot's own utterances and chat messages for the AI. - contextLines = [] - for segment in self._contextBuffer: - speaker = segment.get("speaker", "Unknown") - text = segment.get("text", "") - segSource = segment.get("source", "caption") - prefix = "Chat" if segSource == "chat" else "" - if self._isBotSpeaker(speaker): - contextLines.append(f"[YOU ({self.config.botName})]: {text}") - elif prefix: - contextLines.append(f"[{prefix}: {speaker}]: {text}") - else: - contextLines.append(f"[{speaker}]: {text}") - - # Include session context if provided by the user at session start - sessionContextStr = "" - if self._sessionContext: - sessionContextStr = f"\nSESSION_CONTEXT (background knowledge provided by the user):\n{self._sessionContext}\n" - - # Include summary of earlier conversation if available - summaryStr = "" - if self._contextSummary: - summaryStr = f"\nEARLIER_CONVERSATION_SUMMARY:\n{self._contextSummary}\n" - - # Persistent director prompts: private operator instructions that stay - # in effect across triggers (e.g. "respond in English", "always be brief"). - directorStr = self._buildPersistentDirectorContext() - - transcriptContext = f"BOT_NAME:{self.config.botName}{sessionContextStr}{summaryStr}{directorStr}\nRECENT_TRANSCRIPT:\n" + "\n".join(contextLines) - - # Call SPEECH_TEAMS - try: - aiService = createAiService(self.currentUser, self.mandateId, self.instanceId) - await aiService.ensureAiObjectsInitialized() - - request = AiCallRequest( - prompt=self.config.aiSystemPrompt, - context=transcriptContext, - options=AiCallOptions( - operationType=OperationTypeEnum.SPEECH_TEAMS, - priority=PriorityEnum.SPEED, - ) - ) - - response = await aiService.callAi(request) - - # Parse structured response - try: - speechResult = SpeechTeamsResponse.model_validate_json(response.content) - except Exception: - # Try to extract JSON from response content - try: - jsonStr = response.content - if "```json" in jsonStr: - jsonStr = jsonStr.split("```json")[1].split("```")[0] - elif "```" in jsonStr: - jsonStr = jsonStr.split("```")[1].split("```")[0] - speechResult = SpeechTeamsResponse.model_validate_json(jsonStr.strip()) - except Exception as parseErr: - logger.warning(f"Failed to parse SPEECH_TEAMS response: {parseErr}") - speechResult = SpeechTeamsResponse( - shouldRespond=False, - reasoning=f"Parse error: {str(parseErr)[:100]}", - detectedIntent="none" - ) - - logger.info( - f"SPEECH_TEAMS result: shouldRespond={speechResult.shouldRespond}, " - f"intent={speechResult.detectedIntent}, " - f"reasoning={speechResult.reasoning[:80]}..." - ) - - # Emit analysis event (always, for debug/UI) - await _emitSessionEvent(sessionId, "analysis", { - "shouldRespond": speechResult.shouldRespond, - "detectedIntent": speechResult.detectedIntent, - "reasoning": speechResult.reasoning, - "modelName": response.modelName, - "processingTime": response.processingTime, - "priceCHF": response.priceCHF, - "needsAgent": speechResult.needsAgent, - "agentReason": speechResult.agentReason, - }) - - # Hybrid routing: SPEECH_TEAMS detected a complex request that - # requires the full agent (web research, mail, multi-step). Hand - # off to the agent path; do NOT speak the SPEECH_TEAMS placeholder. - if speechResult.needsAgent: - # Director prompts (persistent + recent one-shot) have already - # delivered files to the operator. The escalation agent MUST see - # them — otherwise it answers "summarize the doc" with general - # babble because the SPEECH_TEAMS prompt itself never had file - # access. We also forward the prior agent analysis so the - # escalation can build on, not duplicate, the earlier work. - briefings = self._collectActiveDirectorBriefings() - briefingFileIds = self._collectDirectorFileIds() - briefingBlock = "" - if briefings: - parts = [] - for b in briefings: - seg = f"- ({b.get('mode')}) {b.get('text', '')}".rstrip() - if b.get("fileIds"): - seg += f"\n attachedFileIds: {', '.join(b['fileIds'])}" - if b.get("note"): - note = b["note"] - seg += ( - "\n priorAgentAnalysis: " - + (note if len(note) <= 800 else note[:800] + "...") - ) - parts.append(seg) - briefingBlock = ( - "\n\nACTIVE_OPERATOR_BRIEFINGS (private; you may read the " - "attached files via summarizeContent / readFile / " - "readContentObjects to answer the user precisely; do NOT " - "quote the directive text itself):\n" + "\n".join(parts) - ) - logger.info( - f"Session {sessionId}: SPEECH_TEAMS escalates to agent. " - f"Reason: {speechResult.agentReason or speechResult.reasoning} | " - f"briefings={len(briefings)}, fileIds={len(briefingFileIds)}" - ) - taskBrief = ( - (speechResult.agentReason - or speechResult.responseText - or "Verarbeite die juengste Sprecheranfrage und antworte ins Meeting.") - + briefingBlock - ) - # Mark escalation as in-flight BEFORE we create the task so the - # ``_aiAnalysisInProgress=False`` released in our finally block - # cannot let a competing speech trigger sneak past the gate - # before the agent task has even been scheduled. - self._agentEscalationInFlight = True - self._currentEscalationTask = asyncio.create_task( - self._runEscalationAndRelease( - sessionId=sessionId, - taskBrief=taskBrief, - briefingFileIds=briefingFileIds, - triggerTranscriptId=triggerTranscript.get("id"), - ) - ) - return - - # Step 4a: Handle STOP intent -- stop audio immediately - if speechResult.detectedIntent == "stop": - logger.info(f"Session {sessionId}: AI detected STOP intent: {speechResult.reasoning}") - if websocket: - try: - await websocket.send_text(json.dumps({ - "type": "stopAudio", - "sessionId": sessionId, - })) - except Exception as stopErr: - logger.warning(f"Failed to send stop command: {stopErr}") - return - - # Step 4b: Respond if AI decided to - if speechResult.shouldRespond and speechResult.responseText: - - if self.config.responseMode == TeamsbotResponseMode.MANUAL: - # In manual mode, suggest but don't send - await _emitSessionEvent(sessionId, "suggestedResponse", { - "responseText": speechResult.responseText, - "detectedIntent": speechResult.detectedIntent, - "reasoning": speechResult.reasoning, - }) - return - - # Determine response channel: per-request (AI) overrides config - channels = speechResult.responseChannels - if channels and isinstance(channels, list): - channelStr = ",".join(str(c).lower().strip() for c in channels) - sendVoice = "voice" in channelStr - sendChat = "chat" in channelStr - logger.info(f"Response channel (from AI): voice={sendVoice}, chat={sendChat}") - else: - channelRaw = self.config.responseChannel - channelStr = (channelRaw.value if hasattr(channelRaw, 'value') else str(channelRaw)).lower().strip() - sendVoice = channelStr in ("voice", "both") - sendChat = channelStr in ("chat", "both") - logger.info(f"Response channel (from config): '{channelStr}'") - - if sendVoice and sendChat: - responseType = TeamsbotResponseType.BOTH - elif sendVoice: - responseType = TeamsbotResponseType.AUDIO - else: - responseType = TeamsbotResponseType.CHAT - - # Suppress duplicate responses in short windows ("repeat loop" protection). - canonicalText = ( - speechResult.responseText - or speechResult.responseTextForVoice - or speechResult.responseTextForChat - or "" - ) - normalizedResponse = (canonicalText or "").strip().lower() - nowTs = time.time() - if ( - normalizedResponse - and self._lastBotResponseText == normalizedResponse - and (nowTs - self._lastBotResponseTs) < 90 - ): - logger.info(f"Session {sessionId}: Suppressing duplicate bot response within 90s window") - await _emitSessionEvent(sessionId, "analysis", { - "shouldRespond": False, - "detectedIntent": speechResult.detectedIntent, - "reasoning": "Suppressed duplicate response within 90s", - "modelName": response.modelName, - "processingTime": response.processingTime, - "priceCHF": response.priceCHF, - }) - return - - # Resolve text per channel (AI can send different content to voice vs chat) - textForVoice = speechResult.responseTextForVoice or speechResult.responseText - textForChat = speechResult.responseTextForChat or speechResult.responseText - storedText = textForChat or textForVoice or speechResult.responseText - - # 4a: Voice response (TTS -> Audio to bot, chunked for long replies) - if sendVoice and textForVoice: - await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { - "status": "requested", - "hasWebSocket": websocket is not None, - "message": "TTS generation requested", - "timestamp": getUtcTimestamp(), - }) - logger.info( - f"Session {sessionId}: TTS requested (websocket_available={websocket is not None})" - ) - if not websocket: - logger.warning( - f"Session {sessionId}: TTS skipped (bot websocket unavailable, likely fallback mode)" - ) - await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { - "status": "unavailable", - "hasWebSocket": False, - "message": "TTS skipped — bot websocket unavailable", - "timestamp": getUtcTimestamp(), - }) - if not sendChat: - sendChat = True - else: - # Long / structured answers → AI condenses for ear; chat keeps full text. - spokenText = await self._summarizeForVoice(sessionId, textForVoice) - cancelHook = self._makeAnswerCancelHook() - async with self._meetingTtsLock: - ttsOutcome = await _speakTextChunked( - websocket=websocket, - voiceInterface=voiceInterface, - sessionId=sessionId, - voiceText=spokenText, - languageCode=self.config.language, - voiceName=self.config.voiceId, - isCancelled=cancelHook, - ) - if ttsOutcome.get("success"): - logger.info( - f"Session {sessionId}: TTS audio dispatched to bot " - f"(chunks={ttsOutcome.get('chunks')}, played={ttsOutcome.get('played')})" - ) - await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { - "status": "dispatched", - "hasWebSocket": True, - "chunks": ttsOutcome.get("chunks"), - "played": ttsOutcome.get("played"), - "timestamp": getUtcTimestamp(), - }) - else: - logger.warning( - f"TTS failed for session {sessionId}: {ttsOutcome.get('error')}" - ) - await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { - "status": "failed", - "hasWebSocket": True, - "chunks": ttsOutcome.get("chunks"), - "played": ttsOutcome.get("played"), - "message": ttsOutcome.get("error"), - "timestamp": getUtcTimestamp(), - }) - if not sendChat: - sendChat = True # Fallback to chat if voice-only and TTS failed - - # 4b: Chat response (send text message to meeting chat) - if sendChat and textForChat: - try: - if websocket: - await websocket.send_text(json.dumps({ - "type": "sendChatMessage", - "sessionId": sessionId, - "text": textForChat, - })) - logger.info(f"Chat response sent for session {sessionId}") - except Exception as chatErr: - logger.warning(f"Chat message send failed for session {sessionId}: {chatErr}") - - # 4b: Store bot response - botResponseData = TeamsbotBotResponse( - sessionId=sessionId, - responseText=storedText, - responseType=responseType, - detectedIntent=speechResult.detectedIntent, - reasoning=speechResult.reasoning, - triggeredByTranscriptId=triggerTranscript.get("id"), - modelName=response.modelName, - processingTime=response.processingTime, - priceCHF=response.priceCHF, - timestamp=getUtcTimestamp(), - ).model_dump() - - createdResponse = interface.createBotResponse(botResponseData) - - # 4c: Emit SSE event - await _emitSessionEvent(sessionId, "botResponse", { - "id": createdResponse.get("id"), - "responseText": storedText, - "responseType": responseType.value, - "detectedIntent": speechResult.detectedIntent, - "reasoning": speechResult.reasoning, - "modelName": response.modelName, - "processingTime": response.processingTime, - "priceCHF": response.priceCHF, - "timestamp": botResponseData.get("timestamp"), - }) - - # Update session response count - session = interface.getSession(sessionId) - if session: - count = session.get("botResponseCount", 0) + 1 - interface.updateSession(sessionId, {"botResponseCount": count}) - - self._lastBotResponseText = normalizedResponse - self._lastBotResponseTs = nowTs - - # Record bot response in transcript (exactly once, regardless of channel) - botTranscriptData = TeamsbotTranscript( - sessionId=sessionId, - speaker=self.config.botName, - text=storedText, - timestamp=getUtcTimestamp(), - confidence=1.0, - language=self.config.language, - isFinal=True, - ).model_dump() - botTranscript = interface.createTranscript(botTranscriptData) - - self._contextBuffer.append({ - "speaker": self.config.botName, - "text": storedText, - "timestamp": getUtcTimestamp(), - "source": "botResponse", - }) - - await _emitSessionEvent(sessionId, "transcript", { - "id": botTranscript.get("id"), - "speaker": self.config.botName, - "text": storedText, - "confidence": 1.0, - "timestamp": getUtcTimestamp(), - "isContinuation": False, - "source": "botResponse", - "speakerResolvedFromHint": False, - }) - - # Reset differential writing tracker so next STT creates a new block - self._lastTranscriptSpeaker = self.config.botName - self._lastTranscriptText = storedText - self._lastTranscriptId = botTranscript.get("id") - - self._followUpWindowEnd = time.time() + 15.0 - logger.info(f"Bot responded in session {sessionId}: intent={speechResult.detectedIntent}, follow-up window open for 15s") - - # Step 5: Execute AI-issued commands (if any) - if speechResult.commands: - await self._executeCommands(sessionId, speechResult.commands, voiceInterface, websocket) - - # When AI used only commands (no responseText), emit botResponse SSE - # so the UI shows the response. Extract text from sendChat commands. - if speechResult.shouldRespond and not speechResult.responseText: - cmdTexts = [ - c.params.get("text", "") for c in speechResult.commands - if c.action == "sendChat" and c.params and c.params.get("text") - ] - combinedText = " ".join(cmdTexts) if cmdTexts else None - if combinedText: - botResponseData = TeamsbotBotResponse( - sessionId=sessionId, - responseText=combinedText, - responseType=TeamsbotResponseType.CHAT, - detectedIntent=speechResult.detectedIntent, - reasoning=speechResult.reasoning, - triggeredByTranscriptId=triggerTranscript.get("id"), - modelName=response.modelName, - processingTime=response.processingTime, - priceCHF=response.priceCHF, - timestamp=getUtcTimestamp(), - ).model_dump() - createdResponse = interface.createBotResponse(botResponseData) - await _emitSessionEvent(sessionId, "botResponse", { - "id": createdResponse.get("id"), - "responseText": combinedText, - "responseType": TeamsbotResponseType.CHAT.value, - "detectedIntent": speechResult.detectedIntent, - "reasoning": speechResult.reasoning, - "modelName": response.modelName, - "processingTime": response.processingTime, - "priceCHF": response.priceCHF, - "timestamp": botResponseData.get("timestamp"), - }) - - session = interface.getSession(sessionId) - if session: - count = session.get("botResponseCount", 0) + 1 - interface.updateSession(sessionId, {"botResponseCount": count}) - - self._followUpWindowEnd = time.time() + 15.0 - logger.info( - f"Bot responded via commands in session {sessionId}: " - f"intent={speechResult.detectedIntent}, follow-up window open for 15s" - ) - - except Exception as e: - logger.error(f"SPEECH_TEAMS analysis failed for session {sessionId}: {type(e).__name__}: {e}", exc_info=True) - await _emitSessionEvent(sessionId, "error", {"message": f"AI analysis failed: {type(e).__name__}: {str(e)}"}) - finally: - self._aiAnalysisInProgress = False - - async def _runEscalationAndRelease( - self, - sessionId: str, - taskBrief: str, - briefingFileIds: List[str], - triggerTranscriptId: Optional[str], - ) -> None: - """Background wrapper for ``_runAgentForMeeting`` that holds the - ``_agentEscalationInFlight`` flag for the entire duration of the agent - run — not just for the moment we schedule the task. Without this - wrapper, ``_aiAnalysisInProgress`` would already be ``False`` while - the agent is still researching, and a fresh SPEECH_TEAMS trigger from - a new utterance would race the agent to the voice channel.""" - try: - await self._runAgentForMeeting( - sessionId=sessionId, - taskText=taskBrief, - fileIds=briefingFileIds, - sourceLabel="speechEscalation", - triggerTranscriptId=triggerTranscriptId, - ) - except asyncio.CancelledError: - logger.info( - f"Session {sessionId}: Escalation agent task cancelled by stop signal" - ) - except Exception as escErr: - logger.error( - f"Session {sessionId}: Escalation agent task failed: " - f"{type(escErr).__name__}: {escErr}", - exc_info=True, - ) - finally: - self._agentEscalationInFlight = False - self._currentEscalationTask = None - # ========================================================================= - # AI Command Execution + # Voice helpers (kept on class) # ========================================================================= - async def _executeCommands( - self, - sessionId: str, - commands: List[TeamsbotCommand], - voiceInterface, - websocket: WebSocket, - ): - """Execute structured commands returned by the AI. - Each command is dispatched to a dedicated handler function.""" - for cmd in commands: - action = cmd.action - params = cmd.params or {} - logger.info(f"Session {sessionId}: Executing command '{action}' with params {params}") - try: - if action == "toggleTranscript": - await self._cmdToggleTranscript(sessionId, params, websocket) - elif action == "toggleChat": - await self._cmdToggleChat(sessionId, params, websocket) - elif action == "sendChat": - await self._cmdSendChat(sessionId, params, websocket) - elif action == "readChat": - await self._cmdReadChat(sessionId, params, voiceInterface, websocket) - elif action == "readAloud": - await self._cmdReadAloud(sessionId, params, voiceInterface, websocket) - elif action == "changeLanguage": - await self._cmdChangeLanguage(sessionId, params) - elif action in ("toggleMic", "toggleCamera"): - await self._cmdToggleMicOrCamera(sessionId, action, params, websocket) - elif action == "sendMail": - await self._cmdSendMail(sessionId, params) - elif action == "storeDocument": - await self._cmdStoreDocument(sessionId, params) - else: - logger.warning(f"Session {sessionId}: Unknown command '{action}'") - except Exception as cmdErr: - logger.warning(f"Session {sessionId}: Command '{action}' failed: {cmdErr}") + _VOICE_DIRECT_MAX_CHARS = 600 + _VOICE_SUMMARY_MAX_CHARS = 350 - async def _cmdToggleTranscript(self, sessionId: str, params: dict, websocket: WebSocket): - """Caption on/off - toggle Teams live transcript capture.""" - enable = params.get("enable", True) - if websocket: - await websocket.send_text(json.dumps({ - "type": "botCommand", - "sessionId": sessionId, - "command": "toggleTranscript", - "params": {"enable": enable}, - })) - - async def _cmdToggleChat(self, sessionId: str, params: dict, websocket: WebSocket): - """Chat on/off - enable/disable meeting chat monitoring.""" - enable = params.get("enable", True) - if websocket: - await websocket.send_text(json.dumps({ - "type": "botCommand", - "sessionId": sessionId, - "command": "toggleChat", - "params": {"enable": enable}, - })) - - async def _cmdSendChat(self, sessionId: str, params: dict, websocket: WebSocket): - """Send a message to the meeting chat and record it in transcript/SSE.""" - chatText = params.get("text", "") - if not chatText: - return - if websocket: - await websocket.send_text(json.dumps({ - "type": "sendChatMessage", - "sessionId": sessionId, - "text": chatText, - })) - logger.info(f"Chat command sent for session {sessionId}") - - from . import interfaceFeatureTeamsbot as interfaceDb - interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) - - transcriptData = TeamsbotTranscript( - sessionId=sessionId, - speaker=self.config.botName, - text=chatText, - timestamp=getUtcTimestamp(), - confidence=1.0, - language=self.config.language, - isFinal=True, - source="chat", - ).model_dump() - createdTranscript = interface.createTranscript(transcriptData) - - self._contextBuffer.append({ - "speaker": self.config.botName, - "text": chatText, - "timestamp": getUtcTimestamp(), - "source": "chat", - }) - self._lastTranscriptSpeaker = self.config.botName - self._lastTranscriptText = chatText - self._lastTranscriptId = createdTranscript.get("id") - self._lastBotResponseText = chatText.strip().lower() - self._lastBotResponseTs = time.time() - - await _emitSessionEvent(sessionId, "transcript", { - "id": createdTranscript.get("id"), - "speaker": self.config.botName, - "text": chatText, - "confidence": 1.0, - "timestamp": getUtcTimestamp(), - "isContinuation": False, - "source": "chat", - "speakerResolvedFromHint": False, - }) - - async def _cmdReadChat( - self, - sessionId: str, - params: dict, - voiceInterface, - websocket: WebSocket, - ): - """Read chat messages (from DB) with optional fromdatetime/todatetime, then speak or send to chat.""" - from . import interfaceFeatureTeamsbot as interfaceDb - interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) - transcripts = interface.getTranscripts(sessionId) - fromDtRaw = params.get("fromdatetime") or params.get("fromDateTime") - toDtRaw = params.get("todatetime") or params.get("toDateTime") - fromTs = datetime.fromisoformat(fromDtRaw).replace(tzinfo=timezone.utc).timestamp() if fromDtRaw else None - toTs = datetime.fromisoformat(toDtRaw).replace(tzinfo=timezone.utc).timestamp() if toDtRaw else None - chatOnly = [t for t in transcripts if t.get("source") in ("chat", "chatHistory")] - if fromTs is not None: - chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) >= fromTs] - if toTs is not None: - chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) <= toTs] - summary = "\n".join(f"[{t.get('speaker', '?')}]: {t.get('text', '')}" for t in chatOnly[-20:]) - if not summary: - summary = "Keine Chat-Nachrichten im angegebenen Zeitraum." - if voiceInterface and websocket: - spokenSummary = await self._summarizeForVoice(sessionId, summary[:2000]) - cancelHook = self._makeAnswerCancelHook() - async with self._meetingTtsLock: - await _speakTextChunked( - websocket=websocket, - voiceInterface=voiceInterface, - sessionId=sessionId, - voiceText=spokenSummary, - languageCode=self.config.language, - voiceName=self.config.voiceId, - isCancelled=cancelHook, - ) - - async def _cmdReadAloud( - self, - sessionId: str, - params: dict, - voiceInterface, - websocket: WebSocket, - ): - """Read text aloud via TTS and play in meeting.""" - readText = params.get("text", "") - if readText and voiceInterface and websocket: - cancelHook = self._makeAnswerCancelHook() - async with self._meetingTtsLock: - await _speakTextChunked( - websocket=websocket, - voiceInterface=voiceInterface, - sessionId=sessionId, - voiceText=_voiceFriendlyMeetingText(readText), - languageCode=self.config.language, - voiceName=self.config.voiceId, - isCancelled=cancelHook, - ) - - async def _cmdChangeLanguage(self, sessionId: str, params: dict): - """Change bot language.""" - newLang = params.get("language", "") - if newLang: - self.config = self.config.model_copy(update={"language": newLang}) - logger.info(f"Session {sessionId}: Language changed to '{newLang}'") - await _emitSessionEvent(sessionId, "languageChanged", {"language": newLang}) - - async def _cmdToggleMicOrCamera( - self, - sessionId: str, - action: str, - params: dict, - websocket: WebSocket, - ): - """Toggle mic or camera in the meeting.""" - if websocket: - await websocket.send_text(json.dumps({ - "type": "botCommand", - "sessionId": sessionId, - "command": action, - "params": params, - })) - - async def _cmdSendMail(self, sessionId: str, params: dict): - """Send email via Service Center MessagingService.""" - recipient = params.get("recipient") or params.get("to", "") - subject = params.get("subject", "") - message = params.get("message") or params.get("body", "") - if not recipient or not subject: - logger.warning(f"Session {sessionId}: sendMail requires recipient and subject") - return - try: - from modules.serviceCenter import ServiceCenterContext, getService - ctx = ServiceCenterContext( - user=self.currentUser, - mandate_id=self.mandateId, - feature_instance_id=self.instanceId, - ) - messaging = getService("messaging", ctx) - success = messaging.sendEmailDirect( - recipient=recipient, - subject=subject, - message=message, - userId=str(self.currentUser.id) if self.currentUser else None, - ) - if success: - logger.info(f"Session {sessionId}: Email sent to {recipient}") - else: - logger.warning(f"Session {sessionId}: Email send failed for {recipient}") - except Exception as e: - logger.warning(f"Session {sessionId}: sendMail failed: {e}") - - async def _cmdStoreDocument(self, sessionId: str, params: dict): - """Store document via Service Center SharepointService.""" - sitePath = params.get("sitePath") or params.get("site", "") - folderPath = params.get("folderPath") or params.get("folder", "") - fileName = params.get("fileName", "document.txt") - content = params.get("content", "") - if isinstance(content, str): - content = content.encode("utf-8") - if not sitePath or not folderPath: - logger.warning(f"Session {sessionId}: storeDocument requires sitePath and folderPath") - return - try: - from modules.serviceCenter import ServiceCenterContext, getService - ctx = ServiceCenterContext( - user=self.currentUser, - mandate_id=self.mandateId, - feature_instance_id=self.instanceId, - ) - sharepoint = getService("sharepoint", ctx) - if not sharepoint.setAccessTokenFromConnection(self.currentUser): - logger.warning(f"Session {sessionId}: SharePoint connection not configured") - return - site = await sharepoint.getSiteByStandardPath(sitePath) - if not site: - logger.warning(f"Session {sessionId}: SharePoint site not found: {sitePath}") - return - result = await sharepoint.uploadFile( - siteId=site["id"], - folderPath=folderPath, - fileName=fileName, - content=content, - ) - if "error" in result: - logger.warning(f"Session {sessionId}: storeDocument failed: {result['error']}") - else: - logger.info(f"Session {sessionId}: Document stored: {fileName}") - except Exception as e: - logger.warning(f"Session {sessionId}: storeDocument failed: {e}") + @staticmethod + def _looksLikeStructuredText(raw: str) -> bool: + """Heuristic: does the original answer have markdown structure?""" + if not raw: + return False + if raw.count("|") >= 4: + return True + if raw.count("\n#") >= 1: + return True + if raw.count("\n- ") + raw.count("\n* ") + raw.count("\n• ") >= 3: + return True + if re.search(r"\n\d+[\.\)]\s", raw): + count = len(re.findall(r"(?m)^\s*\d+[\.\)]\s", raw)) + if count >= 3: + return True + return False # ========================================================================= # Director Prompts (private operator instructions during a live meeting) # ========================================================================= def _collectActiveDirectorBriefings(self) -> List[Dict[str, Any]]: - """Return the deduplicated list of director-prompt briefings that are - currently relevant for the meeting context: every active persistent - prompt PLUS every recent one-shot prompt that still sits in the - ``_recentDirectorBriefings`` pool. Each entry carries ``text``, - ``fileIds`` (UDB attachments), ``mode``, ``promptId`` and ``note`` - (the agent's internal analysis from the SILENT director run, if any). - """ + """Return the deduplicated list of director-prompt briefings.""" seen: Dict[str, Dict[str, Any]] = {} for p in self._activePersistentPrompts: pid = p.get("id") or "" @@ -3003,8 +930,6 @@ class TeamsbotService: for b in self._recentDirectorBriefings: pid = b.get("promptId") or "" if pid in seen: - # Refresh note with the latest analysis if the persistent run - # produced one after the prompt was first loaded from DB. if b.get("note"): seen[pid]["note"] = b["note"] continue @@ -3018,10 +943,7 @@ class TeamsbotService: return [v for v in seen.values() if v.get("text") or v.get("fileIds")] def _collectDirectorFileIds(self) -> List[str]: - """Flat, deduplicated list of UDB file IDs attached to any currently - relevant director prompt (persistent + recent one-shot). Used when - SPEECH_TEAMS escalates to the agent so the agent can actually READ the - documents the operator already provided.""" + """Flat, deduplicated list of UDB file IDs attached to director prompts.""" out: List[str] = [] seen: set = set() for b in self._collectActiveDirectorBriefings(): @@ -3032,19 +954,7 @@ class TeamsbotService: return out def _buildPersistentDirectorContext(self) -> str: - """Render active director-prompt briefings as private operator guidance - for the SPEECH_TEAMS system prompt context block. - - Surfaces three things SPEECH_TEAMS otherwise misses: - - * the operator's directive text (as before) - * the IDs of any UDB files the operator attached — so SPEECH_TEAMS - knows the documents exist and can decide to escalate to the agent, - which has the tooling to read them. - * the agent's previous internal analysis of the prompt (the SILENT - ``MEETING_REPLY/SILENT`` decision's note), so SPEECH_TEAMS can answer - short questions without re-running the agent. - """ + """Render active director-prompt briefings for SPEECH_TEAMS context.""" briefings = self._collectActiveDirectorBriefings() if not briefings: return "" @@ -3078,12 +988,8 @@ class TeamsbotService: internalNote: str, meetingText: str, ) -> None: - """Append a director-prompt briefing to the session-scoped pool so the - attached files and the agent's analysis stay available for subsequent - SPEECH_TEAMS triggers — even after a one-shot prompt was consumed. - Idempotent per ``promptId`` (latest entry wins).""" + """Append a director-prompt briefing to the session-scoped pool.""" pid = prompt.get("id") or "" - # Drop any older entry for the same prompt so we keep the freshest note. self._recentDirectorBriefings = [ b for b in self._recentDirectorBriefings if b.get("promptId") != pid ] @@ -3108,11 +1014,7 @@ class TeamsbotService: mode: TeamsbotDirectorPromptMode, fileIds: List[str], ) -> Dict[str, Any]: - """Persist a new director prompt and trigger immediate agent processing. - - Returns the created prompt record. Processing happens asynchronously - and emits SSE events ('directorPrompt') for the operator UI. - """ + """Persist a new director prompt and trigger immediate agent processing.""" from . import interfaceFeatureTeamsbot as interfaceDb interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) @@ -3128,9 +1030,6 @@ class TeamsbotService: ).model_dump() created = interface.createDirectorPrompt(promptData) - # Persistent prompts join in-memory directives immediately so they - # also influence subsequent SPEECH_TEAMS triggers, not only the - # one-shot agent run we kick off below. if mode == TeamsbotDirectorPromptMode.PERSISTENT: self._activePersistentPrompts.append(created) @@ -3147,7 +1046,7 @@ class TeamsbotService: return created async def removePersistentPrompt(self, promptId: str) -> bool: - """Remove a persistent director prompt (operator clicked 'remove').""" + """Remove a persistent director prompt.""" from . import interfaceFeatureTeamsbot as interfaceDb interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) @@ -3163,9 +1062,6 @@ class TeamsbotService: self._activePersistentPrompts = [ p for p in self._activePersistentPrompts if p.get("id") != promptId ] - # Also drop the briefing copy so SPEECH_TEAMS forgets the doc reference - # immediately; otherwise the bot would keep "remembering" a doc the - # operator just retired. self._recentDirectorBriefings = [ b for b in self._recentDirectorBriefings if b.get("promptId") != promptId ] @@ -3180,8 +1076,7 @@ class TeamsbotService: return True async def _processDirectorPrompt(self, prompt: Dict[str, Any]) -> None: - """Run the agent for a director prompt and deliver the FINAL text into - the meeting via TTS + chat (using the bot's existing channels).""" + """Run the agent for a director prompt and deliver the result.""" from . import interfaceFeatureTeamsbot as interfaceDb sessionId = prompt.get("sessionId") @@ -3196,18 +1091,12 @@ class TeamsbotService: "status": TeamsbotDirectorPromptStatus.RUNNING.value, }) - # Build a task brief for the agent that surfaces the meeting context. recentTranscript = self._renderRecentTranscriptForAgent(maxLines=20) directorText = (prompt.get("text") or "").strip() attachedFileIds = list(prompt.get("fileIds") or []) promptMode = (prompt.get("mode") or "").lower() isPersistentPrompt = promptMode == TeamsbotDirectorPromptMode.PERSISTENT.value.lower() - # Make file attachment EXPLICIT in the brief. The agent service already - # prepends a "## Attached Files & Folders" header via _enrichPromptWithFiles - # when fileIds are passed, but without an explicit instruction the agent - # sometimes goes straight to a generic answer. We force the workflow: - # studyDocs -> form briefing -> decide MEETING_REPLY vs SILENT. filesBlock = "" if attachedFileIds: filesBlock = ( @@ -3219,10 +1108,6 @@ class TeamsbotService: "Meeting-Reply ein, statt allgemein zu antworten.\n" ) - # Persistent prompts that ship documents are usually a "knowledge briefing" - # the operator wants the bot to STUDY now and USE LATER. The SILENT note - # in that case must be a useful, file-grounded summary that subsequent - # SPEECH_TEAMS triggers can pick up — not "noted". persistentNoteHint = "" if isPersistentPrompt and attachedFileIds: persistentNoteHint = ( @@ -3270,7 +1155,6 @@ class TeamsbotService: directorPromptMode=True, ) - # One-shot: mark consumed; persistent: keep active but record success. isPersistent = prompt.get("mode") == TeamsbotDirectorPromptMode.PERSISTENT.value updates: Dict[str, Any] = { "status": TeamsbotDirectorPromptStatus.SUCCEEDED.value, @@ -3305,8 +1189,7 @@ class TeamsbotService: ] def _renderRecentTranscriptForAgent(self, maxLines: int = 20) -> str: - """Render the most recent context buffer entries for inclusion in the - agent task brief (similar to SPEECH_TEAMS context, but plain text).""" + """Render the most recent context buffer entries for agent task brief.""" if not self._contextBuffer: return "(noch keine Aussagen erfasst)" recent = self._contextBuffer[-maxLines:] @@ -3323,21 +1206,13 @@ class TeamsbotService: return "\n".join(lines) async def _interimAgentBusyMessage(self) -> Optional[str]: - """Short spoken/chat line before a potentially long agent run (web, - tools). Phrasing is AI-localised to ``self.config.language`` and - cached per session — no hardcoded language branching. Returns - ``None`` if generation failed; caller must treat that as - 'silently skip the interim notice'.""" + """Short spoken/chat line before a potentially long agent run.""" return await self._pickEphemeralPhrase("agentBusy") async def _interimAgentRoundMessage( self, lastToolLabel: Optional[str] = None ) -> Optional[str]: - """Per-round progress notice for long agent runs (meeting voice / - chat, ephemeral). Generates a single short phrase in the bot's - configured language that describes the current activity. Unlike - the cached ephemeral phrases, this is a per-call AI generation - to avoid mixing English displayLabels into non-English speech.""" + """Per-round progress notice for long agent runs.""" targetLang = (self.config.language or "").strip() or "en-US" botName = (self.config.botName or "the assistant").strip() activityHint = lastToolLabel or "working on the task" @@ -3379,9 +1254,7 @@ class TeamsbotService: return result async def _notifyMeetingEphemeral(self, sessionId: str, text: str) -> None: - """Deliver a short line to the meeting (TTS + chat per config) without - persisting botResponses/transcripts, so the main agent answer stays the - single recorded follow-up.""" + """Deliver a short line to the meeting without persisting botResponses.""" websocket = self._websocket voiceInterface = self._voiceInterface if not websocket: @@ -3440,18 +1313,7 @@ class TeamsbotService: promptId: Optional[str] = None, directorPromptMode: bool = False, ) -> str: - """Run agentService.runAgent for a meeting context, deliver the FINAL - text via the bot's existing TTS + chat channels, and return that text. - - sourceLabel is used for logging and SSE differentiation - ('directorPrompt' or 'speechEscalation'). - - ``directorPromptMode`` activates the silent-by-default protocol for - operator director prompts: interim notices are suppressed, no per-round - meeting updates, and the FINAL text is parsed for an explicit - ``MEETING_REPLY:`` / ``SILENT:`` marker. Only ``MEETING_REPLY`` content - is dispatched to the meeting; everything else stays internal. - """ + """Run agentService.runAgent for a meeting context.""" from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( AgentConfig, AgentEventTypeEnum ) @@ -3464,7 +1326,6 @@ class TeamsbotService: ) agentService = _getServiceCenterService("agent", ctx) - # Workflow id stable per session so RAG/round-memory accumulate per meeting. workflowId = f"teamsbot:{sessionId}" agentConfig = AgentConfig( @@ -3482,8 +1343,6 @@ class TeamsbotService: "timestamp": getUtcTimestamp(), }) - # Director prompts run silently by default — no spontaneous "moment please" - # in the meeting just because the operator gave an internal directive. if not directorPromptMode: try: interimText = await self._interimAgentBusyMessage() @@ -3589,12 +1448,6 @@ class TeamsbotService: "internalNote": internalNote, }) - # Record this prompt as a session-scoped briefing BEFORE we hand - # delivery off. This is what later SPEECH_TEAMS triggers see, so - # if the user attached a doc with mode=PERSISTENT and the agent - # produced a file-grounded SILENT note, that note (and the - # original fileIds) stays available for "summarize the doc" - # follow-up questions in the meeting. try: promptRecord: Dict[str, Any] = {} if promptId: @@ -3619,11 +1472,6 @@ class TeamsbotService: f"Session {sessionId}: Director briefing pool update failed: {briefErr}" ) - # If this was a persistent prompt, the live in-memory copy in - # ``_activePersistentPrompts`` was loaded BEFORE the agent ran - # — refresh its ``responseText`` so subsequent - # ``_collectActiveDirectorBriefings`` calls show the latest - # analysis without waiting for the next session reload. if promptId: for p in self._activePersistentPrompts: if p.get("id") == promptId: @@ -3639,10 +1487,6 @@ class TeamsbotService: triggerTranscriptId=triggerTranscriptId, ) else: - # Silent: persist as internal-only botResponse so the operator - # UI keeps a record, but DO NOT push into the meeting (no TTS, - # no chat send). The director prompt SSE above already carries - # the note for the operator UI. await self._persistInternalDirectorReply( sessionId=sessionId, internalNote=internalNote or finalText, @@ -3669,13 +1513,7 @@ class TeamsbotService: reasoning: str, triggerTranscriptId: Optional[str] = None, ) -> None: - """Send agent text into the meeting via the same channels SPEECH_TEAMS - uses: TTS + chat per config, plus DB persistence and SSE events. - - Uses the websocket/voiceInterface stored on this instance. If the bot - is not connected anymore, the call still records the response in the DB - and emits SSE so the operator UI shows the agent answer. - """ + """Send agent text into the meeting via TTS + chat per config.""" from . import interfaceFeatureTeamsbot as interfaceDb interface = interfaceDb.getInterface(self.currentUser, self.mandateId, self.instanceId) @@ -3696,12 +1534,10 @@ class TeamsbotService: else: responseType = TeamsbotResponseType.CHAT - # Voice (TTS input is voice-sanitized; chat + DB keep full structured text). - # Long agent answers must be chunked: Google TTS rejects single sentences - # > ~5000 bytes, and the Chirp3 voices fail on long comma-heavy lines too. ttsOutcome: Optional[Dict[str, Any]] = None if sendVoice and voiceInterface and websocket: - spokenText = await self._summarizeForVoice(sessionId, text) + from .serviceConversation import _summarizeForVoice + spokenText = await _summarizeForVoice(self, sessionId, text) cancelHook = self._makeAnswerCancelHook() async with self._meetingTtsLock: ttsOutcome = await _speakTextChunked( @@ -3729,7 +1565,6 @@ class TeamsbotService: if not sendChat: sendChat = True - # Chat if sendChat and websocket: try: await websocket.send_text(json.dumps({ @@ -3741,7 +1576,6 @@ class TeamsbotService: except Exception as chatErr: logger.warning(f"Session {sessionId}: Agent chat delivery failed: {chatErr}") - # Persist as botResponse + transcript so it shows up in history/UI. intentEnum, intentMeta = _coercePersistedDetectedIntent(detectedIntent) reasoningForDb = ( f"{reasoning} [{intentMeta}]" if intentMeta else reasoning @@ -3821,13 +1655,7 @@ class TeamsbotService: promptId: Optional[str], triggerTranscriptId: Optional[str] = None, ) -> None: - """Record a director-prompt agent reply as INTERNAL (operator-UI only). - - Unlike ``_deliverTextToMeeting`` this never dispatches TTS or chat into - the meeting, never appends to the meeting context buffer, and does not - create a meeting transcript line. It only persists a botResponse and - emits an SSE event so the operator UI shows what the agent decided. - """ + """Record a director-prompt agent reply as INTERNAL (operator-UI only).""" from . import interfaceFeatureTeamsbot as interfaceDb note = (internalNote or "").strip() @@ -3876,20 +1704,16 @@ class TeamsbotService: ) # ========================================================================= - # Greeting (AI-localised, no hardcoded language strings) + # Greeting (AI-localised) # ========================================================================= async def _generateGreetingText(self, languageCode: str) -> str: - """Generate the bot's join greeting via AI in ``languageCode`` and the - configured persona. Returns empty string on failure — the caller must - treat that as 'skip the greeting' (NEVER fall back to a hardcoded - localised string).""" + """Generate the bot's join greeting via AI.""" targetLang = (languageCode or self.config.language or "").strip() or "en-US" botName = (self.config.botName or "the assistant").strip() firstName = botName.split(" ")[0] if botName else botName persona = (self.config.aiSystemPrompt or "").strip() - # English instructions to the LLM; the OUTPUT must be in ``targetLang``. prompt = ( f"You are localizing the join greeting for a meeting assistant.\n\n" f"Assistant display name (use exactly this, no translation): {firstName}\n\n" @@ -3935,7 +1759,6 @@ class TeamsbotService: return "" text = response.content.strip() - # Strip any wrapping quotes/code fences the model might have added. text = re.sub(r"^```.*?\n", "", text, flags=re.DOTALL) text = re.sub(r"\n```\s*$", "", text) text = text.strip().strip("\"'`").strip() @@ -3956,15 +1779,7 @@ class TeamsbotService: voiceInterface: Any, websocket: WebSocket, ) -> None: - """Centralised dispatcher for the bot's join greeting: speaks the - text via TTS into the meeting and (optionally) tells the bot to post - it in the meeting chat. Persists the greeting as a bot transcript / - botResponse so it appears in the operator UI history. - - ``sendToChat`` is ``False`` for the legacy ``voiceGreeting`` path - (the bot already chatted itself) and ``True`` for the new - ``requestGreeting`` path where the Gateway owns chat dispatch too. - """ + """Dispatch the bot's join greeting (TTS + optional chat).""" try: await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { "status": "requested", @@ -4065,12 +1880,11 @@ class TeamsbotService: ) # ========================================================================= - # Context Summarization (for long sessions) + # Context Summarization # ========================================================================= async def _summarizeSessionContext(self, sessionId: str, rawContext: str) -> str: - """Summarize a long user-provided session context to its essential points. - This reduces token usage in every subsequent AI call.""" + """Summarize a long user-provided session context.""" try: aiService = createAiService(self.currentUser, self.mandateId, self.instanceId) await aiService.ensureAiObjectsInitialized() @@ -4096,25 +1910,21 @@ class TeamsbotService: return summary except Exception as e: logger.warning(f"Session context summarization failed for {sessionId}: {e}") - - # Fallback: return original (truncated if very long) + return rawContext[:2000] if len(rawContext) > 2000 else rawContext async def _summarizeContextBuffer(self, sessionId: str): - """Summarize the older part of the context buffer to preserve information - without exceeding the context window. This runs in the background.""" + """Summarize the older part of the context buffer.""" try: if self._contextSummary: - return # Already summarized recently + return - # Take the older half of the buffer for summarization halfPoint = len(self._contextBuffer) // 2 oldSegments = self._contextBuffer[:halfPoint] if len(oldSegments) < 10: - return # Not enough to summarize + return - # Build text to summarize lines = [] for seg in oldSegments: speaker = seg.get("speaker", "Unknown") @@ -4155,9 +1965,8 @@ class TeamsbotService: transcripts = interface.getTranscripts(sessionId) if not transcripts or len(transcripts) < 5: - return # Not enough content for a summary + return - # Build full transcript fullTranscript = "\n".join( f"[{t.get('speaker', 'Unknown')}]: {t.get('text', '')}" for t in transcripts diff --git a/modules/features/teamsbot/serviceCommands.py b/modules/features/teamsbot/serviceCommands.py new file mode 100644 index 00000000..55f16bf0 --- /dev/null +++ b/modules/features/teamsbot/serviceCommands.py @@ -0,0 +1,305 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Teamsbot Service — AI command execution logic. + +Extracted from service.py. All functions accept `service` (a TeamsbotService +instance) as the first parameter so the class can delegate to them. +""" + +import logging +import json +import asyncio +from datetime import datetime, timezone +from typing import List + +from fastapi import WebSocket + +from modules.shared.timeUtils import getUtcTimestamp + +from .datamodelTeamsbot import ( + TeamsbotTranscript, + TeamsbotCommand, +) + +logger = logging.getLogger(__name__) + + +async def _executeCommands( + service, + sessionId: str, + commands: List[TeamsbotCommand], + voiceInterface, + websocket: WebSocket, +): + """Execute structured commands returned by the AI.""" + for cmd in commands: + action = cmd.action + params = cmd.params or {} + logger.info(f"Session {sessionId}: Executing command '{action}' with params {params}") + try: + if action == "toggleTranscript": + await _cmdToggleTranscript(service, sessionId, params, websocket) + elif action == "toggleChat": + await _cmdToggleChat(service, sessionId, params, websocket) + elif action == "sendChat": + await _cmdSendChat(service, sessionId, params, websocket) + elif action == "readChat": + await _cmdReadChat(service, sessionId, params, voiceInterface, websocket) + elif action == "readAloud": + await _cmdReadAloud(service, sessionId, params, voiceInterface, websocket) + elif action == "changeLanguage": + await _cmdChangeLanguage(service, sessionId, params) + elif action in ("toggleMic", "toggleCamera"): + await _cmdToggleMicOrCamera(service, sessionId, action, params, websocket) + elif action == "sendMail": + await _cmdSendMail(service, sessionId, params) + elif action == "storeDocument": + await _cmdStoreDocument(service, sessionId, params) + else: + logger.warning(f"Session {sessionId}: Unknown command '{action}'") + except Exception as cmdErr: + logger.warning(f"Session {sessionId}: Command '{action}' failed: {cmdErr}") + + +async def _cmdToggleTranscript(service, sessionId: str, params: dict, websocket: WebSocket): + """Caption on/off - toggle Teams live transcript capture.""" + enable = params.get("enable", True) + if websocket: + await websocket.send_text(json.dumps({ + "type": "botCommand", + "sessionId": sessionId, + "command": "toggleTranscript", + "params": {"enable": enable}, + })) + + +async def _cmdToggleChat(service, sessionId: str, params: dict, websocket: WebSocket): + """Chat on/off - enable/disable meeting chat monitoring.""" + enable = params.get("enable", True) + if websocket: + await websocket.send_text(json.dumps({ + "type": "botCommand", + "sessionId": sessionId, + "command": "toggleChat", + "params": {"enable": enable}, + })) + + +async def _cmdSendChat(service, sessionId: str, params: dict, websocket: WebSocket): + """Send a message to the meeting chat and record it in transcript/SSE.""" + from .service import _emitSessionEvent + + chatText = params.get("text", "") + if not chatText: + return + if websocket: + await websocket.send_text(json.dumps({ + "type": "sendChatMessage", + "sessionId": sessionId, + "text": chatText, + })) + logger.info(f"Chat command sent for session {sessionId}") + + from . import interfaceFeatureTeamsbot as interfaceDb + interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId) + + transcriptData = TeamsbotTranscript( + sessionId=sessionId, + speaker=service.config.botName, + text=chatText, + timestamp=getUtcTimestamp(), + confidence=1.0, + language=service.config.language, + isFinal=True, + source="chat", + ).model_dump() + createdTranscript = interface.createTranscript(transcriptData) + + import time + service._contextBuffer.append({ + "speaker": service.config.botName, + "text": chatText, + "timestamp": getUtcTimestamp(), + "source": "chat", + }) + service._lastTranscriptSpeaker = service.config.botName + service._lastTranscriptText = chatText + service._lastTranscriptId = createdTranscript.get("id") + service._lastBotResponseText = chatText.strip().lower() + service._lastBotResponseTs = time.time() + + await _emitSessionEvent(sessionId, "transcript", { + "id": createdTranscript.get("id"), + "speaker": service.config.botName, + "text": chatText, + "confidence": 1.0, + "timestamp": getUtcTimestamp(), + "isContinuation": False, + "source": "chat", + "speakerResolvedFromHint": False, + }) + + +async def _cmdReadChat( + service, + sessionId: str, + params: dict, + voiceInterface, + websocket: WebSocket, +): + """Read chat messages (from DB) with optional fromdatetime/todatetime, then speak or send to chat.""" + from .service import _speakTextChunked + from .serviceConversation import _summarizeForVoice + + from . import interfaceFeatureTeamsbot as interfaceDb + interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId) + transcripts = interface.getTranscripts(sessionId) + fromDtRaw = params.get("fromdatetime") or params.get("fromDateTime") + toDtRaw = params.get("todatetime") or params.get("toDateTime") + fromTs = datetime.fromisoformat(fromDtRaw).replace(tzinfo=timezone.utc).timestamp() if fromDtRaw else None + toTs = datetime.fromisoformat(toDtRaw).replace(tzinfo=timezone.utc).timestamp() if toDtRaw else None + chatOnly = [t for t in transcripts if t.get("source") in ("chat", "chatHistory")] + if fromTs is not None: + chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) >= fromTs] + if toTs is not None: + chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) <= toTs] + summary = "\n".join(f"[{t.get('speaker', '?')}]: {t.get('text', '')}" for t in chatOnly[-20:]) + if not summary: + summary = "Keine Chat-Nachrichten im angegebenen Zeitraum." + if voiceInterface and websocket: + spokenSummary = await _summarizeForVoice(service, sessionId, summary[:2000]) + cancelHook = service._makeAnswerCancelHook() + async with service._meetingTtsLock: + await _speakTextChunked( + websocket=websocket, + voiceInterface=voiceInterface, + sessionId=sessionId, + voiceText=spokenSummary, + languageCode=service.config.language, + voiceName=service.config.voiceId, + isCancelled=cancelHook, + ) + + +async def _cmdReadAloud( + service, + sessionId: str, + params: dict, + voiceInterface, + websocket: WebSocket, +): + """Read text aloud via TTS and play in meeting.""" + from .service import _speakTextChunked, _voiceFriendlyMeetingText + + readText = params.get("text", "") + if readText and voiceInterface and websocket: + cancelHook = service._makeAnswerCancelHook() + async with service._meetingTtsLock: + await _speakTextChunked( + websocket=websocket, + voiceInterface=voiceInterface, + sessionId=sessionId, + voiceText=_voiceFriendlyMeetingText(readText), + languageCode=service.config.language, + voiceName=service.config.voiceId, + isCancelled=cancelHook, + ) + + +async def _cmdChangeLanguage(service, sessionId: str, params: dict): + """Change bot language.""" + from .service import _emitSessionEvent + + newLang = params.get("language", "") + if newLang: + service.config = service.config.model_copy(update={"language": newLang}) + logger.info(f"Session {sessionId}: Language changed to '{newLang}'") + await _emitSessionEvent(sessionId, "languageChanged", {"language": newLang}) + + +async def _cmdToggleMicOrCamera( + service, + sessionId: str, + action: str, + params: dict, + websocket: WebSocket, +): + """Toggle mic or camera in the meeting.""" + if websocket: + await websocket.send_text(json.dumps({ + "type": "botCommand", + "sessionId": sessionId, + "command": action, + "params": params, + })) + + +async def _cmdSendMail(service, sessionId: str, params: dict): + """Send email via Service Center MessagingService.""" + recipient = params.get("recipient") or params.get("to", "") + subject = params.get("subject", "") + message = params.get("message") or params.get("body", "") + if not recipient or not subject: + logger.warning(f"Session {sessionId}: sendMail requires recipient and subject") + return + try: + from modules.serviceCenter import ServiceCenterContext, getService + ctx = ServiceCenterContext( + user=service.currentUser, + mandate_id=service.mandateId, + feature_instance_id=service.instanceId, + ) + messaging = getService("messaging", ctx) + success = messaging.sendEmailDirect( + recipient=recipient, + subject=subject, + message=message, + userId=str(service.currentUser.id) if service.currentUser else None, + ) + if success: + logger.info(f"Session {sessionId}: Email sent to {recipient}") + else: + logger.warning(f"Session {sessionId}: Email send failed for {recipient}") + except Exception as e: + logger.warning(f"Session {sessionId}: sendMail failed: {e}") + + +async def _cmdStoreDocument(service, sessionId: str, params: dict): + """Store document via Service Center SharepointService.""" + sitePath = params.get("sitePath") or params.get("site", "") + folderPath = params.get("folderPath") or params.get("folder", "") + fileName = params.get("fileName", "document.txt") + content = params.get("content", "") + if isinstance(content, str): + content = content.encode("utf-8") + if not sitePath or not folderPath: + logger.warning(f"Session {sessionId}: storeDocument requires sitePath and folderPath") + return + try: + from modules.serviceCenter import ServiceCenterContext, getService + ctx = ServiceCenterContext( + user=service.currentUser, + mandate_id=service.mandateId, + feature_instance_id=service.instanceId, + ) + sharepoint = getService("sharepoint", ctx) + if not sharepoint.setAccessTokenFromConnection(service.currentUser): + logger.warning(f"Session {sessionId}: SharePoint connection not configured") + return + site = await sharepoint.getSiteByStandardPath(sitePath) + if not site: + logger.warning(f"Session {sessionId}: SharePoint site not found: {sitePath}") + return + result = await sharepoint.uploadFile( + siteId=site["id"], + folderPath=folderPath, + fileName=fileName, + content=content, + ) + if "error" in result: + logger.warning(f"Session {sessionId}: storeDocument failed: {result['error']}") + else: + logger.info(f"Session {sessionId}: Document stored: {fileName}") + except Exception as e: + logger.warning(f"Session {sessionId}: storeDocument failed: {e}") diff --git a/modules/features/teamsbot/serviceConversation.py b/modules/features/teamsbot/serviceConversation.py new file mode 100644 index 00000000..bf844d89 --- /dev/null +++ b/modules/features/teamsbot/serviceConversation.py @@ -0,0 +1,996 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Teamsbot Service — Conversation & AI analysis logic. + +Extracted from service.py. All functions accept `service` (a TeamsbotService +instance) as the first parameter so the class can delegate to them. +""" + +import logging +import json +import re +import asyncio +import time +from typing import Optional, Dict, Any, List + +from fastapi import WebSocket + +from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum +from modules.shared.timeUtils import getUtcTimestamp + +from .datamodelTeamsbot import ( + TeamsbotTranscript, + TeamsbotBotResponse, + TeamsbotResponseType, + TeamsbotResponseMode, + TeamsbotResponseChannel, + SpeechTeamsResponse, + TeamsbotDirectorPromptMode, + TeamsbotDirectorPromptStatus, +) + +logger = logging.getLogger(__name__) + + +async def _analyzeAndRespond( + service, + sessionId: str, + interface, + voiceInterface, + websocket: WebSocket, + triggerTranscript: Dict[str, Any], +): + """Run SPEECH_TEAMS AI analysis and respond if needed.""" + from .service import ( + _emitSessionEvent, createAiService, _speakTextChunked, + _voiceFriendlyMeetingText, + TEAMSBOT_AGENT_MAX_ROUNDS, TEAMSBOT_AGENT_MAX_COST_CHF, + ) + + if service._aiAnalysisInProgress: + logger.info(f"Session {sessionId}: AI analysis already in progress, skipping duplicate trigger") + return + if service._agentEscalationInFlight: + logger.info( + f"Session {sessionId}: Agent escalation still in flight — " + f"skipping new SPEECH_TEAMS trigger to prevent overlapping replies" + ) + return + service._aiAnalysisInProgress = True + service._lastAiCallTime = time.time() + + contextLines = [] + for segment in service._contextBuffer: + speaker = segment.get("speaker", "Unknown") + text = segment.get("text", "") + segSource = segment.get("source", "caption") + prefix = "Chat" if segSource == "chat" else "" + if service._isBotSpeaker(speaker): + contextLines.append(f"[YOU ({service.config.botName})]: {text}") + elif prefix: + contextLines.append(f"[{prefix}: {speaker}]: {text}") + else: + contextLines.append(f"[{speaker}]: {text}") + + sessionContextStr = "" + if service._sessionContext: + sessionContextStr = f"\nSESSION_CONTEXT (background knowledge provided by the user):\n{service._sessionContext}\n" + + summaryStr = "" + if service._contextSummary: + summaryStr = f"\nEARLIER_CONVERSATION_SUMMARY:\n{service._contextSummary}\n" + + directorStr = service._buildPersistentDirectorContext() + + transcriptContext = f"BOT_NAME:{service.config.botName}{sessionContextStr}{summaryStr}{directorStr}\nRECENT_TRANSCRIPT:\n" + "\n".join(contextLines) + + try: + aiService = createAiService(service.currentUser, service.mandateId, service.instanceId) + await aiService.ensureAiObjectsInitialized() + + request = AiCallRequest( + prompt=service.config.aiSystemPrompt, + context=transcriptContext, + options=AiCallOptions( + operationType=OperationTypeEnum.SPEECH_TEAMS, + priority=PriorityEnum.SPEED, + ) + ) + + response = await aiService.callAi(request) + + try: + speechResult = SpeechTeamsResponse.model_validate_json(response.content) + except Exception: + try: + jsonStr = response.content + if "```json" in jsonStr: + jsonStr = jsonStr.split("```json")[1].split("```")[0] + elif "```" in jsonStr: + jsonStr = jsonStr.split("```")[1].split("```")[0] + speechResult = SpeechTeamsResponse.model_validate_json(jsonStr.strip()) + except Exception as parseErr: + logger.warning(f"Failed to parse SPEECH_TEAMS response: {parseErr}") + speechResult = SpeechTeamsResponse( + shouldRespond=False, + reasoning=f"Parse error: {str(parseErr)[:100]}", + detectedIntent="none" + ) + + logger.info( + f"SPEECH_TEAMS result: shouldRespond={speechResult.shouldRespond}, " + f"intent={speechResult.detectedIntent}, " + f"reasoning={speechResult.reasoning[:80]}..." + ) + + await _emitSessionEvent(sessionId, "analysis", { + "shouldRespond": speechResult.shouldRespond, + "detectedIntent": speechResult.detectedIntent, + "reasoning": speechResult.reasoning, + "modelName": response.modelName, + "processingTime": response.processingTime, + "priceCHF": response.priceCHF, + "needsAgent": speechResult.needsAgent, + "agentReason": speechResult.agentReason, + }) + + if speechResult.needsAgent: + briefings = service._collectActiveDirectorBriefings() + briefingFileIds = service._collectDirectorFileIds() + briefingBlock = "" + if briefings: + parts = [] + for b in briefings: + seg = f"- ({b.get('mode')}) {b.get('text', '')}".rstrip() + if b.get("fileIds"): + seg += f"\n attachedFileIds: {', '.join(b['fileIds'])}" + if b.get("note"): + note = b["note"] + seg += ( + "\n priorAgentAnalysis: " + + (note if len(note) <= 800 else note[:800] + "...") + ) + parts.append(seg) + briefingBlock = ( + "\n\nACTIVE_OPERATOR_BRIEFINGS (private; you may read the " + "attached files via summarizeContent / readFile / " + "readContentObjects to answer the user precisely; do NOT " + "quote the directive text itself):\n" + "\n".join(parts) + ) + logger.info( + f"Session {sessionId}: SPEECH_TEAMS escalates to agent. " + f"Reason: {speechResult.agentReason or speechResult.reasoning} | " + f"briefings={len(briefings)}, fileIds={len(briefingFileIds)}" + ) + taskBrief = ( + (speechResult.agentReason + or speechResult.responseText + or "Verarbeite die juengste Sprecheranfrage und antworte ins Meeting.") + + briefingBlock + ) + service._agentEscalationInFlight = True + service._currentEscalationTask = asyncio.create_task( + _runEscalationAndRelease( + service, + sessionId=sessionId, + taskBrief=taskBrief, + briefingFileIds=briefingFileIds, + triggerTranscriptId=triggerTranscript.get("id"), + ) + ) + return + + if speechResult.detectedIntent == "stop": + logger.info(f"Session {sessionId}: AI detected STOP intent: {speechResult.reasoning}") + if websocket: + try: + await websocket.send_text(json.dumps({ + "type": "stopAudio", + "sessionId": sessionId, + })) + except Exception as stopErr: + logger.warning(f"Failed to send stop command: {stopErr}") + return + + if speechResult.shouldRespond and speechResult.responseText: + + if service.config.responseMode == TeamsbotResponseMode.MANUAL: + await _emitSessionEvent(sessionId, "suggestedResponse", { + "responseText": speechResult.responseText, + "detectedIntent": speechResult.detectedIntent, + "reasoning": speechResult.reasoning, + }) + return + + channels = speechResult.responseChannels + if channels and isinstance(channels, list): + channelStr = ",".join(str(c).lower().strip() for c in channels) + sendVoice = "voice" in channelStr + sendChat = "chat" in channelStr + logger.info(f"Response channel (from AI): voice={sendVoice}, chat={sendChat}") + else: + channelRaw = service.config.responseChannel + channelStr = (channelRaw.value if hasattr(channelRaw, 'value') else str(channelRaw)).lower().strip() + sendVoice = channelStr in ("voice", "both") + sendChat = channelStr in ("chat", "both") + logger.info(f"Response channel (from config): '{channelStr}'") + + if sendVoice and sendChat: + responseType = TeamsbotResponseType.BOTH + elif sendVoice: + responseType = TeamsbotResponseType.AUDIO + else: + responseType = TeamsbotResponseType.CHAT + + canonicalText = ( + speechResult.responseText + or speechResult.responseTextForVoice + or speechResult.responseTextForChat + or "" + ) + normalizedResponse = (canonicalText or "").strip().lower() + nowTs = time.time() + if ( + normalizedResponse + and service._lastBotResponseText == normalizedResponse + and (nowTs - service._lastBotResponseTs) < 90 + ): + logger.info(f"Session {sessionId}: Suppressing duplicate bot response within 90s window") + await _emitSessionEvent(sessionId, "analysis", { + "shouldRespond": False, + "detectedIntent": speechResult.detectedIntent, + "reasoning": "Suppressed duplicate response within 90s", + "modelName": response.modelName, + "processingTime": response.processingTime, + "priceCHF": response.priceCHF, + }) + return + + textForVoice = speechResult.responseTextForVoice or speechResult.responseText + textForChat = speechResult.responseTextForChat or speechResult.responseText + storedText = textForChat or textForVoice or speechResult.responseText + + if sendVoice and textForVoice: + await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { + "status": "requested", + "hasWebSocket": websocket is not None, + "message": "TTS generation requested", + "timestamp": getUtcTimestamp(), + }) + logger.info( + f"Session {sessionId}: TTS requested (websocket_available={websocket is not None})" + ) + if not websocket: + logger.warning( + f"Session {sessionId}: TTS skipped (bot websocket unavailable, likely fallback mode)" + ) + await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { + "status": "unavailable", + "hasWebSocket": False, + "message": "TTS skipped — bot websocket unavailable", + "timestamp": getUtcTimestamp(), + }) + if not sendChat: + sendChat = True + else: + spokenText = await _summarizeForVoice(service, sessionId, textForVoice) + cancelHook = service._makeAnswerCancelHook() + async with service._meetingTtsLock: + ttsOutcome = await _speakTextChunked( + websocket=websocket, + voiceInterface=voiceInterface, + sessionId=sessionId, + voiceText=spokenText, + languageCode=service.config.language, + voiceName=service.config.voiceId, + isCancelled=cancelHook, + ) + if ttsOutcome.get("success"): + logger.info( + f"Session {sessionId}: TTS audio dispatched to bot " + f"(chunks={ttsOutcome.get('chunks')}, played={ttsOutcome.get('played')})" + ) + await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { + "status": "dispatched", + "hasWebSocket": True, + "chunks": ttsOutcome.get("chunks"), + "played": ttsOutcome.get("played"), + "timestamp": getUtcTimestamp(), + }) + else: + logger.warning( + f"TTS failed for session {sessionId}: {ttsOutcome.get('error')}" + ) + await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { + "status": "failed", + "hasWebSocket": True, + "chunks": ttsOutcome.get("chunks"), + "played": ttsOutcome.get("played"), + "message": ttsOutcome.get("error"), + "timestamp": getUtcTimestamp(), + }) + if not sendChat: + sendChat = True + + if sendChat and textForChat: + try: + if websocket: + await websocket.send_text(json.dumps({ + "type": "sendChatMessage", + "sessionId": sessionId, + "text": textForChat, + })) + logger.info(f"Chat response sent for session {sessionId}") + except Exception as chatErr: + logger.warning(f"Chat message send failed for session {sessionId}: {chatErr}") + + botResponseData = TeamsbotBotResponse( + sessionId=sessionId, + responseText=storedText, + responseType=responseType, + detectedIntent=speechResult.detectedIntent, + reasoning=speechResult.reasoning, + triggeredByTranscriptId=triggerTranscript.get("id"), + modelName=response.modelName, + processingTime=response.processingTime, + priceCHF=response.priceCHF, + timestamp=getUtcTimestamp(), + ).model_dump() + + createdResponse = interface.createBotResponse(botResponseData) + + await _emitSessionEvent(sessionId, "botResponse", { + "id": createdResponse.get("id"), + "responseText": storedText, + "responseType": responseType.value, + "detectedIntent": speechResult.detectedIntent, + "reasoning": speechResult.reasoning, + "modelName": response.modelName, + "processingTime": response.processingTime, + "priceCHF": response.priceCHF, + "timestamp": botResponseData.get("timestamp"), + }) + + session = interface.getSession(sessionId) + if session: + count = session.get("botResponseCount", 0) + 1 + interface.updateSession(sessionId, {"botResponseCount": count}) + + service._lastBotResponseText = normalizedResponse + service._lastBotResponseTs = nowTs + + botTranscriptData = TeamsbotTranscript( + sessionId=sessionId, + speaker=service.config.botName, + text=storedText, + timestamp=getUtcTimestamp(), + confidence=1.0, + language=service.config.language, + isFinal=True, + ).model_dump() + botTranscript = interface.createTranscript(botTranscriptData) + + service._contextBuffer.append({ + "speaker": service.config.botName, + "text": storedText, + "timestamp": getUtcTimestamp(), + "source": "botResponse", + }) + + await _emitSessionEvent(sessionId, "transcript", { + "id": botTranscript.get("id"), + "speaker": service.config.botName, + "text": storedText, + "confidence": 1.0, + "timestamp": getUtcTimestamp(), + "isContinuation": False, + "source": "botResponse", + "speakerResolvedFromHint": False, + }) + + service._lastTranscriptSpeaker = service.config.botName + service._lastTranscriptText = storedText + service._lastTranscriptId = botTranscript.get("id") + + service._followUpWindowEnd = time.time() + 15.0 + logger.info(f"Bot responded in session {sessionId}: intent={speechResult.detectedIntent}, follow-up window open for 15s") + + if speechResult.commands: + from .serviceCommands import _executeCommands + await _executeCommands(service, sessionId, speechResult.commands, voiceInterface, websocket) + + if speechResult.shouldRespond and not speechResult.responseText: + cmdTexts = [ + c.params.get("text", "") for c in speechResult.commands + if c.action == "sendChat" and c.params and c.params.get("text") + ] + combinedText = " ".join(cmdTexts) if cmdTexts else None + if combinedText: + botResponseData = TeamsbotBotResponse( + sessionId=sessionId, + responseText=combinedText, + responseType=TeamsbotResponseType.CHAT, + detectedIntent=speechResult.detectedIntent, + reasoning=speechResult.reasoning, + triggeredByTranscriptId=triggerTranscript.get("id"), + modelName=response.modelName, + processingTime=response.processingTime, + priceCHF=response.priceCHF, + timestamp=getUtcTimestamp(), + ).model_dump() + createdResponse = interface.createBotResponse(botResponseData) + await _emitSessionEvent(sessionId, "botResponse", { + "id": createdResponse.get("id"), + "responseText": combinedText, + "responseType": TeamsbotResponseType.CHAT.value, + "detectedIntent": speechResult.detectedIntent, + "reasoning": speechResult.reasoning, + "modelName": response.modelName, + "processingTime": response.processingTime, + "priceCHF": response.priceCHF, + "timestamp": botResponseData.get("timestamp"), + }) + + session = interface.getSession(sessionId) + if session: + count = session.get("botResponseCount", 0) + 1 + interface.updateSession(sessionId, {"botResponseCount": count}) + + service._followUpWindowEnd = time.time() + 15.0 + logger.info( + f"Bot responded via commands in session {sessionId}: " + f"intent={speechResult.detectedIntent}, follow-up window open for 15s" + ) + + except Exception as e: + logger.error(f"SPEECH_TEAMS analysis failed for session {sessionId}: {type(e).__name__}: {e}", exc_info=True) + await _emitSessionEvent(sessionId, "error", {"message": f"AI analysis failed: {type(e).__name__}: {str(e)}"}) + finally: + service._aiAnalysisInProgress = False + + +async def _processTranscript( + service, + sessionId: str, + speaker: str, + text: str, + isFinal: bool, + interface, + voiceInterface, + websocket: WebSocket, + source: str = "caption", + speakerResolvedFromHint: Optional[bool] = None, +): + """Process a transcript segment from captions or chat messages.""" + from .service import _emitSessionEvent + + text = text.strip() + if not text: + return + + if source in ("caption", "speakerHint"): + service._registerSpeakerHint(speaker, text, sessionId) + + if ( + source == "speakerHint" + and isFinal + and not service._isBotSpeaker(speaker) + and service.config.responseMode != TeamsbotResponseMode.TRANSCRIBE_ONLY + and service._detectBotName(text) + ): + triggerTranscript = {"id": None, "speaker": speaker, "text": text, "source": source} + isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, triggerTranscript) + if isNew: + logger.info(f"Session {sessionId}: Bot name in caption, debounce trigger started") + asyncio.create_task(_checkPendingNameTrigger(service)) + service._currentQuickAckTask = asyncio.create_task( + _runQuickAck(service, sessionId) + ) + return + + if source == "chatHistory": + transcriptData = TeamsbotTranscript( + sessionId=sessionId, + speaker=speaker, + text=text, + timestamp=getUtcTimestamp(), + confidence=1.0, + language=service.config.language, + isFinal=True, + source="chatHistory", + ).model_dump() + createdTranscript = interface.createTranscript(transcriptData) + + await _emitSessionEvent(sessionId, "transcript", { + "id": createdTranscript.get("id"), + "speaker": speaker, + "text": text, + "confidence": 1.0, + "timestamp": getUtcTimestamp(), + "isContinuation": False, + "source": "chatHistory", + "isHistory": True, + }) + logger.debug(f"Session {sessionId}: Chat history stored (no AI trigger): [{speaker}] {text[:60]}") + return + + isBotSpeaker = service._isBotSpeaker(speaker) + if isBotSpeaker and source != "chat": + logger.debug(f"Session {sessionId}: Ignoring own bot caption from: [{speaker}] {text[:80]}...") + return + + sttPauseThreshold = 5.0 + isMerge = ( + source == "audioCapture" + and service._lastTranscriptSpeaker == speaker + and service._lastTranscriptText is not None + and service._lastTranscriptId is not None + and (time.time() - service._lastSttTime) < sttPauseThreshold + ) + + if isMerge: + mergedText = f"{service._lastTranscriptText} {text}" + interface.updateTranscript(service._lastTranscriptId, { + "text": mergedText, + "isFinal": isFinal, + }) + service._lastTranscriptText = mergedText + createdTranscript = {"id": service._lastTranscriptId} + + if service._contextBuffer and service._contextBuffer[-1].get("speaker") == speaker: + service._contextBuffer[-1]["text"] = mergedText + else: + transcriptData = TeamsbotTranscript( + sessionId=sessionId, + speaker=speaker, + text=text, + timestamp=getUtcTimestamp(), + confidence=1.0, + language=service.config.language, + isFinal=isFinal, + source=source, + ).model_dump() + + createdTranscript = interface.createTranscript(transcriptData) + + service._lastTranscriptSpeaker = speaker + service._lastTranscriptText = text + service._lastTranscriptId = createdTranscript.get("id") + + if source == "audioCapture" and speaker == "Unknown": + service._unattributedTranscriptIds.append(createdTranscript.get("id")) + + service._contextBuffer.append({ + "speaker": speaker or "Unknown", + "text": text, + "timestamp": getUtcTimestamp(), + "source": source, + }) + + maxSegments = service.config.contextWindowSegments + if len(service._contextBuffer) > maxSegments: + if not service._contextSummary and len(service._contextBuffer) > maxSegments * 1.5: + asyncio.create_task(service._summarizeContextBuffer(sessionId)) + service._contextBuffer = service._contextBuffer[-maxSegments:] + + session = interface.getSession(sessionId) + if session: + count = session.get("transcriptSegmentCount", 0) + 1 + interface.updateSession(sessionId, {"transcriptSegmentCount": count}) + + if source == "audioCapture": + service._lastSttTime = time.time() + + displayText = service._lastTranscriptText if isMerge else text + await _emitSessionEvent(sessionId, "transcript", { + "id": createdTranscript.get("id"), + "speaker": speaker, + "text": displayText, + "confidence": 1.0, + "timestamp": getUtcTimestamp(), + "isContinuation": isMerge, + "source": source, + "speakerResolvedFromHint": ( + speakerResolvedFromHint + if speakerResolvedFromHint is not None + else False + ), + }) + + if not isFinal: + return + + if service.config.responseMode == TeamsbotResponseMode.TRANSCRIBE_ONLY: + return + + if source == "chat" and isBotSpeaker: + return + + if service._isStopPhrase(text): + logger.info( + f"Session {sessionId}: Stop phrase detected ('{text.strip()[:60]}'), " + f"hard-cancelling in-flight speech immediately" + ) + from .serviceWebSocket import _cancelInFlightSpeech + await _cancelInFlightSpeech( + service, + sessionId=sessionId, + websocket=websocket, + reason="userStopPhrase", + ) + return + + if service._pendingNameTrigger: + service._pendingNameTrigger["lastActivity"] = time.time() + + if service._detectBotName(text): + isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript) + if isNew: + asyncio.create_task(_checkPendingNameTrigger(service)) + service._currentQuickAckTask = asyncio.create_task( + _runQuickAck(service, sessionId) + ) + return + + if ( + source == "audioCapture" + and not service._isBotSpeaker(speaker) + and time.time() < service._followUpWindowEnd + and not service._pendingNameTrigger + ): + isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript) + if isNew: + logger.info(f"Session {sessionId}: Follow-up window trigger (no name needed)") + asyncio.create_task(_checkPendingNameTrigger(service)) + return + + if not service._pendingNameTrigger: + shouldTrigger = service._shouldTriggerAnalysis(text) + if shouldTrigger: + logger.info(f"Session {sessionId}: Periodic trigger (buffer: {len(service._contextBuffer)} segments)") + await _analyzeAndRespond(service, sessionId, interface, voiceInterface, websocket, createdTranscript) + + +async def _summarizeForVoice( + service, + sessionId: str, + rawAnswer: str, +) -> str: + """Return a SHORT, naturally-spoken paraphrase of ``rawAnswer`` for TTS.""" + from .service import _voiceFriendlyMeetingText, createAiService + + if not rawAnswer or not rawAnswer.strip(): + return "" + + sanitised = _voiceFriendlyMeetingText(rawAnswer) + if ( + len(sanitised) <= service._VOICE_DIRECT_MAX_CHARS + and not service._looksLikeStructuredText(rawAnswer) + ): + return sanitised + + targetLang = (service.config.language or "de-DE").strip() + botName = (service.config.botName or "").strip() or "the assistant" + persona = (service.config.aiSystemPrompt or "").strip() + personaBlock = ( + f"\n\nBOT PERSONA / TONE:\n{persona}\n" + if persona else "" + ) + + prompt = ( + f"You are condensing a long written answer into a SHORT spoken " + f"paraphrase that the assistant '{botName}' will say out loud " + f"into a Microsoft Teams meeting. The full written answer is " + f"already in the meeting chat — your job is to summarise it for " + f"the EAR, not the eye.\n\n" + f"STRICT REQUIREMENTS:\n" + f"1. Output language: BCP-47 '{targetLang}'. No other language.\n" + f"2. 1 to 3 sentences, max ~{service._VOICE_SUMMARY_MAX_CHARS} characters total.\n" + f"3. Natural spoken style — no headings, no bullet points, no " + f"tables, no markdown, no emojis, no enumerations like 'Erstens... " + f"Zweitens...' unless that genuinely flows in speech.\n" + f"4. Capture the essence and the most important conclusion. Do " + f"NOT try to fit every detail. Listeners can read the chat for " + f"the full version.\n" + f"5. End by gently pointing the audience to the chat for details, " + f"e.g. 'Details stehen im Chat.' (adapted to the target language).\n" + f"6. Output ONLY the spoken text. No JSON, no quotes around it, " + f"no preamble like 'Here is the summary:'.\n" + f"{personaBlock}\n" + f"FULL WRITTEN ANSWER (markdown-formatted, sometimes long):\n" + f"---\n{rawAnswer.strip()[:6000]}\n---\n" + ) + + try: + aiService = createAiService( + service.currentUser, service.mandateId, service.instanceId + ) + await aiService.ensureAiObjectsInitialized() + request = AiCallRequest( + prompt=prompt, + context="", + options=AiCallOptions( + operationType=OperationTypeEnum.DATA_ANALYSE, + priority=PriorityEnum.SPEED, + ), + ) + response = await aiService.callAi(request) + except Exception as aiErr: + logger.warning( + f"Session {sessionId}: Voice summary AI call failed: {aiErr}" + ) + return sanitised[: service._VOICE_DIRECT_MAX_CHARS] + + if not response or response.errorCount != 0 or not response.content: + logger.warning( + f"Session {sessionId}: Voice summary returned empty/error" + ) + return sanitised[: service._VOICE_DIRECT_MAX_CHARS] + + spoken = response.content.strip() + spoken = _voiceFriendlyMeetingText(spoken) + if not spoken: + return sanitised[: service._VOICE_DIRECT_MAX_CHARS] + + logger.info( + f"Session {sessionId}: Voice summary generated " + f"(orig={len(rawAnswer)} chars, sanitised={len(sanitised)}, " + f"spoken={len(spoken)})" + ) + return spoken + + +async def _pickQuickAckText(service) -> Optional[str]: + """Return a short ack text in the bot's configured language.""" + return await _pickEphemeralPhrase(service, "quickAck") + + +async def _pickEphemeralPhrase( + service, + kind: str, + substitutions: Optional[Dict[str, Any]] = None, +) -> Optional[str]: + """Round-robin selector over the cached phrase pool for ``kind``.""" + variants = await _getEphemeralPhrases(service, kind) + if not variants: + return None + idx = service._phrasePoolIdx.get(kind, 0) % len(variants) + service._phrasePoolIdx[kind] = (idx + 1) % len(variants) + chosen = variants[idx] + if substitutions: + try: + chosen = chosen.format(**substitutions) + except (KeyError, IndexError, ValueError) as fmtErr: + logger.debug( + f"Ephemeral phrase substitution failed for kind={kind}: {fmtErr}" + ) + return chosen + + +async def _getEphemeralPhrases(service, kind: str) -> List[str]: + """Return the cached pool of AI-generated variants for ``kind``.""" + cached = service._phrasePool.get(kind) + if cached: + return cached + async with service._phrasePoolLock: + cached = service._phrasePool.get(kind) + if cached: + return cached + phrases = await _generateEphemeralPhrases(service, kind, 4) + if phrases: + service._phrasePool[kind] = phrases + return phrases + + +async def _generateEphemeralPhrases( + service, kind: str, count: int +) -> List[str]: + """Ask the AI to produce ``count`` short utterances for ``kind``.""" + from .service import createAiService, _EPHEMERAL_PHRASE_INTENTS + + intent = _EPHEMERAL_PHRASE_INTENTS.get(kind) + if not intent: + logger.warning(f"Unknown ephemeral phrase kind requested: {kind}") + return [] + + targetLang = (service.config.language or "").strip() or "en-US" + botName = (service.config.botName or "the assistant").strip() + persona = (service.config.aiSystemPrompt or "").strip() + + prompt = ( + f"You are localizing short SPOKEN-LANGUAGE utterances for a " + f"meeting assistant named '{botName}'.\n\n" + f"Persona / style guide for the assistant:\n" + f"{persona or '(no persona configured — use a neutral, polite, professional tone)'}\n\n" + f"Target spoken language (BCP-47 code): {targetLang}\n\n" + f"Utterance intent:\n{intent}\n\n" + f"Generate {count} DIFFERENT variants matching this intent, in " + f"the target language. Variants should feel natural when spoken " + f"aloud, not robotic. Do NOT include the assistant's name in " + f"the variants.\n\n" + f"Output STRICTLY a JSON array of {count} plain-text strings, " + f"with no markdown fences, no commentary, no surrounding " + f"quotation marks beyond the JSON syntax itself. Example " + f"format: [\"...\", \"...\", \"...\", \"...\"]" + ) + + try: + aiService = createAiService( + service.currentUser, service.mandateId, service.instanceId + ) + await aiService.ensureAiObjectsInitialized() + request = AiCallRequest( + prompt=prompt, + context="", + options=AiCallOptions( + operationType=OperationTypeEnum.DATA_ANALYSE, + priority=PriorityEnum.SPEED, + ), + ) + response = await aiService.callAi(request) + except Exception as aiErr: + logger.warning( + f"Ephemeral phrase generation failed (kind={kind}, lang={targetLang}): {aiErr}" + ) + return [] + + if not response or response.errorCount != 0 or not response.content: + logger.warning( + f"Ephemeral phrase generation returned empty/error " + f"(kind={kind}, lang={targetLang})" + ) + return [] + + raw = response.content.strip() + raw = re.sub(r"^```(?:json)?\s*", "", raw) + raw = re.sub(r"\s*```\s*$", "", raw) + try: + arr = json.loads(raw) + except json.JSONDecodeError as parseErr: + logger.warning( + f"Ephemeral phrase generation: could not parse JSON " + f"(kind={kind}, lang={targetLang}): {parseErr} " + f"raw={raw[:200]}" + ) + return [] + if not isinstance(arr, list): + return [] + cleaned = [ + str(v).strip() + for v in arr + if isinstance(v, str) and str(v).strip() + ] + cleaned = cleaned[:count] + if cleaned: + logger.info( + f"Ephemeral phrase pool generated (kind={kind}, " + f"lang={targetLang}, count={len(cleaned)})" + ) + return cleaned + + +async def _runQuickAck(service, sessionId: str) -> None: + """Background task: speak a short ack into the meeting via TTS.""" + from .service import _emitSessionEvent, _speakTextChunked + + websocket = service._websocket + voiceInterface = service._voiceInterface + if websocket is None or voiceInterface is None: + return + if not service._shouldFireQuickAck(): + return + ackText = await _pickQuickAckText(service) + if not ackText: + return + service._lastQuickAckTs = time.time() + try: + await _emitSessionEvent(sessionId, "quickAck", { + "text": ackText, + "timestamp": getUtcTimestamp(), + }) + cancelHook = service._makeAnswerCancelHook() + async with service._meetingTtsLock: + outcome = await _speakTextChunked( + websocket=websocket, + voiceInterface=voiceInterface, + sessionId=sessionId, + voiceText=ackText, + languageCode=service.config.language, + voiceName=service.config.voiceId, + isCancelled=cancelHook, + ) + if not outcome.get("success"): + logger.info( + f"Session {sessionId}: Quick ack TTS failed silently " + f"({outcome.get('error')}) — main response will still go through" + ) + except asyncio.CancelledError: + logger.info(f"Session {sessionId}: Quick ack cancelled by stop signal") + except Exception as ackErr: + logger.warning(f"Session {sessionId}: Quick ack failed: {ackErr}") + finally: + service._currentQuickAckTask = None + + +async def _checkPendingNameTrigger(service, delaySec: float = 3.0): + """Async loop: fire the pending name trigger once the speaker is quiet.""" + await asyncio.sleep(delaySec) + if not service._pendingNameTrigger: + return + + now = time.time() + lastActivity = service._pendingNameTrigger.get("lastActivity", 0) + detectedAt = service._pendingNameTrigger.get("detectedAt", 0) + quietSec = now - lastActivity + totalWaitSec = now - detectedAt + + if quietSec >= 3.0 or totalWaitSec >= 15.0: + trigger = service._pendingNameTrigger + service._pendingNameTrigger = None + logger.info( + f"Session {trigger['sessionId']}: Debounced name trigger fires " + f"(quiet={quietSec:.1f}s, totalWait={totalWaitSec:.1f}s)" + ) + await _analyzeAndRespond( + service, + trigger["sessionId"], + trigger["interface"], + trigger["voiceInterface"], + trigger["websocket"], + trigger["triggerTranscript"], + ) + else: + remaining = max(0.5, 3.0 - quietSec) + asyncio.create_task(_checkPendingNameTrigger(service, remaining)) + + +async def _warmEphemeralPhrasePool(service, sessionId: str) -> None: + """Fire-and-forget: generate ephemeral phrase pool for all kinds.""" + from .service import _EPHEMERAL_PHRASE_INTENTS + + try: + for kind in _EPHEMERAL_PHRASE_INTENTS: + try: + await _getEphemeralPhrases(service, kind) + except Exception as innerErr: + logger.warning( + f"Session {sessionId}: Phrase pool warmup failed for " + f"kind={kind}: {innerErr}" + ) + except Exception as warmErr: + logger.warning( + f"Session {sessionId}: Phrase pool warmup task crashed: {warmErr}" + ) + + +async def _runEscalationAndRelease( + service, + sessionId: str, + taskBrief: str, + briefingFileIds: List[str], + triggerTranscriptId: Optional[str], +) -> None: + """Background wrapper for ``_runAgentForMeeting`` that holds the + ``_agentEscalationInFlight`` flag for the duration of the agent run.""" + try: + await service._runAgentForMeeting( + sessionId=sessionId, + taskText=taskBrief, + fileIds=briefingFileIds, + sourceLabel="speechEscalation", + triggerTranscriptId=triggerTranscriptId, + ) + except asyncio.CancelledError: + logger.info( + f"Session {sessionId}: Escalation agent task cancelled by stop signal" + ) + except Exception as escErr: + logger.error( + f"Session {sessionId}: Escalation agent task failed: " + f"{type(escErr).__name__}: {escErr}", + exc_info=True, + ) + finally: + service._agentEscalationInFlight = False + service._currentEscalationTask = None diff --git a/modules/features/teamsbot/serviceWebSocket.py b/modules/features/teamsbot/serviceWebSocket.py new file mode 100644 index 00000000..2c462624 --- /dev/null +++ b/modules/features/teamsbot/serviceWebSocket.py @@ -0,0 +1,545 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Teamsbot Service — WebSocket handler & audio chunk processing. + +Extracted from service.py. All functions accept `service` (a TeamsbotService +instance) as the first parameter so the class can delegate to them. +""" + +import logging +import json +import asyncio +import time +import base64 +from typing import Optional, Dict, Any + +from fastapi import WebSocket + +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + + +async def handleBotWebSocket(service, websocket: WebSocket, sessionId: str): + """Main WebSocket handler for Browser Bot communication.""" + from . import interfaceFeatureTeamsbot as interfaceDb + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + from .service import _activeServices, _emitSessionEvent, sessionEvents + from .serviceConversation import _processTranscript, _warmEphemeralPhrasePool + + interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId) + voiceInterface = getVoiceInterface(service.currentUser, service.mandateId) + + session = interface.getSession(sessionId) + if session: + rawContext = session.get("sessionContext") + if rawContext and len(rawContext) > 500: + logger.info(f"Session {sessionId}: Summarizing long session context ({len(rawContext)} chars)...") + service._sessionContext = await service._summarizeSessionContext(sessionId, rawContext) + elif rawContext: + service._sessionContext = rawContext + if service._sessionContext: + logger.info(f"Session {sessionId}: Session context ready ({len(service._sessionContext)} chars)") + + try: + systemBot = interface.getActiveSystemBot(service.mandateId) + service._botAccountEmail = systemBot.get("email") if systemBot else None + if service._botAccountEmail: + logger.info(f"Session {sessionId}: Bot account email resolved: {service._botAccountEmail}") + except Exception: + service._botAccountEmail = None + + service._activeSessionId = sessionId + service._websocket = websocket + service._voiceInterface = voiceInterface + _activeServices[sessionId] = service + + try: + await _emitSessionEvent(sessionId, "botConnectionState", { + "connected": True, + "timestamp": getUtcTimestamp(), + }) + except Exception: + pass + + try: + service._activePersistentPrompts = interface.getActivePersistentPrompts(sessionId) or [] + if service._activePersistentPrompts: + logger.info( + f"Session {sessionId}: Loaded {len(service._activePersistentPrompts)} active persistent director prompt(s)" + ) + except Exception as restoreErr: + logger.warning(f"Session {sessionId}: Could not restore persistent director prompts: {restoreErr}") + service._activePersistentPrompts = [] + + asyncio.create_task(_warmEphemeralPhrasePool(service, sessionId)) + + logger.info(f"[WS] Handler started for session {sessionId}") + + try: + msgCount = 0 + while True: + data = await websocket.receive() + msgCount += 1 + + if "text" not in data: + logger.debug(f"[WS] session={sessionId} msg #{msgCount}: non-text data (keys: {list(data.keys())})") + continue + + message = json.loads(data["text"]) + msgType = message.get("type") + if msgType not in ("audioChunk", "ping"): + logger.info(f"[WS] session={sessionId} msg #{msgCount}: type={msgType}") + + if msgType == "transcript": + transcript = message.get("transcript", {}) + source = transcript.get("source", "caption") + speaker = transcript.get("speaker", "Unknown") + textPreview = (transcript.get("text", "") or "")[:60] + logger.info(f"[WS] Transcript (source={source}, speaker={speaker}): {textPreview}...") + await _processTranscript( + service, + sessionId=sessionId, + speaker=transcript.get("speaker", "Unknown"), + text=transcript.get("text", ""), + isFinal=transcript.get("isFinal", True), + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + source=source, + ) + + elif msgType == "chatMessage": + chat = message.get("chat", {}) + isHistory = chat.get("isHistory", False) + source = "chatHistory" if isHistory else "chat" + logger.info( + f"[WS] Chat{'[HISTORY]' if isHistory else ''}: " + f"speaker={chat.get('speaker')}, text={chat.get('text', '')[:60]}..." + ) + await _processTranscript( + service, + sessionId=sessionId, + speaker=chat.get("speaker", "Unknown"), + text=chat.get("text", ""), + isFinal=True, + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + source=source, + ) + + elif msgType == "status": + status = message.get("status") + errorMessage = message.get("message") + logger.info(f"[WS] Status: status={status}, message={errorMessage}") + await _handleBotStatus(service, sessionId, status, errorMessage, interface) + + elif msgType == "audioChunk": + audioData = message.get("audio", {}) + audioBase64 = audioData.get("data", "") + sampleRate = audioData.get("sampleRate", 16000) + captureDiagnostics = audioData.get("captureDiagnostics") or {} + if audioBase64: + await _processAudioChunk( + service, + sessionId=sessionId, + audioBase64=audioBase64, + sampleRate=sampleRate, + captureDiagnostics=captureDiagnostics, + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + ) + + elif msgType == "voiceGreeting": + greetingText = message.get("text", "") + greetingLang = message.get("language", service.config.language) + logger.info( + f"[WS] Voice greeting (legacy): text={greetingText[:60]}..., language={greetingLang}" + ) + if greetingText and voiceInterface: + await service._dispatchGreetingToMeeting( + sessionId=sessionId, + greetingText=greetingText, + greetingLang=greetingLang, + sendToChat=False, + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + ) + + elif msgType == "requestGreeting": + requestedLang = ( + message.get("language") or service.config.language or "" + ).strip() or "en-US" + botNameHint = ( + message.get("botName") or service.config.botName or "" + ).strip() or service.config.botName + logger.info( + f"[WS] Greeting request from bot: language={requestedLang}, name={botNameHint}" + ) + if voiceInterface: + try: + greetingText = await service._generateGreetingText( + requestedLang + ) + except Exception as genErr: + logger.warning( + f"Greeting generation failed for session {sessionId}: {genErr}" + ) + greetingText = "" + if greetingText: + await service._dispatchGreetingToMeeting( + sessionId=sessionId, + greetingText=greetingText, + greetingLang=requestedLang, + sendToChat=True, + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + ) + else: + logger.warning( + f"Session {sessionId}: Skipping greeting — AI generation produced no text" + ) + + elif msgType == "ping": + await websocket.send_text(json.dumps({"type": "pong"})) + + elif msgType == "ttsPlaybackAck": + playback = message.get("playback", {}) or {} + status = playback.get("status", "unknown") + ackMessage = playback.get("message") or "Bot playback status update" + logger.info( + f"[WS] TTS playback ack: status={status}, format={playback.get('format')}, " + f"bytesBase64={playback.get('bytesBase64')}" + ) + await _emitSessionEvent(sessionId, "ttsDeliveryStatus", { + "status": f"playback_{status}", + "hasWebSocket": True, + "message": ackMessage, + "timestamp": playback.get("timestamp") or getUtcTimestamp(), + "format": playback.get("format"), + "bytesBase64": playback.get("bytesBase64"), + }) + + elif msgType == "mfaChallenge": + mfaData = message.get("mfa", {}) + mfaType = mfaData.get("type", "unknown") + displayNumber = mfaData.get("displayNumber") + prompt = mfaData.get("prompt", "") + logger.info(f"[WS] MFA challenge: type={mfaType}, number={displayNumber}, prompt={prompt[:60]}") + + await _emitSessionEvent(sessionId, "mfaChallenge", { + "mfaType": mfaType, + "displayNumber": displayNumber, + "prompt": prompt, + "timestamp": getUtcTimestamp(), + }) + + from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks + mfaQueue = asyncio.Queue() + mfaCodeQueues[sessionId] = mfaQueue + + mfaWaitTasks[sessionId] = asyncio.create_task( + _waitAndForwardMfa(sessionId, mfaQueue, websocket) + ) + + elif msgType == "chatSendFailed": + errorData = message.get("error", {}) + reason = errorData.get("reason", "unknown") + failedText = errorData.get("text", "") + logger.warning( + f"[WS] Chat send failed for session {sessionId}: " + f"reason={reason}, text={failedText[:60]}" + ) + await _emitSessionEvent(sessionId, "chatSendFailed", { + "reason": reason, + "message": errorData.get("message", "Chat message could not be sent"), + "text": failedText, + "timestamp": getUtcTimestamp(), + }) + + elif msgType == "mfaResolved": + success = message.get("success", False) + logger.info(f"[WS] MFA resolved: success={success}") + from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks + task = mfaWaitTasks.pop(sessionId, None) + if task and not task.done(): + task.cancel() + mfaCodeQueues.pop(sessionId, None) + await _emitSessionEvent(sessionId, "mfaResolved", { + "success": success, + "timestamp": getUtcTimestamp(), + }) + + except Exception as e: + if "disconnect" not in str(e).lower(): + logger.error(f"[WS] Error for session {sessionId}: {type(e).__name__}: {e}") + finally: + if _activeServices.get(sessionId) is service: + _activeServices.pop(sessionId, None) + service._websocket = None + service._voiceInterface = None + service._activeSessionId = None + try: + await _emitSessionEvent(sessionId, "botConnectionState", { + "connected": False, + "timestamp": getUtcTimestamp(), + }) + except Exception: + pass + + logger.info(f"[WS] Handler ended for session {sessionId} after {msgCount} messages") + + +async def _waitAndForwardMfa(sid: str, queue: asyncio.Queue, ws: WebSocket): + """Wait for an MFA code from the operator and forward it to the bot.""" + from .service import _emitSessionEvent + from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks + + try: + mfaResponse = await asyncio.wait_for(queue.get(), timeout=120.0) + logger.info(f"[WS] MFA response received for session {sid}: action={mfaResponse.get('action')}") + await ws.send_text(json.dumps({ + "type": "mfaResponse", + "sessionId": sid, + "mfa": mfaResponse, + })) + except asyncio.TimeoutError: + logger.warning(f"[WS] MFA response timeout for session {sid}") + await ws.send_text(json.dumps({ + "type": "mfaResponse", + "sessionId": sid, + "mfa": {"action": "timeout"}, + })) + await _emitSessionEvent(sid, "mfaChallenge", { + "mfaType": "timeout", + "prompt": "MFA-Zeitlimit ueberschritten. Bitte erneut versuchen.", + }) + except asyncio.CancelledError: + logger.info(f"[WS] MFA wait cancelled for session {sid} (resolved via page)") + finally: + mfaCodeQueues.pop(sid, None) + mfaWaitTasks.pop(sid, None) + + +async def _handleBotStatus( + service, + sessionId: str, + status: str, + errorMessage: Optional[str], + interface, +): + """Handle status updates from the browser bot.""" + from .service import _emitSessionEvent + from .datamodelTeamsbot import TeamsbotSessionStatus + + logger.info(f"Bot status update for session {sessionId}: {status}") + + statusMap = { + "connecting": TeamsbotSessionStatus.JOINING.value, + "launching": TeamsbotSessionStatus.JOINING.value, + "navigating": TeamsbotSessionStatus.JOINING.value, + "in_lobby": TeamsbotSessionStatus.JOINING.value, + "joined": TeamsbotSessionStatus.ACTIVE.value, + "in_meeting": TeamsbotSessionStatus.ACTIVE.value, + "left": TeamsbotSessionStatus.ENDED.value, + "error": TeamsbotSessionStatus.ERROR.value, + } + + dbStatus = statusMap.get(status, TeamsbotSessionStatus.ACTIVE.value) + + updates = {"status": dbStatus} + if errorMessage: + updates["errorMessage"] = errorMessage + if dbStatus == TeamsbotSessionStatus.ACTIVE.value: + updates["startedAt"] = getUtcTimestamp() + elif dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]: + updates["endedAt"] = getUtcTimestamp() + + interface.updateSession(sessionId, updates) + await _emitSessionEvent(sessionId, "statusChange", {"status": status, "errorMessage": errorMessage}) + + if dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]: + if service._audioBuffer: + logger.info(f"[AudioChunk] Flushing remaining buffer on session end ({len(service._audioBuffer)} bytes)") + service._audioBuffer = b"" + service._audioBufferStartTime = 0.0 + service._audioBufferLastChunkTime = 0.0 + + if dbStatus == TeamsbotSessionStatus.ENDED.value: + asyncio.create_task(service._generateMeetingSummary(sessionId)) + + +async def _processAudioChunk( + service, + sessionId: str, + audioBase64: str, + sampleRate: int, + captureDiagnostics: Optional[Dict[str, Any]], + interface, + voiceInterface, + websocket: WebSocket, +): + """Process an audio chunk from WebRTC capture.""" + from .serviceConversation import _processTranscript + + _MIN_CHUNK_SEC = 1.0 + _STALE_TIMEOUT_SEC = 3.0 + + try: + audioBytes = base64.b64decode(audioBase64) + if len(audioBytes) < 500: + return + + if captureDiagnostics: + trackId = captureDiagnostics.get("trackId") + readyState = captureDiagnostics.get("readyState") + rms = captureDiagnostics.get("rms") + nativeSampleRate = captureDiagnostics.get("nativeSampleRate") + logger.debug( + f"[AudioChunk] diagnostics: track={trackId}, readyState={readyState}, " + f"rms={rms}, nativeRate={nativeSampleRate}, bytes={len(audioBytes)}" + ) + + isSilent = False + if captureDiagnostics and captureDiagnostics.get("rms") is not None: + try: + rmsVal = float(captureDiagnostics.get("rms")) + if rmsVal < 0.0003: + isSilent = True + except Exception: + pass + + if not voiceInterface: + logger.warning(f"[AudioChunk] No voice interface available for session {sessionId}") + return + + now = time.time() + effectiveRate = sampleRate if sampleRate and sampleRate > 0 else 16000 + + if not isSilent: + if not service._audioBuffer: + service._audioBufferStartTime = now + service._audioBuffer += audioBytes + service._audioBufferLastChunkTime = now + service._audioBufferSampleRate = effectiveRate + + bufferDuration = len(service._audioBuffer) / (effectiveRate * 2) if service._audioBuffer else 0.0 + bufferAge = (now - service._audioBufferStartTime) if service._audioBuffer else 0.0 + + shouldFlush = ( + service._audioBuffer + and ( + bufferDuration >= _MIN_CHUNK_SEC + or (bufferAge >= _STALE_TIMEOUT_SEC and bufferDuration > 0.3) + ) + ) + + if not shouldFlush: + return + + flushBytes = service._audioBuffer + flushRate = service._audioBufferSampleRate + service._audioBuffer = b"" + service._audioBufferStartTime = 0.0 + service._audioBufferLastChunkTime = 0.0 + + flushDuration = len(flushBytes) / (flushRate * 2) + logger.info(f"[AudioChunk] Flushing buffer: {len(flushBytes)} bytes, {flushDuration:.1f}s, {flushRate}Hz") + + phraseHints = list(service._knownSpeakers) + if service.config.botName: + phraseHints.append(service.config.botName) + + sttResult = await voiceInterface.speechToText( + audioContent=flushBytes, + language=service.config.language or "de-DE", + sampleRate=flushRate, + channels=1, + skipFallbacks=True, + phraseHints=phraseHints if phraseHints else None, + audioFormat="linear16", + ) + + if sttResult and sttResult.get("success") and sttResult.get("text"): + text = sttResult["text"].strip() + if text: + resolvedSpeaker = service._resolveSpeakerForAudioCapture() + fromCaption = resolvedSpeaker.get("speakerResolvedFromHint", False) + logger.info( + f"[AudioChunk] STT result: speaker={resolvedSpeaker.get('speaker', 'Meeting Audio')} " + f"(fromCaption={fromCaption}), text={text[:80]}..." + ) + await _processTranscript( + service, + sessionId=sessionId, + speaker=resolvedSpeaker["speaker"], + text=text, + isFinal=True, + interface=interface, + voiceInterface=voiceInterface, + websocket=websocket, + source="audioCapture", + speakerResolvedFromHint=resolvedSpeaker["speakerResolvedFromHint"], + ) + except Exception as e: + logger.error(f"[AudioChunk] STT error for session {sessionId}: {type(e).__name__}: {e}") + + +async def _cancelInFlightSpeech( + service, + sessionId: str, + websocket: Optional[WebSocket], + reason: str, +) -> None: + """Hard stop everything the bot is currently doing in the meeting.""" + from .service import _emitSessionEvent + + service._answerGenerationCounter += 1 + gen = service._answerGenerationCounter + logger.info( + f"Session {sessionId}: Cancelling in-flight speech " + f"(reason={reason}, gen={gen})" + ) + + if service._pendingNameTrigger: + logger.info( + f"Session {sessionId}: Dropping pending debounced name " + f"trigger (was queued before stop)" + ) + service._pendingNameTrigger = None + + for taskAttr in ("_currentEscalationTask", "_currentQuickAckTask"): + task = getattr(service, taskAttr, None) + if task is not None and not task.done(): + logger.info( + f"Session {sessionId}: Cancelling background task " + f"{taskAttr}" + ) + task.cancel() + + if websocket is not None: + try: + await websocket.send_text(json.dumps({ + "type": "stopAudio", + "sessionId": sessionId, + "reason": reason, + })) + except Exception as stopErr: + logger.warning( + f"Session {sessionId}: Failed to send stopAudio to " + f"browser bot: {stopErr}" + ) + + try: + await _emitSessionEvent(sessionId, "speechCancelled", { + "reason": reason, + "generation": gen, + "timestamp": getUtcTimestamp(), + }) + except Exception: + pass diff --git a/modules/features/trustee/handlerTrusteeAccounting.py b/modules/features/trustee/handlerTrusteeAccounting.py new file mode 100644 index 00000000..212d20e3 --- /dev/null +++ b/modules/features/trustee/handlerTrusteeAccounting.py @@ -0,0 +1,371 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Business logic for Trustee accounting integration endpoints. +Extracted from routeFeatureTrustee.py for maintainability. +""" + +import json +import logging +import time +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +_CONFIG_PLACEHOLDER = "***" + + +class SaveAccountingConfigBody(BaseModel): + """Request body for saving accounting config.""" + connectorType: str = "" + displayLabel: str = "" + config: Dict[str, Any] = Field(default_factory=dict, description="Connector credentials (e.g. clientName, apiKey)") + + +def getConfigMasked(connectorType: str, plainConfig: Dict[str, Any]) -> Dict[str, str]: + """Build config with secret values replaced by placeholder for GET response.""" + from .accounting.accountingRegistry import getAccountingRegistry + connector = getAccountingRegistry().getConnector(connectorType) + if not connector: + return {k: (v if isinstance(v, str) else str(v)) for k, v in (plainConfig or {}).items()} + secretKeys = {f.key for f in connector.getRequiredConfigFields() if f.secret} + return { + k: _CONFIG_PLACEHOLDER if k in secretKeys else (v if isinstance(v, str) else str(v) if v is not None else "") + for k, v in (plainConfig or {}).items() + } + + +async def refreshChartSilently(interface, instanceId: str) -> None: + """Best-effort chart-of-accounts cache refresh. Logs but does not raise on failure.""" + try: + from .accounting.accountingBridge import AccountingBridge + bridge = AccountingBridge(interface) + charts = await bridge.refreshChartOfAccounts(instanceId) + logger.info(f"Chart cache refreshed: {len(charts)} entries for instance {instanceId}") + except Exception as e: + logger.warning(f"Chart cache refresh failed (non-critical): {e}") + + +def readAccountingConfig(interface, instanceId: str) -> Dict[str, Any]: + """Read and return the masked accounting config for an instance.""" + from .datamodelFeatureTrustee import TrusteeAccountingConfig + from modules.shared.configuration import decryptValue + + records = interface.db.getRecordset( + TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True} + ) + if not records: + return {"configured": False} + + record = {k: v for k, v in records[0].items() if not k.startswith("_")} + encryptedConfig = record.pop("encryptedConfig", None) + record["configured"] = True + if encryptedConfig: + try: + plain = json.loads(decryptValue(encryptedConfig, keyName="accountingConfig")) + record["configMasked"] = getConfigMasked(record.get("connectorType", ""), plain) + except Exception: + record["configMasked"] = {} + else: + record["configMasked"] = {} + return record + + +async def saveAccountingConfig(interface, instanceId: str, mandateId: str, body: "SaveAccountingConfigBody") -> Dict[str, Any]: + """Save or update accounting config with encrypted credentials and config merging.""" + import uuid as _uuid + from .datamodelFeatureTrustee import TrusteeAccountingConfig + from modules.shared.configuration import encryptValue, decryptValue + + plainConfig = body.config if isinstance(body.config, dict) else {} + if not plainConfig and body.connectorType: + logger.warning("Accounting config save: config is empty (credentials will not be stored or updated)") + else: + logger.info( + "Accounting config save: instanceId=%s connectorType=%s configKeys=%s", + instanceId, body.connectorType, list(plainConfig.keys()) + ) + + existing = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId}) + if existing: + configId = existing[0].get("id") + updatePayload = { + "connectorType": body.connectorType or "", + "displayLabel": body.displayLabel or "", + "isActive": True, + } + if plainConfig: + existingEnc = existing[0].get("encryptedConfig") or "" + merged = {} + if existingEnc: + try: + merged = json.loads(decryptValue(existingEnc, keyName="accountingConfig")) + except Exception: + pass + for k, v in plainConfig.items(): + if v is not None and str(v).strip() and str(v).strip() != _CONFIG_PLACEHOLDER: + merged[k] = v + updatePayload["encryptedConfig"] = encryptValue(json.dumps(merged), keyName="accountingConfig") + interface.db.recordModify(TrusteeAccountingConfig, configId, updatePayload) + await refreshChartSilently(interface, instanceId) + return {"message": "Accounting config updated", "id": configId} + + if not plainConfig: + return None # Signal to route handler: raise 400 + + encryptedConfig = encryptValue(json.dumps(plainConfig), keyName="accountingConfig") + configRecord = { + "id": str(_uuid.uuid4()), + "featureInstanceId": instanceId, + "connectorType": body.connectorType or "", + "displayLabel": body.displayLabel or "", + "encryptedConfig": encryptedConfig, + "isActive": True, + "mandateId": mandateId, + } + interface.db.recordCreate(TrusteeAccountingConfig, configRecord) + await refreshChartSilently(interface, instanceId) + return {"message": "Accounting config created", "id": configRecord["id"]} + + +def getImportStatus(interface, instanceId: str) -> Dict[str, Any]: + """Get counts of imported TrusteeData* records for this instance.""" + from .datamodelFeatureTrustee import ( + TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, + TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig, + ) + filt = {"featureInstanceId": instanceId} + counts = { + "accounts": len(interface.db.getRecordset(TrusteeDataAccount, recordFilter=filt) or []), + "journalEntries": len(interface.db.getRecordset(TrusteeDataJournalEntry, recordFilter=filt) or []), + "journalLines": len(interface.db.getRecordset(TrusteeDataJournalLine, recordFilter=filt) or []), + "contacts": len(interface.db.getRecordset(TrusteeDataContact, recordFilter=filt) or []), + "accountBalances": len(interface.db.getRecordset(TrusteeDataAccountBalance, recordFilter=filt) or []), + } + cfgRecords = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True}) + if cfgRecords: + cfg = cfgRecords[0] + counts["lastSyncAt"] = cfg.get("lastSyncAt") + counts["lastSyncStatus"] = cfg.get("lastSyncStatus") + counts["lastSyncErrorMessage"] = cfg.get("lastSyncErrorMessage") + counts["lastSyncDateFrom"] = cfg.get("lastSyncDateFrom") + counts["lastSyncDateTo"] = cfg.get("lastSyncDateTo") + counts["lastSyncCounts"] = cfg.get("lastSyncCounts") + return counts + + +def wipeImportedData(interface, instanceId: str) -> Dict[str, Any]: + """Delete all TrusteeData* rows imported for this instance and reset sync markers.""" + from .datamodelFeatureTrustee import ( + TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, + TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig, + ) + from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache + + removed: Dict[str, int] = {} + for tableName, model in [ + ("accounts", TrusteeDataAccount), + ("journalEntries", TrusteeDataJournalEntry), + ("journalLines", TrusteeDataJournalLine), + ("contacts", TrusteeDataContact), + ("accountBalances", TrusteeDataAccountBalance), + ]: + try: + removed[tableName] = int(interface.db.recordDeleteWhere(model, {"featureInstanceId": instanceId}) or 0) + except Exception as ex: + logger.warning("wipeImportedData: failed for %s: %s", tableName, ex) + removed[tableName] = 0 + + cfgRecords = interface.db.getRecordset( + TrusteeAccountingConfig, + recordFilter={"featureInstanceId": instanceId, "isActive": True}, + ) + if cfgRecords: + cfgId = cfgRecords[0].get("id") + if cfgId: + try: + interface.db.recordModify(TrusteeAccountingConfig, cfgId, { + "lastSyncAt": None, + "lastSyncStatus": None, + "lastSyncErrorMessage": None, + "lastSyncDateFrom": None, + "lastSyncDateTo": None, + "lastSyncCounts": None, + }) + except Exception as ex: + logger.warning("wipeImportedData: failed to reset lastSync* on cfg %s: %s", cfgId, ex) + + cacheCleared = clearFeatureQueryCache(instanceId) + logger.info("wipeImportedData instance=%s removed=%s cacheCleared=%s", instanceId, removed, cacheCleared) + return { + "removed": removed, + "totalRemoved": sum(removed.values()), + "cacheCleared": cacheCleared, + "featureInstanceId": instanceId, + } + + +def exportAccountingData(interface, instanceId: str, mandateId: str) -> Dict[str, Any]: + """Build the export payload for all TrusteeData* tables for this instance.""" + from .datamodelFeatureTrustee import ( + TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, + TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig, + ) + _filter = {"featureInstanceId": instanceId} + + tables: Dict[str, Any] = {} + for tableName, model in [ + ("TrusteeDataAccount", TrusteeDataAccount), + ("TrusteeDataJournalEntry", TrusteeDataJournalEntry), + ("TrusteeDataJournalLine", TrusteeDataJournalLine), + ("TrusteeDataContact", TrusteeDataContact), + ("TrusteeDataAccountBalance", TrusteeDataAccountBalance), + ]: + records = interface.db.getRecordset(model, recordFilter=_filter) or [] + tables[tableName] = records + + cfgRecords = interface.db.getRecordset( + TrusteeAccountingConfig, + recordFilter={"featureInstanceId": instanceId, "isActive": True}, + ) + syncInfo = {} + if cfgRecords: + cfg = cfgRecords[0] + syncInfo = { + "connectorType": cfg.get("connectorType", ""), + "lastSyncAt": cfg.get("lastSyncAt"), + "lastSyncStatus": cfg.get("lastSyncStatus", ""), + } + + return { + "exportedAt": time.time(), + "featureInstanceId": instanceId, + "mandateId": mandateId, + "syncInfo": syncInfo, + "tables": tables, + } + + +# --------------------------------------------------------------------------- +# Background Job Handlers +# --------------------------------------------------------------------------- + +TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE = "trusteeAccountingPush" +TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE = "trusteeAccountingSync" + + +async def accountingPushJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]: + """BackgroundJob handler: pushes a batch of positions to the external accounting system.""" + from modules.security.rootAccess import getRootUser + from .accounting.accountingBridge import AccountingBridge, SyncResult + from .interfaceFeatureTrustee import getInterface + + instanceId = job["featureInstanceId"] + mandateId = job["mandateId"] + payload = job.get("payload") or {} + positionIds: List[str] = list(payload.get("positionIds") or []) + if not positionIds: + return {"total": 0, "success": 0, "skipped": 0, "errors": 0, "results": []} + + rootUser = getRootUser() + interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId) + bridge = AccountingBridge(interface) + + results = [] + total = len(positionIds) + progressCb( + 2, + messageKey="Sync wird vorbereitet ({total} Position(en))...", + messageParams={"total": total}, + ) + + try: + connector, plainConfig, configRecord = await bridge._resolveConnectorAndConfig(instanceId) + except Exception as resolveErr: + logger.exception("Accounting push: failed to resolve connector/config") + progressCb(100, messageKey="Verbindungsaufbau fehlgeschlagen.") + raise resolveErr + + if not connector or not plainConfig: + results = [SyncResult(success=False, errorMessage="No active accounting configuration found") for _ in positionIds] + progressCb(100, messageKey="Keine aktive Buchhaltungs-Konfiguration gefunden.") + return { + "total": len(results), + "success": 0, + "skipped": 0, + "errors": len(results), + "results": [r.model_dump() for r in results], + } + + for index, positionId in enumerate(positionIds, start=1): + result = await bridge.pushPositionToAccounting( + instanceId, + positionId, + _resolvedConnector=connector, + _resolvedPlainConfig=plainConfig, + _resolvedConfigRecord=configRecord, + ) + results.append(result) + pct = 5 + int(90 * index / total) + progressCb( + pct, + messageKey="Position {index}/{total} verarbeitet", + messageParams={"index": index, "total": total}, + ) + + skipped = [r for r in results if not r.success and r.errorMessage and "already synced" in r.errorMessage] + failed = [r for r in results if not r.success and r not in skipped] + if skipped: + logger.info("Accounting sync: %s position(s) already synced, skipped", len(skipped)) + if failed: + logger.warning( + "Accounting sync had %s failure(s): %s", + len(failed), + "; ".join(r.errorMessage or "unknown" for r in failed[:3]), + ) + + progressCb(100, messageKey="Sync abgeschlossen.") + return { + "total": len(results), + "success": sum(1 for r in results if r.success), + "skipped": len(skipped), + "errors": len(failed), + "results": [r.model_dump() for r in results], + } + + +async def accountingSyncJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]: + """BackgroundJob handler: imports accounting data from the external system.""" + from modules.security.rootAccess import getRootUser + from .accounting.accountingDataSync import AccountingDataSync + from .interfaceFeatureTrustee import getInterface + + instanceId = job["featureInstanceId"] + mandateId = job["mandateId"] + payload = job.get("payload") or {} + rootUser = getRootUser() + + progressCb(5, messageKey="Initialisiere Import...") + interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId) + sync = AccountingDataSync(interface) + progressCb(10, messageKey="Verbinde mit Buchhaltungssystem...") + result = await sync.importData( + featureInstanceId=instanceId, + mandateId=mandateId, + dateFrom=payload.get("dateFrom"), + dateTo=payload.get("dateTo"), + progressCb=progressCb, + ) + progressCb(100, messageKey="Import abgeschlossen.") + return result + + +# Register background job handlers +try: + from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler + registerJobHandler(TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE, accountingPushJobHandler) + registerJobHandler(TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, accountingSyncJobHandler) +except Exception as _regErr: + logger.warning("Failed to register accounting job handlers: %s", _regErr) diff --git a/modules/features/trustee/mainTrustee.py b/modules/features/trustee/mainTrustee.py index 7c686f3e..4bcee319 100644 --- a/modules/features/trustee/mainTrustee.py +++ b/modules/features/trustee/mainTrustee.py @@ -1097,3 +1097,27 @@ def onMandateDelete(mandateId: str, instances: list) -> None: except Exception as e: logger.warning(f"Failed to cascade-delete trustee data for mandate {mandateId}: {e}") + +def onUserDelete(userId: str, currentUser) -> dict: + """Delete/anonymize user data from the trustee database (GDPR).""" + from modules.system.gdprDeletion import deleteUserDataFromDatabase + from modules.connectors.connectorDbPostgre import DatabaseConnector + from modules.shared.configuration import APP_CONFIG + + dbName = "poweron_trustee" + try: + db = DatabaseConnector( + dbHost=APP_CONFIG.get("DB_HOST", "localhost"), + dbDatabase=dbName, + dbUser=APP_CONFIG.get("DB_USER"), + dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"), + dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), + userId=None, + ) + stats = deleteUserDataFromDatabase(db, userId, dbName) + db.close() + return stats + except Exception as e: + logger.warning(f"onUserDelete trustee failed: {e}") + return {"database": dbName, "tablesProcessed": 0, "recordsDeleted": 0, "recordsAnonymized": 0, "errors": [str(e)]} + diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 45a37ca3..8b2452af 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -1395,6 +1395,19 @@ def delete_position( # ===== Accounting Integration Endpoints ===== +from .handlerTrusteeAccounting import ( + SaveAccountingConfigBody, + readAccountingConfig, + saveAccountingConfig as _saveAccountingConfig, + refreshChartSilently as _refreshChartSilently, + getImportStatus as _getImportStatus, + wipeImportedData as _wipeImportedData, + exportAccountingData as _exportAccountingData, + TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE, + TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, +) + + @router.get("/{instanceId}/accounting/connectors") @limiter.limit("30/minute") def get_available_accounting_connectors( @@ -1408,23 +1421,6 @@ def get_available_accounting_connectors( return getAccountingRegistry().getAvailableConnectors() -# Placeholder returned for secret config fields so frontend can prefill form without sending real secrets. -_CONFIG_PLACEHOLDER = "***" - - -def _getConfigMasked(connectorType: str, plainConfig: Dict[str, Any]) -> Dict[str, str]: - """Build config with secret values replaced by placeholder for GET response.""" - from .accounting.accountingRegistry import getAccountingRegistry - connector = getAccountingRegistry().getConnector(connectorType) - if not connector: - return {k: (v if isinstance(v, str) else str(v)) for k, v in (plainConfig or {}).items()} - secretKeys = {f.key for f in connector.getRequiredConfigFields() if f.secret} - return { - k: _CONFIG_PLACEHOLDER if k in secretKeys else (v if isinstance(v, str) else str(v) if v is not None else "") - for k, v in (plainConfig or {}).items() - } - - @router.get("/{instanceId}/accounting/config") @limiter.limit("30/minute") def get_accounting_config( @@ -1432,33 +1428,10 @@ def get_accounting_config( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Get the active accounting config for this instance. Credentials are masked (secret fields = ***) for form prefill.""" + """Get the active accounting config for this instance. Credentials are masked.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import TrusteeAccountingConfig - from modules.shared.configuration import decryptValue - records = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True}) - if not records: - return {"configured": False} - record = {k: v for k, v in records[0].items() if not k.startswith("_")} - encryptedConfig = record.pop("encryptedConfig", None) - record["configured"] = True - if encryptedConfig: - try: - plain = json.loads(decryptValue(encryptedConfig, keyName="accountingConfig")) - record["configMasked"] = _getConfigMasked(record.get("connectorType", ""), plain) - except Exception: - record["configMasked"] = {} - else: - record["configMasked"] = {} - return record - - -class SaveAccountingConfigBody(BaseModel): - """Request body for saving accounting config. Ensures 'config' is present and used.""" - connectorType: str = "" - displayLabel: str = "" - config: Dict[str, Any] = Field(default_factory=dict, description="Connector credentials (e.g. clientName, apiKey)") + return readAccountingConfig(interface, instanceId) @router.post("/{instanceId}/accounting/config", status_code=201) @@ -1469,73 +1442,16 @@ async def save_accounting_config( body: SaveAccountingConfigBody = Body(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Save or update the accounting config for this instance. - - Body: { connectorType, displayLabel, config: { clientName, apiKey, ... } } - The 'config' object is stored encrypted; without it credentials would be empty in DB. - """ + """Save or update the accounting config for this instance.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - - from .datamodelFeatureTrustee import TrusteeAccountingConfig - from modules.shared.configuration import encryptValue - - plainConfig = body.config if isinstance(body.config, dict) else {} - # When updating, empty config is normal (frontend never receives credentials from GET). - # Do not overwrite encryptedConfig with empty – keep existing credentials. - if not plainConfig and body.connectorType: - logger.warning("Accounting config save: config is empty (credentials will not be stored or updated)") - else: - logger.info( - "Accounting config save: instanceId=%s connectorType=%s configKeys=%s", - instanceId, body.connectorType, list(plainConfig.keys()) - ) - - existing = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId}) - if existing: - configId = existing[0].get("id") - updatePayload = { - "connectorType": body.connectorType or "", - "displayLabel": body.displayLabel or "", - "isActive": True, - } - if plainConfig: - # Merge with existing: placeholder or empty = keep existing value (so form prefill does not overwrite secrets). - from modules.shared.configuration import decryptValue - existingEnc = existing[0].get("encryptedConfig") or "" - merged = {} - if existingEnc: - try: - merged = json.loads(decryptValue(existingEnc, keyName="accountingConfig")) - except Exception: - pass - for k, v in plainConfig.items(): - if v is not None and str(v).strip() and str(v).strip() != _CONFIG_PLACEHOLDER: - merged[k] = v - updatePayload["encryptedConfig"] = encryptValue(json.dumps(merged), keyName="accountingConfig") - interface.db.recordModify(TrusteeAccountingConfig, configId, updatePayload) - await _refreshChartSilently(interface, instanceId) - return {"message": "Accounting config updated", "id": configId} - - if not plainConfig: + result = await _saveAccountingConfig(interface, instanceId, mandateId, body) + if result is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=routeApiMsg("config is required for new integration (e.g. clientName, apiKey).") ) - encryptedConfig = encryptValue(json.dumps(plainConfig), keyName="accountingConfig") - - configRecord = { - "id": str(uuid.uuid4()), - "featureInstanceId": instanceId, - "connectorType": body.connectorType or "", - "displayLabel": body.displayLabel or "", - "encryptedConfig": encryptedConfig, - "isActive": True, - "mandateId": mandateId, - } - interface.db.recordCreate(TrusteeAccountingConfig, configRecord) - await _refreshChartSilently(interface, instanceId) - return {"message": "Accounting config created", "id": configRecord["id"]} + return result @router.post("/{instanceId}/accounting/test-connection") @@ -1545,7 +1461,7 @@ async def test_accounting_connection( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Test the connection to the configured accounting system. On success, refreshes the local chart-of-accounts cache.""" + """Test the connection to the configured accounting system.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) from .accounting.accountingBridge import AccountingBridge @@ -1581,7 +1497,7 @@ async def get_chart_of_accounts( accountType: Optional[str] = Query(None, description="Filter by type: expense, asset, liability, revenue"), context: RequestContext = Depends(getRequestContext) ) -> List[Dict[str, Any]]: - """Load the chart of accounts from the connected accounting system. Optional filter by accountType.""" + """Load the chart of accounts from the connected accounting system.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) from .accounting.accountingBridge import AccountingBridge @@ -1590,17 +1506,6 @@ async def get_chart_of_accounts( return [c.model_dump() for c in charts] -async def _refreshChartSilently(interface, instanceId: str) -> None: - """Best-effort chart-of-accounts cache refresh. Logs but does not raise on failure.""" - try: - from .accounting.accountingBridge import AccountingBridge - bridge = AccountingBridge(interface) - charts = await bridge.refreshChartOfAccounts(instanceId) - logger.info(f"Chart cache refreshed: {len(charts)} entries for instance {instanceId}") - except Exception as e: - logger.warning(f"Chart cache refresh failed (non-critical): {e}") - - @router.post("/{instanceId}/accounting/refresh-chart") @limiter.limit("5/minute") async def refresh_chart_of_accounts( @@ -1617,108 +1522,6 @@ async def refresh_chart_of_accounts( return {"message": f"Chart of accounts refreshed: {len(charts)} entries", "count": len(charts)} -TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE = "trusteeAccountingPush" - - -async def _trusteeAccountingPushJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]: - """BackgroundJob handler: pushes a batch of positions to the external - accounting system. Runs in the worker without blocking the original HTTP - request, so the user can continue navigating while the sync runs. - - Reads inputs from `job["payload"]` (`positionIds`) and reports incremental - progress via `progressCb(percent, message)`. The job result has the same - shape that the legacy synchronous endpoint used to return. - """ - from modules.security.rootAccess import getRootUser - from .accounting.accountingBridge import AccountingBridge - - instanceId = job["featureInstanceId"] - mandateId = job["mandateId"] - payload = job.get("payload") or {} - positionIds: List[str] = list(payload.get("positionIds") or []) - if not positionIds: - return {"total": 0, "success": 0, "skipped": 0, "errors": 0, "results": []} - - rootUser = getRootUser() - interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId) - bridge = AccountingBridge(interface) - - results = [] - total = len(positionIds) - progressCb( - 2, - messageKey="Sync wird vorbereitet ({total} Position(en))...", - messageParams={"total": total}, - ) - - # Resolve connector + plain config once to avoid decryption rate-limits - # (mirrors the optimisation in pushBatchToAccounting). We push positions - # one-by-one inside the job so we can emit incremental progress and so - # one bad row never aborts the rest. - from .accounting.accountingBridge import SyncResult - try: - connector, plainConfig, configRecord = await bridge._resolveConnectorAndConfig(instanceId) - except Exception as resolveErr: - logger.exception("Accounting push: failed to resolve connector/config") - progressCb(100, messageKey="Verbindungsaufbau fehlgeschlagen.") - raise resolveErr - - if not connector or not plainConfig: - results = [SyncResult(success=False, errorMessage="No active accounting configuration found") for _ in positionIds] - progressCb(100, messageKey="Keine aktive Buchhaltungs-Konfiguration gefunden.") - return { - "total": len(results), - "success": 0, - "skipped": 0, - "errors": len(results), - "results": [r.model_dump() for r in results], - } - - for index, positionId in enumerate(positionIds, start=1): - result = await bridge.pushPositionToAccounting( - instanceId, - positionId, - _resolvedConnector=connector, - _resolvedPlainConfig=plainConfig, - _resolvedConfigRecord=configRecord, - ) - results.append(result) - # Reserve 5..95% for the push loop, keep the tail for summary. - pct = 5 + int(90 * index / total) - progressCb( - pct, - messageKey="Position {index}/{total} verarbeitet", - messageParams={"index": index, "total": total}, - ) - - skipped = [r for r in results if not r.success and r.errorMessage and "already synced" in r.errorMessage] - failed = [r for r in results if not r.success and r not in skipped] - if skipped: - logger.info("Accounting sync: %s position(s) already synced, skipped", len(skipped)) - if failed: - logger.warning( - "Accounting sync had %s failure(s): %s", - len(failed), - "; ".join(r.errorMessage or "unknown" for r in failed[:3]), - ) - - progressCb(100, messageKey="Sync abgeschlossen.") - return { - "total": len(results), - "success": sum(1 for r in results if r.success), - "skipped": len(skipped), - "errors": len(failed), - "results": [r.model_dump() for r in results], - } - - -try: - from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler as _registerPushJobHandler - _registerPushJobHandler(TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE, _trusteeAccountingPushJobHandler) -except Exception as _pushRegErr: - logger.warning("Failed to register trusteeAccountingPush job handler: %s", _pushRegErr) - - @router.post("/{instanceId}/accounting/sync", status_code=status.HTTP_202_ACCEPTED) @limiter.limit("5/minute") async def sync_positions_to_accounting( @@ -1727,21 +1530,10 @@ async def sync_positions_to_accounting( data: Dict[str, Any] = Body(...), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Submit a background job that pushes positions to the accounting system. - - Body: ``{ positionIds: [...] }`` - - Returns ``{ jobId, status: "pending" }`` immediately so the user is not - blocked while the (potentially long) external accounting calls run. - Clients poll ``GET /api/jobs/{jobId}`` until status is ``SUCCESS`` / - ``ERROR`` and then read the same ``{ total, success, skipped, errors, - results }`` payload from ``job.result`` that the legacy synchronous - endpoint returned. - """ + """Submit a background job that pushes positions to the accounting system.""" from modules.serviceCenter.services.serviceBackgroundJobs import startJob mandateId = _validateInstanceAccess(instanceId, context) - positionIds = data.get("positionIds", []) if not positionIds: raise HTTPException(status_code=400, detail=routeApiMsg("positionIds required")) @@ -1771,11 +1563,7 @@ async def sync_single_position_to_accounting( bridge = AccountingBridge(interface) result = await bridge.pushPositionToAccounting(instanceId, positionId) if not result.success: - logger.warning( - "Accounting sync failed for positionId=%s: %s", - positionId, - result.errorMessage or "unknown", - ) + logger.warning("Accounting sync failed for positionId=%s: %s", positionId, result.errorMessage or "unknown") return result.model_dump() @@ -1791,8 +1579,7 @@ def get_sync_status( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) from .datamodelFeatureTrustee import TrusteeAccountingSync records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId}) - items = [{k: v for k, v in r.items() if not k.startswith("_")} for r in records] - return {"items": items} + return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]} @router.get("/{instanceId}/accounting/sync-status/{positionId}") @@ -1808,52 +1595,11 @@ def get_position_sync_status( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) from .datamodelFeatureTrustee import TrusteeAccountingSync records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"positionId": positionId, "featureInstanceId": instanceId}) - items = [{k: v for k, v in r.items() if not k.startswith("_")} for r in records] - return {"items": items} + return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]} # ===== Accounting Data Import ===== -TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE = "trusteeAccountingSync" - - -async def _trusteeAccountingSyncJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]: - """BackgroundJob handler: imports accounting data from the external system. - - Reads inputs from `job["payload"]` (dateFrom, dateTo, userId) and runs - `AccountingDataSync.importData(...)` in the worker's event loop without - blocking the original HTTP request that submitted the job. - """ - from modules.security.rootAccess import getRootUser - from .accounting.accountingDataSync import AccountingDataSync - - instanceId = job["featureInstanceId"] - mandateId = job["mandateId"] - payload = job.get("payload") or {} - rootUser = getRootUser() - - progressCb(5, messageKey="Initialisiere Import...") - interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId) - sync = AccountingDataSync(interface) - progressCb(10, messageKey="Verbinde mit Buchhaltungssystem...") - result = await sync.importData( - featureInstanceId=instanceId, - mandateId=mandateId, - dateFrom=payload.get("dateFrom"), - dateTo=payload.get("dateTo"), - progressCb=progressCb, - ) - progressCb(100, messageKey="Import abgeschlossen.") - return result - - -try: - from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler - registerJobHandler(TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, _trusteeAccountingSyncJobHandler) -except Exception as _regErr: - logger.warning("Failed to register trusteeAccountingSync job handler: %s", _regErr) - - @router.post("/{instanceId}/accounting/import-data", status_code=status.HTTP_202_ACCEPTED) @limiter.limit("3/minute") async def import_accounting_data( @@ -1862,18 +1608,11 @@ async def import_accounting_data( data: Dict[str, Any] = Body(default={}), context: RequestContext = Depends(getRequestContext) ) -> Dict[str, Any]: - """Submit a background job to import accounting data. - - Returns immediately with `{ jobId }`; clients poll `GET /api/jobs/{jobId}` - until status is SUCCESS / ERROR. - """ + """Submit a background job to import accounting data.""" from modules.serviceCenter.services.serviceBackgroundJobs import startJob mandateId = _validateInstanceAccess(instanceId, context) - payload = { - "dateFrom": data.get("dateFrom"), - "dateTo": data.get("dateTo"), - } + payload = {"dateFrom": data.get("dateFrom"), "dateTo": data.get("dateTo")} jobId = await startJob( TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, payload, @@ -1894,28 +1633,7 @@ def get_import_status( """Get counts of imported TrusteeData* records for this instance.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import ( - TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, - TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig, - ) - filt = {"featureInstanceId": instanceId} - counts = { - "accounts": len(interface.db.getRecordset(TrusteeDataAccount, recordFilter=filt) or []), - "journalEntries": len(interface.db.getRecordset(TrusteeDataJournalEntry, recordFilter=filt) or []), - "journalLines": len(interface.db.getRecordset(TrusteeDataJournalLine, recordFilter=filt) or []), - "contacts": len(interface.db.getRecordset(TrusteeDataContact, recordFilter=filt) or []), - "accountBalances": len(interface.db.getRecordset(TrusteeDataAccountBalance, recordFilter=filt) or []), - } - cfgRecords = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True}) - if cfgRecords: - cfg = cfgRecords[0] - counts["lastSyncAt"] = cfg.get("lastSyncAt") - counts["lastSyncStatus"] = cfg.get("lastSyncStatus") - counts["lastSyncErrorMessage"] = cfg.get("lastSyncErrorMessage") - counts["lastSyncDateFrom"] = cfg.get("lastSyncDateFrom") - counts["lastSyncDateTo"] = cfg.get("lastSyncDateTo") - counts["lastSyncCounts"] = cfg.get("lastSyncCounts") - return counts + return _getImportStatus(interface, instanceId) # ===== AI Data Cache ===== @@ -1927,12 +1645,7 @@ def clear_ai_data_cache( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext), ) -> Dict[str, Any]: - """Clear ONLY the AI feature-data query result cache (in-memory, ~5 min TTL). - - Important: this does NOT touch the synchronised ``TrusteeData*`` tables. - The synced rows (chart of accounts, journal entries/lines, contacts, balances) - stay exactly as imported. To wipe those rows, use POST .../wipe-imported-data. - """ + """Clear the AI feature-data query result cache (in-memory).""" _validateInstanceAccess(instanceId, context) from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache removed = clearFeatureQueryCache(instanceId) @@ -1946,66 +1659,10 @@ def wipe_imported_accounting_data( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext), ) -> Dict[str, Any]: - """Delete every ``TrusteeData*`` row imported for this feature instance. - - Use when the source system was changed, test data needs to be cleared, or - the user suspects stale rows from earlier connector versions. Also resets - the ``lastSync*`` markers on the active config so the UI no longer reports - a stale "letzter Import" status. The connector configuration / credentials - remain untouched -- only synchronised payload data is removed. - """ + """Delete all TrusteeData* rows imported for this feature instance and reset sync markers.""" mandateId = _validateInstanceAccess(instanceId, context) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - from .datamodelFeatureTrustee import ( - TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, - TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig, - ) - from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache - - removed: Dict[str, int] = {} - for tableName, model in [ - ("accounts", TrusteeDataAccount), - ("journalEntries", TrusteeDataJournalEntry), - ("journalLines", TrusteeDataJournalLine), - ("contacts", TrusteeDataContact), - ("accountBalances", TrusteeDataAccountBalance), - ]: - try: - removed[tableName] = int(interface.db.recordDeleteWhere(model, {"featureInstanceId": instanceId}) or 0) - except Exception as ex: - logger.warning("wipeImportedData: failed for %s: %s", tableName, ex) - removed[tableName] = 0 - - cfgRecords = interface.db.getRecordset( - TrusteeAccountingConfig, - recordFilter={"featureInstanceId": instanceId, "isActive": True}, - ) - if cfgRecords: - cfgId = cfgRecords[0].get("id") - if cfgId: - try: - interface.db.recordModify(TrusteeAccountingConfig, cfgId, { - "lastSyncAt": None, - "lastSyncStatus": None, - "lastSyncErrorMessage": None, - "lastSyncDateFrom": None, - "lastSyncDateTo": None, - "lastSyncCounts": None, - }) - except Exception as ex: - logger.warning("wipeImportedData: failed to reset lastSync* on cfg %s: %s", cfgId, ex) - - cacheCleared = clearFeatureQueryCache(instanceId) - logger.info( - "wipeImportedData instance=%s removed=%s cacheCleared=%s", - instanceId, removed, cacheCleared, - ) - return { - "removed": removed, - "totalRemoved": sum(removed.values()), - "cacheCleared": cacheCleared, - "featureInstanceId": instanceId, - } + return _wipeImportedData(interface, instanceId) # ===== Data Export ===== @@ -2017,52 +1674,10 @@ def export_accounting_data( instanceId: str = Path(..., description="Feature Instance ID"), context: RequestContext = Depends(getRequestContext), ) -> Response: - """Export all TrusteeData* tables for this instance as a JSON download (admin only).""" + """Export all TrusteeData* tables for this instance as a JSON download.""" mandateId = _validateInstanceAccess(instanceId, context) - - from .datamodelFeatureTrustee import ( - TrusteeDataAccount, - TrusteeDataJournalEntry, - TrusteeDataJournalLine, - TrusteeDataContact, - TrusteeDataAccountBalance, - TrusteeAccountingConfig, - ) interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) - _filter = {"featureInstanceId": instanceId} - - tables: Dict[str, Any] = {} - for tableName, model in [ - ("TrusteeDataAccount", TrusteeDataAccount), - ("TrusteeDataJournalEntry", TrusteeDataJournalEntry), - ("TrusteeDataJournalLine", TrusteeDataJournalLine), - ("TrusteeDataContact", TrusteeDataContact), - ("TrusteeDataAccountBalance", TrusteeDataAccountBalance), - ]: - records = interface.db.getRecordset(model, recordFilter=_filter) or [] - tables[tableName] = records - - cfgRecords = interface.db.getRecordset( - TrusteeAccountingConfig, - recordFilter={"featureInstanceId": instanceId, "isActive": True}, - ) - syncInfo = {} - if cfgRecords: - cfg = cfgRecords[0] - syncInfo = { - "connectorType": cfg.get("connectorType", ""), - "lastSyncAt": cfg.get("lastSyncAt"), - "lastSyncStatus": cfg.get("lastSyncStatus", ""), - } - - payload = { - "exportedAt": time.time(), - "featureInstanceId": instanceId, - "mandateId": mandateId, - "syncInfo": syncInfo, - "tables": tables, - } - + payload = _exportAccountingData(interface, instanceId, mandateId) jsonBytes = json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8") return Response( content=jsonBytes, diff --git a/modules/features/trustee/workflows/__init__.py b/modules/features/trustee/workflows/__init__.py new file mode 100644 index 00000000..976edabd --- /dev/null +++ b/modules/features/trustee/workflows/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Trustee feature-owned workflow methods.""" diff --git a/modules/workflows/methods/methodTrustee/__init__.py b/modules/features/trustee/workflows/methodTrustee/__init__.py similarity index 100% rename from modules/workflows/methods/methodTrustee/__init__.py rename to modules/features/trustee/workflows/methodTrustee/__init__.py diff --git a/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py b/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py similarity index 99% rename from modules/workflows/methods/methodTrustee/actions/extractFromFiles.py rename to modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py index fa677e2b..d28c8a3c 100644 --- a/modules/workflows/methods/methodTrustee/actions/extractFromFiles.py +++ b/modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py @@ -19,7 +19,7 @@ from typing import Dict, Any, List, Optional, Tuple from modules.datamodels.datamodelChat import ActionResult, ActionDocument, ChatDocument, ChatMessage from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum -from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodTrustee/actions/processDocuments.py b/modules/features/trustee/workflows/methodTrustee/actions/processDocuments.py similarity index 100% rename from modules/workflows/methods/methodTrustee/actions/processDocuments.py rename to modules/features/trustee/workflows/methodTrustee/actions/processDocuments.py diff --git a/modules/workflows/methods/methodTrustee/actions/queryData.py b/modules/features/trustee/workflows/methodTrustee/actions/queryData.py similarity index 100% rename from modules/workflows/methods/methodTrustee/actions/queryData.py rename to modules/features/trustee/workflows/methodTrustee/actions/queryData.py diff --git a/modules/workflows/methods/methodTrustee/actions/refreshAccountingData.py b/modules/features/trustee/workflows/methodTrustee/actions/refreshAccountingData.py similarity index 100% rename from modules/workflows/methods/methodTrustee/actions/refreshAccountingData.py rename to modules/features/trustee/workflows/methodTrustee/actions/refreshAccountingData.py diff --git a/modules/workflows/methods/methodTrustee/actions/syncToAccounting.py b/modules/features/trustee/workflows/methodTrustee/actions/syncToAccounting.py similarity index 100% rename from modules/workflows/methods/methodTrustee/actions/syncToAccounting.py rename to modules/features/trustee/workflows/methodTrustee/actions/syncToAccounting.py diff --git a/modules/workflows/methods/methodTrustee/methodTrustee.py b/modules/features/trustee/workflows/methodTrustee/methodTrustee.py similarity index 100% rename from modules/workflows/methods/methodTrustee/methodTrustee.py rename to modules/features/trustee/workflows/methodTrustee/methodTrustee.py diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 287e60df..5ac5d089 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -111,7 +111,7 @@ def initBootstrap(db: DatabaseConnector) -> None: logger.warning(f"Mandate retention purge failed: {e}") # Let features run their own bootstrap logic via lifecycle hooks - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules for _fCode, _fMod in loadFeatureMainModules().items(): _bootHook = getattr(_fMod, "onBootstrap", None) if _bootHook: @@ -172,7 +172,7 @@ def initRootMandateFeatures(db: DatabaseConnector, mandateId: str) -> None: """ from modules.datamodels.datamodelFeatures import FeatureInstance from modules.interfaces.interfaceFeatures import getFeatureInterface - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules logger.info("Initializing root mandate features") @@ -241,7 +241,7 @@ def initRootMandateFeatures(db: DatabaseConnector, mandateId: str) -> None: def _cleanupRemovedFeatureInstances(db: DatabaseConnector) -> None: """Remove feature instances whose featureCode no longer exists in the codebase.""" from modules.datamodels.datamodelFeatures import FeatureInstance - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules mainModules = loadFeatureMainModules() activeCodes = set() @@ -1144,7 +1144,7 @@ def _createUiContextRules(db: DatabaseConnector) -> None: Args: db: Database connector instance """ - from modules.system.mainSystem import NAVIGATION_SECTIONS + from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS uiRules = [] adminId = _getRoleId(db, "admin") @@ -1200,7 +1200,7 @@ def _ensureUiContextRules(db: DatabaseConnector) -> None: Args: db: Database connector instance """ - from modules.system.mainSystem import NAVIGATION_SECTIONS + from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS # Template role IDs adminId = _getRoleId(db, "admin") diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 60aac0e8..6ebadaaf 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -1579,7 +1579,7 @@ class AppObjects: from modules.datamodels.datamodelFeatures import FeatureInstance from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate from modules.interfaces.interfaceFeatures import getFeatureInterface - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules plan = BUILTIN_PLANS.get(planKey) if not plan: raise ValueError(f"Unknown plan: {planKey}") @@ -1871,7 +1871,7 @@ class AppObjects: instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId}) # 0-pre. Let features cascade-delete their own data via lifecycle hooks - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules for _fCode, _fMod in loadFeatureMainModules().items(): _hook = getattr(_fMod, "onMandateDelete", None) if _hook: diff --git a/modules/interfaces/interfaceDbSubscription.py b/modules/interfaces/interfaceDbSubscription.py index bdcaeb2b..b6cb26ff 100644 --- a/modules/interfaces/interfaceDbSubscription.py +++ b/modules/interfaces/interfaceDbSubscription.py @@ -30,7 +30,7 @@ from modules.datamodels.datamodelSubscription import ( getEffectiveLimits, ) -from modules.shared.serviceExceptions import SubscriptionCapacityException +from modules.datamodels.serviceExceptions import SubscriptionCapacityException logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py index 4c1b29b8..5f239c01 100644 --- a/modules/interfaces/interfaceFeatures.py +++ b/modules/interfaces/interfaceFeatures.py @@ -287,7 +287,7 @@ class FeatureInterface: RuntimeError: If templates exist but cannot be copied. Caller decides whether to swallow or re-raise. """ - from modules.system.registry import loadFeatureMainModules + from modules.shared.featureDiscovery import loadFeatureMainModules mainModules = loadFeatureMainModules() featureModule = mainModules.get(featureCode) if not featureModule: diff --git a/modules/routes/billingWebhookHandler.py b/modules/routes/billingWebhookHandler.py new file mode 100644 index 00000000..ecfe37b4 --- /dev/null +++ b/modules/routes/billingWebhookHandler.py @@ -0,0 +1,399 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Stripe webhook and subscription business logic for billing. +Extracted from routeBilling.py for maintainability. +""" + +import logging +from datetime import datetime, timezone +from typing import Any, Dict, Optional + +from fastapi import HTTPException + +from modules.datamodels.datamodelBilling import ( + BillingTransaction, + TransactionTypeEnum, + ReferenceTypeEnum, +) +from modules.shared.i18nRegistry import apiRouteContext + +routeApiMsg = apiRouteContext("routeBilling") +logger = logging.getLogger(__name__) + + +def creditStripeSessionIfNeeded( + billingInterface, + session: Dict[str, Any], + eventId: Optional[str] = None, + CheckoutConfirmResponse=None, +): + """Credit balance from Stripe Checkout session if not already credited. + Uses Checkout session ID for idempotency across webhook + manual confirmation flows. + """ + from modules.serviceCenter.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF + + session_id = session.get("id") + metadata = session.get("metadata") or {} + mandate_id = metadata.get("mandateId") + user_id = metadata.get("userId") or None + amount_chf_str = metadata.get("amountChf", "0") + + if not session_id: + raise HTTPException(status_code=400, detail=routeApiMsg("Stripe session id missing")) + if not mandate_id: + raise HTTPException(status_code=400, detail=routeApiMsg("Invalid session metadata: mandateId missing")) + + existing_payment_tx = billingInterface.getPaymentTransactionByReferenceId(session_id) + if existing_payment_tx: + if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId): + billingInterface.createStripeWebhookEvent(eventId) + return CheckoutConfirmResponse( + credited=False, + alreadyCredited=True, + sessionId=session_id, + mandateId=mandate_id, + amountChf=float(existing_payment_tx.get("amount", 0.0)), + ) + + try: + amount_chf = float(amount_chf_str) + except (TypeError, ValueError): + amount_chf = None + + if amount_chf is None or amount_chf not in ALLOWED_AMOUNTS_CHF: + amount_total = session.get("amount_total") + if amount_total is not None: + amount_chf = amount_total / 100.0 + else: + raise HTTPException(status_code=400, detail=routeApiMsg("Invalid amount in Stripe session")) + + settings = billingInterface.getSettings(mandate_id) + if not settings: + raise HTTPException(status_code=404, detail=routeApiMsg("Billing settings not found")) + + account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0) + + transaction = BillingTransaction( + accountId=account["id"], + transactionType=TransactionTypeEnum.CREDIT, + amount=amount_chf, + description="Stripe-Zahlung", + referenceType=ReferenceTypeEnum.PAYMENT, + referenceId=session_id, + createdByUserId=user_id, + ) + billingInterface.createTransaction(transaction) + + if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId): + billingInterface.createStripeWebhookEvent(eventId) + + logger.info(f"Stripe credit applied: {amount_chf} CHF for session {session_id} on mandate {mandate_id}") + return CheckoutConfirmResponse( + credited=True, + alreadyCredited=False, + sessionId=session_id, + mandateId=mandate_id, + amountChf=amount_chf, + ) + + +def handleSubscriptionCheckoutCompleted(session, eventId: str, getRootInterface) -> None: + """Handle checkout.session.completed for mode=subscription. + Resolves the local PENDING record by ID from webhook metadata and transitions it.""" + from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface + from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan + from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import ( + getService as getSubscriptionService, + _notifySubscriptionChange, + ) + from modules.security.rootAccess import getRootUser + + if not isinstance(session, dict): + from modules.shared.stripeClient import stripeToDict + session = stripeToDict(session) + + metadata = session.get("metadata") or {} + subscriptionRecordId = metadata.get("subscriptionRecordId") + mandateId = metadata.get("mandateId") + planKey = metadata.get("planKey", "") + platformUrl = metadata.get("platformUrl", "") + + if not subscriptionRecordId: + stripeSub = session.get("subscription") + if stripeSub: + try: + from modules.shared.stripeClient import getStripeClient + stripe = getStripeClient() + from modules.shared.stripeClient import stripeToDict + subObj = stripeToDict(stripe.Subscription.retrieve(stripeSub)) + metadata = subObj.get("metadata") or {} + subscriptionRecordId = metadata.get("subscriptionRecordId") + mandateId = metadata.get("mandateId") + planKey = metadata.get("planKey", "") + platformUrl = platformUrl or metadata.get("platformUrl", "") + except Exception as e: + logger.error( + "Stripe Subscription.retrieve(%s) failed during checkout " + "metadata recovery: %s", stripeSub, e, + ) + raise + + stripeSubId = session.get("subscription") + + if not mandateId or not subscriptionRecordId: + logger.warning("Subscription checkout missing metadata: %s", metadata) + return + + subInterface = getSubRootInterface() + rootUser = getRootUser() + + sub = subInterface.getById(subscriptionRecordId) + if not sub: + logger.error("Subscription record %s not found for checkout webhook", subscriptionRecordId) + return + if sub.get("status") != SubscriptionStatusEnum.PENDING.value: + logger.warning("Subscription %s is %s, expected PENDING — skipping", subscriptionRecordId, sub.get("status")) + return + + stripeData: Dict[str, Any] = {} + if stripeSubId: + stripeData["stripeSubscriptionId"] = stripeSubId + try: + from modules.shared.stripeClient import getStripeClient + stripe = getStripeClient() + from modules.shared.stripeClient import stripeToDict + stripeSub = stripeToDict(stripe.Subscription.retrieve(stripeSubId, expand=["items"])) + + if stripeSub.get("current_period_start"): + stripeData["currentPeriodStart"] = float(stripeSub["current_period_start"]) + if stripeSub.get("current_period_end"): + stripeData["currentPeriodEnd"] = float(stripeSub["current_period_end"]) + + from modules.serviceCenter.services.serviceSubscription.stripeBootstrap import getStripePricesForPlan + priceMapping = getStripePricesForPlan(planKey) + items = stripeSub.get("items") or {} + if not isinstance(items, dict): + items = dict(items) + for item in items.get("data", []): + priceId = (item.get("price") or {}).get("id", "") + if priceMapping and priceId == priceMapping.stripePriceIdUsers: + stripeData["stripeItemIdUsers"] = item["id"] + elif priceMapping and priceId == priceMapping.stripePriceIdInstances: + stripeData["stripeItemIdInstances"] = item["id"] + except Exception as e: + logger.error( + "Error retrieving Stripe subscription %s during checkout " + "completion (will be retried by Stripe): %s", + stripeSubId, e, + ) + raise + + if stripeData: + subInterface.updateFields(subscriptionRecordId, stripeData) + + operative = subInterface.getOperativeForMandate(mandateId) + hasActivePredecessor = operative is not None and operative["id"] != subscriptionRecordId + predecessorIsTrial = ( + hasActivePredecessor + and operative.get("status") == SubscriptionStatusEnum.TRIALING.value + ) + + if hasActivePredecessor and predecessorIsTrial: + try: + subInterface.forceExpire(operative["id"]) + logger.info( + "Trial subscription %s expired immediately for mandate %s due to paid upgrade %s", + operative["id"], mandateId, subscriptionRecordId, + ) + except Exception as e: + logger.error("Failed to expire trial predecessor %s: %s", operative["id"], e) + toStatus = SubscriptionStatusEnum.ACTIVE + elif hasActivePredecessor: + toStatus = SubscriptionStatusEnum.SCHEDULED + if operative.get("recurring", True): + operativeStripeId = operative.get("stripeSubscriptionId") + if operativeStripeId: + try: + from modules.shared.stripeClient import getStripeClient + stripe = getStripeClient() + stripe.Subscription.modify(operativeStripeId, cancel_at_period_end=True) + except Exception as e: + logger.error("Failed to set cancel_at_period_end on predecessor %s: %s", operativeStripeId, e) + subInterface.updateFields(operative["id"], {"recurring": False}) + effectiveFrom = operative.get("currentPeriodEnd") + if effectiveFrom: + subInterface.updateFields(subscriptionRecordId, {"effectiveFrom": effectiveFrom}) + else: + toStatus = SubscriptionStatusEnum.ACTIVE + + try: + subInterface.transitionStatus( + subscriptionRecordId, SubscriptionStatusEnum.PENDING, toStatus, + {"recurring": True}, + ) + except Exception as e: + logger.error("Failed to transition subscription %s: %s", subscriptionRecordId, e) + return + + subService = getSubscriptionService(rootUser, mandateId) + subService.invalidateCache(mandateId) + + if toStatus == SubscriptionStatusEnum.ACTIVE: + plan = getPlan(planKey) + updatedSub = subInterface.getById(subscriptionRecordId) + _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=updatedSub, platformUrl=platformUrl) + + try: + billingIf = getRootInterface() + billingIf.creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") + except Exception as ex: + logger.error("creditSubscriptionBudget on activation failed: %s", ex) + + logger.info( + "Checkout completed: sub=%s -> %s, mandate=%s, plan=%s", + subscriptionRecordId, toStatus.value, mandateId, planKey, + ) + + +def handleSubscriptionWebhook(event, getRootInterface) -> None: + """Process Stripe subscription webhook events. + All record resolution is by stripeSubscriptionId.""" + from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface + from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan + from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import ( + getService as getSubscriptionService, + _notifySubscriptionChange, + ) + from modules.security.rootAccess import getRootUser + + obj = event.data.object + rawSub = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription") + stripeSubId = rawSub.get("id") if isinstance(rawSub, dict) else rawSub + if not stripeSubId: + logger.warning("Subscription webhook %s has no subscription ID", event.type) + return + + subInterface = getSubRootInterface() + sub = subInterface.getByStripeSubscriptionId(stripeSubId) + if not sub: + logger.warning("No local record for Stripe subscription %s (event: %s)", stripeSubId, event.type) + return + + subId = sub["id"] + mandateId = sub["mandateId"] + currentStatus = SubscriptionStatusEnum(sub["status"]) + rootUser = getRootUser() + subService = getSubscriptionService(rootUser, mandateId) + + subMetadata = obj.get("metadata") or {} + webhookPlatformUrl = subMetadata.get("platformUrl", "") + + if event.type == "customer.subscription.updated": + stripeStatus = obj.get("status", "") + + periodData: Dict[str, Any] = {} + if obj.get("current_period_start"): + periodData["currentPeriodStart"] = float(obj["current_period_start"]) + if obj.get("current_period_end"): + periodData["currentPeriodEnd"] = float(obj["current_period_end"]) + if periodData: + subInterface.updateFields(subId, periodData) + + if stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.SCHEDULED: + subInterface.transitionStatus(subId, SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE) + subService.invalidateCache(mandateId) + planKey = sub.get("planKey", "") + plan = getPlan(planKey) + refreshedSub = subInterface.getById(subId) + _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedSub, platformUrl=webhookPlatformUrl) + try: + getRootInterface().creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") + except Exception as ex: + logger.error("creditSubscriptionBudget SCHEDULED->ACTIVE failed: %s", ex) + logger.info("SCHEDULED -> ACTIVE for sub %s (mandate %s)", subId, mandateId) + + elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.PAST_DUE: + subInterface.transitionStatus(subId, SubscriptionStatusEnum.PAST_DUE, SubscriptionStatusEnum.ACTIVE) + subService.invalidateCache(mandateId) + logger.info("PAST_DUE -> ACTIVE for sub %s (mandate %s)", subId, mandateId) + + elif stripeStatus == "past_due" and currentStatus == SubscriptionStatusEnum.ACTIVE: + subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE) + subService.invalidateCache(mandateId) + logger.info("ACTIVE -> PAST_DUE for sub %s (mandate %s)", subId, mandateId) + + elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.ACTIVE: + subService.invalidateCache(mandateId) + logger.info("Period renewed for sub %s (mandate %s)", subId, mandateId) + + elif event.type == "customer.subscription.deleted": + if currentStatus not in (SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE, + SubscriptionStatusEnum.SCHEDULED): + logger.info("Ignoring deletion for sub %s in status %s", subId, currentStatus.value) + return + + subInterface.transitionStatus(subId, currentStatus, SubscriptionStatusEnum.EXPIRED) + subService.invalidateCache(mandateId) + logger.info("Sub %s -> EXPIRED (Stripe deleted, mandate %s)", subId, mandateId) + + scheduled = subInterface.getScheduledForMandate(mandateId) + if scheduled: + try: + subInterface.transitionStatus( + scheduled["id"], SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE, + ) + subService.invalidateCache(mandateId) + plan = getPlan(scheduled.get("planKey", "")) + refreshedScheduled = subInterface.getById(scheduled["id"]) + _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedScheduled, platformUrl=webhookPlatformUrl) + logger.info("Promoted SCHEDULED sub %s -> ACTIVE (mandate %s)", scheduled["id"], mandateId) + except Exception as e: + logger.error("Failed to promote SCHEDULED sub %s: %s", scheduled["id"], e) + + elif event.type == "invoice.payment_failed": + if currentStatus == SubscriptionStatusEnum.ACTIVE: + subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE) + subService.invalidateCache(mandateId) + plan = getPlan(sub.get("planKey", "")) + _notifySubscriptionChange(mandateId, "payment_failed", plan, subscriptionRecord=sub, platformUrl=webhookPlatformUrl) + logger.info("Payment failed for sub %s (mandate %s)", subId, mandateId) + + elif event.type == "customer.subscription.trial_will_end": + logger.info("Trial ending soon for sub %s (mandate %s)", subId, mandateId) + try: + from modules.system.notifyMandateAdmins import notifyMandateAdmins + notifyMandateAdmins( + mandateId, + "[PowerOn] Testphase endet bald", + "Testphase endet bald", + [ + "Die kostenlose Testphase für Ihren Mandanten endet in Kürze.", + "Bitte wählen Sie einen Plan unter Billing-Verwaltung › Abonnement.", + ], + ) + except Exception as e: + logger.error("Failed to notify about trial ending: %s", e) + + elif event.type == "invoice.paid": + period_ts = obj.get("period_start") + periodLabel = "" + if period_ts: + period_start_at = datetime.fromtimestamp(int(period_ts), tz=timezone.utc) + periodLabel = period_start_at.strftime("%Y-%m-%d") + try: + billing_if = getRootInterface() + billing_if.resetStorageBillingPeriod(mandateId, period_start_at) + billing_if.reconcileMandateStorageBilling(mandateId) + except Exception as ex: + logger.error("Storage billing on invoice.paid failed: %s", ex) + + planKey = sub.get("planKey", "") + try: + billing_if = getRootInterface() + billing_if.creditSubscriptionBudget(mandateId, planKey, periodLabel=periodLabel or "Periodenverlängerung") + except Exception as ex: + logger.error("creditSubscriptionBudget on invoice.paid failed: %s", ex) + + logger.info("Invoice paid for sub %s (mandate %s)", subId, mandateId) + return None diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index 19341394..058038c0 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -330,80 +330,10 @@ def _getStripeClient(): return getStripeClient() -def _creditStripeSessionIfNeeded( - billingInterface, - session: Dict[str, Any], - eventId: Optional[str] = None, -) -> CheckoutConfirmResponse: - """ - Credit balance from Stripe Checkout session if not already credited. - Uses Checkout session ID for idempotency across webhook + manual confirmation flows. - """ - from modules.serviceCenter.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF - - session_id = session.get("id") - metadata = session.get("metadata") or {} - mandate_id = metadata.get("mandateId") - user_id = metadata.get("userId") or None - amount_chf_str = metadata.get("amountChf", "0") - - if not session_id: - raise HTTPException(status_code=400, detail=routeApiMsg("Stripe session id missing")) - if not mandate_id: - raise HTTPException(status_code=400, detail=routeApiMsg("Invalid session metadata: mandateId missing")) - - existing_payment_tx = billingInterface.getPaymentTransactionByReferenceId(session_id) - if existing_payment_tx: - if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId): - billingInterface.createStripeWebhookEvent(eventId) - return CheckoutConfirmResponse( - credited=False, - alreadyCredited=True, - sessionId=session_id, - mandateId=mandate_id, - amountChf=float(existing_payment_tx.get("amount", 0.0)), - ) - - try: - amount_chf = float(amount_chf_str) - except (TypeError, ValueError): - amount_chf = None - - if amount_chf is None or amount_chf not in ALLOWED_AMOUNTS_CHF: - amount_total = session.get("amount_total") - if amount_total is not None: - amount_chf = amount_total / 100.0 - else: - raise HTTPException(status_code=400, detail=routeApiMsg("Invalid amount in Stripe session")) - - settings = billingInterface.getSettings(mandate_id) - if not settings: - raise HTTPException(status_code=404, detail=routeApiMsg("Billing settings not found")) - - account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0) - - transaction = BillingTransaction( - accountId=account["id"], - transactionType=TransactionTypeEnum.CREDIT, - amount=amount_chf, - description="Stripe-Zahlung", - referenceType=ReferenceTypeEnum.PAYMENT, - referenceId=session_id, - createdByUserId=user_id, - ) - billingInterface.createTransaction(transaction) - - if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId): - billingInterface.createStripeWebhookEvent(eventId) - - logger.info(f"Stripe credit applied: {amount_chf} CHF for session {session_id} on mandate {mandate_id}") - return CheckoutConfirmResponse( - credited=True, - alreadyCredited=False, - sessionId=session_id, - mandateId=mandate_id, - amountChf=amount_chf, - ) +def _creditStripeSessionIfNeeded(billingInterface, session: Dict[str, Any], eventId: Optional[str] = None) -> CheckoutConfirmResponse: + """Credit balance from Stripe Checkout session if not already credited.""" + from .billingWebhookHandler import creditStripeSessionIfNeeded + return creditStripeSessionIfNeeded(billingInterface, session, eventId, CheckoutConfirmResponse) # ============================================================================= @@ -1148,314 +1078,15 @@ async def stripeWebhook( def handleSubscriptionCheckoutCompleted(session, eventId: str) -> None: - """Handle checkout.session.completed for mode=subscription. - Resolves the local PENDING record by ID from webhook metadata and transitions it.""" - from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface - from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan - from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import ( - getService as getSubscriptionService, - _notifySubscriptionChange, - ) - from modules.security.rootAccess import getRootUser - - if not isinstance(session, dict): - from modules.shared.stripeClient import stripeToDict - session = stripeToDict(session) - - metadata = session.get("metadata") or {} - subscriptionRecordId = metadata.get("subscriptionRecordId") - mandateId = metadata.get("mandateId") - planKey = metadata.get("planKey", "") - - platformUrl = metadata.get("platformUrl", "") - - if not subscriptionRecordId: - stripeSub = session.get("subscription") - if stripeSub: - try: - from modules.shared.stripeClient import getStripeClient - stripe = getStripeClient() - from modules.shared.stripeClient import stripeToDict - subObj = stripeToDict(stripe.Subscription.retrieve(stripeSub)) - metadata = subObj.get("metadata") or {} - subscriptionRecordId = metadata.get("subscriptionRecordId") - mandateId = metadata.get("mandateId") - planKey = metadata.get("planKey", "") - platformUrl = platformUrl or metadata.get("platformUrl", "") - except Exception as e: - # Stripe lookup is the only way to recover the metadata at this - # point — if it fails we MUST surface it, otherwise the webhook - # later short-circuits with "missing metadata" and the user - # silently gets stuck in PENDING. - logger.error( - "Stripe Subscription.retrieve(%s) failed during checkout " - "metadata recovery: %s", stripeSub, e, - ) - raise - - stripeSubId = session.get("subscription") - - if not mandateId or not subscriptionRecordId: - logger.warning("Subscription checkout missing metadata: %s", metadata) - return - - subInterface = getSubRootInterface() - rootUser = getRootUser() - - sub = subInterface.getById(subscriptionRecordId) - if not sub: - logger.error("Subscription record %s not found for checkout webhook", subscriptionRecordId) - return - if sub.get("status") != SubscriptionStatusEnum.PENDING.value: - logger.warning("Subscription %s is %s, expected PENDING — skipping", subscriptionRecordId, sub.get("status")) - return - - stripeData: Dict[str, Any] = {} - if stripeSubId: - stripeData["stripeSubscriptionId"] = stripeSubId - try: - from modules.shared.stripeClient import getStripeClient - stripe = getStripeClient() - from modules.shared.stripeClient import stripeToDict - stripeSub = stripeToDict(stripe.Subscription.retrieve(stripeSubId, expand=["items"])) - - if stripeSub.get("current_period_start"): - stripeData["currentPeriodStart"] = float(stripeSub["current_period_start"]) - if stripeSub.get("current_period_end"): - stripeData["currentPeriodEnd"] = float(stripeSub["current_period_end"]) - - from modules.serviceCenter.services.serviceSubscription.stripeBootstrap import getStripePricesForPlan - priceMapping = getStripePricesForPlan(planKey) - items = stripeSub.get("items") or {} - if not isinstance(items, dict): - items = dict(items) - for item in items.get("data", []): - priceId = (item.get("price") or {}).get("id", "") - if priceMapping and priceId == priceMapping.stripePriceIdUsers: - stripeData["stripeItemIdUsers"] = item["id"] - elif priceMapping and priceId == priceMapping.stripePriceIdInstances: - stripeData["stripeItemIdInstances"] = item["id"] - except Exception as e: - # Without these enrichment fields the activation completes anyway - # (status flips to ACTIVE/SCHEDULED below), but periods + Stripe - # item-IDs are missing on the local record, which breaks later - # add-on billing and renewal accounting. Re-raise so the webhook - # is retried by Stripe instead of silently shipping a broken row. - logger.error( - "Error retrieving Stripe subscription %s during checkout " - "completion (will be retried by Stripe): %s", - stripeSubId, e, - ) - raise - - if stripeData: - subInterface.updateFields(subscriptionRecordId, stripeData) - - operative = subInterface.getOperativeForMandate(mandateId) - hasActivePredecessor = operative is not None and operative["id"] != subscriptionRecordId - predecessorIsTrial = ( - hasActivePredecessor - and operative.get("status") == SubscriptionStatusEnum.TRIALING.value - ) - - if hasActivePredecessor and predecessorIsTrial: - try: - subInterface.forceExpire(operative["id"]) - logger.info( - "Trial subscription %s expired immediately for mandate %s due to paid upgrade %s", - operative["id"], mandateId, subscriptionRecordId, - ) - except Exception as e: - logger.error("Failed to expire trial predecessor %s: %s", operative["id"], e) - toStatus = SubscriptionStatusEnum.ACTIVE - elif hasActivePredecessor: - toStatus = SubscriptionStatusEnum.SCHEDULED - if operative.get("recurring", True): - operativeStripeId = operative.get("stripeSubscriptionId") - if operativeStripeId: - try: - from modules.shared.stripeClient import getStripeClient - stripe = getStripeClient() - stripe.Subscription.modify(operativeStripeId, cancel_at_period_end=True) - except Exception as e: - logger.error("Failed to set cancel_at_period_end on predecessor %s: %s", operativeStripeId, e) - subInterface.updateFields(operative["id"], {"recurring": False}) - effectiveFrom = operative.get("currentPeriodEnd") - if effectiveFrom: - subInterface.updateFields(subscriptionRecordId, {"effectiveFrom": effectiveFrom}) - else: - toStatus = SubscriptionStatusEnum.ACTIVE - - try: - subInterface.transitionStatus( - subscriptionRecordId, SubscriptionStatusEnum.PENDING, toStatus, - {"recurring": True}, - ) - except Exception as e: - logger.error("Failed to transition subscription %s: %s", subscriptionRecordId, e) - return - - subService = getSubscriptionService(rootUser, mandateId) - subService.invalidateCache(mandateId) - - if toStatus == SubscriptionStatusEnum.ACTIVE: - plan = getPlan(planKey) - updatedSub = subInterface.getById(subscriptionRecordId) - _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=updatedSub, platformUrl=platformUrl) - - try: - billingIf = getRootInterface() - billingIf.creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") - except Exception as ex: - logger.error("creditSubscriptionBudget on activation failed: %s", ex) - - logger.info( - "Checkout completed: sub=%s -> %s, mandate=%s, plan=%s", - subscriptionRecordId, toStatus.value, mandateId, planKey, - ) + """Handle checkout.session.completed for mode=subscription.""" + from .billingWebhookHandler import handleSubscriptionCheckoutCompleted as _handler + _handler(session, eventId, getRootInterface) def _handleSubscriptionWebhook(event) -> None: - """Process Stripe subscription webhook events. - All record resolution is by stripeSubscriptionId — no mandate-based guessing.""" - from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface - from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan - from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import ( - getService as getSubscriptionService, - _notifySubscriptionChange, - ) - from modules.security.rootAccess import getRootUser - - obj = event.data.object - rawSub = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription") - stripeSubId = rawSub.get("id") if isinstance(rawSub, dict) else rawSub - if not stripeSubId: - logger.warning("Subscription webhook %s has no subscription ID", event.type) - return - - subInterface = getSubRootInterface() - sub = subInterface.getByStripeSubscriptionId(stripeSubId) - if not sub: - logger.warning("No local record for Stripe subscription %s (event: %s)", stripeSubId, event.type) - return - - subId = sub["id"] - mandateId = sub["mandateId"] - currentStatus = SubscriptionStatusEnum(sub["status"]) - rootUser = getRootUser() - subService = getSubscriptionService(rootUser, mandateId) - - subMetadata = obj.get("metadata") or {} - webhookPlatformUrl = subMetadata.get("platformUrl", "") - - if event.type == "customer.subscription.updated": - stripeStatus = obj.get("status", "") - - periodData: Dict[str, Any] = {} - if obj.get("current_period_start"): - periodData["currentPeriodStart"] = float(obj["current_period_start"]) - if obj.get("current_period_end"): - periodData["currentPeriodEnd"] = float(obj["current_period_end"]) - if periodData: - subInterface.updateFields(subId, periodData) - - if stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.SCHEDULED: - subInterface.transitionStatus(subId, SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE) - subService.invalidateCache(mandateId) - planKey = sub.get("planKey", "") - plan = getPlan(planKey) - refreshedSub = subInterface.getById(subId) - _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedSub, platformUrl=webhookPlatformUrl) - try: - getRootInterface().creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung") - except Exception as ex: - logger.error("creditSubscriptionBudget SCHEDULED->ACTIVE failed: %s", ex) - logger.info("SCHEDULED -> ACTIVE for sub %s (mandate %s)", subId, mandateId) - - elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.PAST_DUE: - subInterface.transitionStatus(subId, SubscriptionStatusEnum.PAST_DUE, SubscriptionStatusEnum.ACTIVE) - subService.invalidateCache(mandateId) - logger.info("PAST_DUE -> ACTIVE for sub %s (mandate %s)", subId, mandateId) - - elif stripeStatus == "past_due" and currentStatus == SubscriptionStatusEnum.ACTIVE: - subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE) - subService.invalidateCache(mandateId) - logger.info("ACTIVE -> PAST_DUE for sub %s (mandate %s)", subId, mandateId) - - elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.ACTIVE: - subService.invalidateCache(mandateId) - logger.info("Period renewed for sub %s (mandate %s)", subId, mandateId) - - elif event.type == "customer.subscription.deleted": - if currentStatus not in (SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE, - SubscriptionStatusEnum.SCHEDULED): - logger.info("Ignoring deletion for sub %s in status %s", subId, currentStatus.value) - return - - subInterface.transitionStatus(subId, currentStatus, SubscriptionStatusEnum.EXPIRED) - subService.invalidateCache(mandateId) - logger.info("Sub %s -> EXPIRED (Stripe deleted, mandate %s)", subId, mandateId) - - scheduled = subInterface.getScheduledForMandate(mandateId) - if scheduled: - try: - subInterface.transitionStatus( - scheduled["id"], SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE, - ) - subService.invalidateCache(mandateId) - plan = getPlan(scheduled.get("planKey", "")) - refreshedScheduled = subInterface.getById(scheduled["id"]) - _notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedScheduled, platformUrl=webhookPlatformUrl) - logger.info("Promoted SCHEDULED sub %s -> ACTIVE (mandate %s)", scheduled["id"], mandateId) - except Exception as e: - logger.error("Failed to promote SCHEDULED sub %s: %s", scheduled["id"], e) - - elif event.type == "invoice.payment_failed": - if currentStatus == SubscriptionStatusEnum.ACTIVE: - subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE) - subService.invalidateCache(mandateId) - plan = getPlan(sub.get("planKey", "")) - _notifySubscriptionChange(mandateId, "payment_failed", plan, subscriptionRecord=sub, platformUrl=webhookPlatformUrl) - logger.info("Payment failed for sub %s (mandate %s)", subId, mandateId) - - elif event.type == "customer.subscription.trial_will_end": - logger.info("Trial ending soon for sub %s (mandate %s)", subId, mandateId) - try: - from modules.system.notifyMandateAdmins import notifyMandateAdmins - notifyMandateAdmins( - mandateId, - "[PowerOn] Testphase endet bald", - "Testphase endet bald", - [ - "Die kostenlose Testphase für Ihren Mandanten endet in Kürze.", - "Bitte wählen Sie einen Plan unter Billing-Verwaltung › Abonnement.", - ], - ) - except Exception as e: - logger.error("Failed to notify about trial ending: %s", e) - - elif event.type == "invoice.paid": - period_ts = obj.get("period_start") - periodLabel = "" - if period_ts: - period_start_at = datetime.fromtimestamp(int(period_ts), tz=timezone.utc) - periodLabel = period_start_at.strftime("%Y-%m-%d") - try: - billing_if = getRootInterface() - billing_if.resetStorageBillingPeriod(mandateId, period_start_at) - billing_if.reconcileMandateStorageBilling(mandateId) - except Exception as ex: - logger.error("Storage billing on invoice.paid failed: %s", ex) - - planKey = sub.get("planKey", "") - try: - billing_if = getRootInterface() - billing_if.creditSubscriptionBudget(mandateId, planKey, periodLabel=periodLabel or "Periodenverlängerung") - except Exception as ex: - logger.error("creditSubscriptionBudget on invoice.paid failed: %s", ex) - - logger.info("Invoice paid for sub %s (mandate %s)", subId, mandateId) - return None + """Process Stripe subscription webhook events.""" + from .billingWebhookHandler import handleSubscriptionWebhook as _handler + _handler(event, getRootInterface) @router.get("/admin/accounts/{targetMandateId}", response_model=List[AccountSummary]) diff --git a/modules/routes/routeClickup.py b/modules/routes/routeClickup.py index 7a869a9f..a6c6745d 100644 --- a/modules/routes/routeClickup.py +++ b/modules/routes/routeClickup.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request, sta from modules.auth import getCurrentUser, limiter from modules.datamodels.datamodelUam import AuthAuthority, User, UserConnection from modules.interfaces.interfaceDbApp import getInterface -from modules.serviceHub import getInterface as getServices +from modules.serviceCenter.serviceHub import getInterface as getServices from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeClickup") diff --git a/modules/routes/routeSharepoint.py b/modules/routes/routeSharepoint.py index 1ee21900..b144328e 100644 --- a/modules/routes/routeSharepoint.py +++ b/modules/routes/routeSharepoint.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Query, Request, sta from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User, UserConnection from modules.interfaces.interfaceDbApp import getInterface -from modules.serviceHub import getInterface as getServices +from modules.serviceCenter.serviceHub import getInterface as getServices from modules.shared.i18nRegistry import apiRouteContext routeApiMsg = apiRouteContext("routeSharepoint") diff --git a/modules/security/rootAccess.py b/modules/security/rootAccess.py index d87c22e8..1735891d 100644 --- a/modules/security/rootAccess.py +++ b/modules/security/rootAccess.py @@ -4,7 +4,7 @@ Root access management for system-level operations. Provides secure access to root user and DbApp database connector. -Bei leerer Datenbank wird automatisch Bootstrap ausgeführt. +Bootstrap is guaranteed by app.py lifespan before any access. """ import logging @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) _rootDbAppConnector = None _rootUser = None -_bootstrapExecuted = False + def getRootDbAppConnector() -> DatabaseConnector: """ @@ -39,34 +39,12 @@ def getRootDbAppConnector() -> DatabaseConnector: return _rootDbAppConnector -def _ensureBootstrap(): - """ - Führt Bootstrap aus, falls noch nicht geschehen. - Wird automatisch aufgerufen, wenn getRootUser() keinen User findet. - """ - global _bootstrapExecuted - - if _bootstrapExecuted: - return - - logger.info("Running bootstrap to initialize database") - - # Import here to avoid circular imports - from modules.interfaces.interfaceBootstrap import initBootstrap - - dbApp = getRootDbAppConnector() - initBootstrap(dbApp) - - _bootstrapExecuted = True - logger.info("Bootstrap completed") - - def getRootUser() -> User: """ Returns the root user (initial user from database). Used for system-level operations that require root privileges. - - Falls kein User existiert, wird Bootstrap automatisch ausgeführt. + + Raises RuntimeError if no user exists (bootstrap incomplete). """ global _rootUser @@ -74,19 +52,15 @@ def getRootUser() -> User: dbApp = getRootDbAppConnector() initialUserId = dbApp.getInitialId(UserInDB) - # Wenn kein User existiert, Bootstrap ausführen if not initialUserId: - logger.info("No initial user found, running bootstrap") - _ensureBootstrap() - - # Nochmal versuchen nach Bootstrap - initialUserId = dbApp.getInitialId(UserInDB) - if not initialUserId: - raise ValueError("No initial user ID found in database after bootstrap") + raise RuntimeError( + "No root user found - bootstrap incomplete. " + "Ensure app.py lifespan runs initBootstrap before any service access." + ) users = dbApp.getRecordset(UserInDB, recordFilter={"id": initialUserId}) if not users: - raise ValueError("Initial user not found in database") + raise RuntimeError("Initial user not found in database") user_data = users[0] _rootUser = User(**user_data) diff --git a/modules/serviceCenter/core/serviceStreaming/eventManager.py b/modules/serviceCenter/core/serviceStreaming/eventManager.py index 180430eb..823dbda1 100644 --- a/modules/serviceCenter/core/serviceStreaming/eventManager.py +++ b/modules/serviceCenter/core/serviceStreaming/eventManager.py @@ -1,222 +1,8 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -""" -Event manager for SSE streaming. -Manages event queues for Server-Sent Events (SSE) streaming across features. -""" +"""Re-export shim — canonical source is modules.shared.eventManager.""" -import logging -import asyncio -from typing import Dict, Optional, Any - -logger = logging.getLogger(__name__) - - -class EventManager: - """ - Manages event queues for SSE streaming. - Each workflow has its own async queue for events. - """ - - def __init__(self): - """Initialize the event manager.""" - self._queues: Dict[str, asyncio.Queue] = {} - self._cleanup_tasks: Dict[str, asyncio.Task] = {} - self._agent_tasks: Dict[str, asyncio.Task] = {} - self._cancelled: Dict[str, bool] = {} - - def create_queue(self, workflow_id: str) -> asyncio.Queue: - """ - Create an event queue for a workflow. - - Args: - workflow_id: Workflow ID - - Returns: - Async queue for events - """ - if workflow_id in self._cleanup_tasks: - self._cleanup_tasks[workflow_id].cancel() - del self._cleanup_tasks[workflow_id] - logger.debug(f"Cancelled pending cleanup for workflow {workflow_id}") - - if workflow_id not in self._queues: - self._queues[workflow_id] = asyncio.Queue() - logger.debug(f"Created event queue for workflow {workflow_id}") - else: - old = self._queues[workflow_id] - while not old.empty(): - try: - old.get_nowait() - except asyncio.QueueEmpty: - break - logger.debug(f"Reusing event queue for workflow {workflow_id} (drained stale events)") - return self._queues[workflow_id] - - def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]: - """ - Get the event queue for a workflow. - - Args: - workflow_id: Workflow ID - - Returns: - Async queue if exists, None otherwise - """ - return self._queues.get(workflow_id) - - def has_queue(self, workflow_id: str) -> bool: - """ - Check if a queue exists for a workflow. - - Args: - workflow_id: Workflow ID - - Returns: - True if queue exists, False otherwise - """ - return workflow_id in self._queues - - def register_agent_task(self, workflow_id: str, task: asyncio.Task) -> None: - """Register the asyncio Task running the agent for a workflow.""" - self._agent_tasks[workflow_id] = task - self._cancelled.pop(workflow_id, None) - - def is_cancelled(self, workflow_id: str) -> bool: - """Check if a workflow has been cancelled.""" - return self._cancelled.get(workflow_id, False) - - async def cancel_agent(self, workflow_id: str) -> bool: - """Cancel the running agent task for a workflow. Returns True if cancelled.""" - self._cancelled[workflow_id] = True - task = self._agent_tasks.pop(workflow_id, None) - if task and not task.done(): - task.cancel() - logger.info(f"Cancelled agent task for workflow {workflow_id}") - return True - logger.debug(f"No running agent task found for workflow {workflow_id}") - return False - - def _unregister_agent_task(self, workflow_id: str) -> None: - """Remove the agent task reference after completion.""" - self._agent_tasks.pop(workflow_id, None) - self._cancelled.pop(workflow_id, None) - - async def emit_event( - self, - context_id: str, - event_type: str, - data: Dict[str, Any], - event_category: str = "chat", - message: Optional[str] = None, - step: Optional[str] = None - ) -> None: - """ - Emit an event to the queue for a workflow. - - Args: - context_id: Workflow ID (context) - event_type: Type of event (e.g., "chatdata", "complete", "error") - data: Event data dictionary - event_category: Category of event (e.g., "chat", "workflow") - message: Optional message string - step: Optional step identifier - """ - queue = self._queues.get(context_id) - if not queue: - # DEBUG level: This is normal for background workflows without active SSE listener - return - - event = { - "type": event_type, - "data": data, - "category": event_category, - "message": message, - "step": step - } - - try: - await queue.put(event) - if event_type not in ("chunk",): - logger.debug(f"Emitted {event_type} event for workflow {context_id}") - except Exception as e: - logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True) - - async def cleanup(self, workflow_id: str, delay: float = 60.0) -> None: - """ - Schedule cleanup of a queue after a delay. - - Args: - workflow_id: Workflow ID - delay: Delay in seconds before cleanup - """ - # Cancel existing cleanup task if any - if workflow_id in self._cleanup_tasks: - self._cleanup_tasks[workflow_id].cancel() - - async def _cleanup(): - try: - await asyncio.sleep(delay) - if workflow_id in self._queues: - # Drain remaining events - queue = self._queues[workflow_id] - while not queue.empty(): - try: - queue.get_nowait() - except asyncio.QueueEmpty: - break - - # Remove queue - del self._queues[workflow_id] - logger.info(f"Cleaned up event queue for workflow {workflow_id}") - except asyncio.CancelledError: - logger.debug(f"Cleanup cancelled for workflow {workflow_id}") - except Exception as e: - logger.error(f"Error during cleanup for workflow {workflow_id}: {e}", exc_info=True) - finally: - if workflow_id in self._cleanup_tasks: - del self._cleanup_tasks[workflow_id] - - # Schedule cleanup - task = asyncio.create_task(_cleanup()) - self._cleanup_tasks[workflow_id] = task - - def shutdown(self) -> None: - """Cancel all pending cleanup and agent tasks for fast process exit. - - Injects ``None`` sentinels into every live queue so that SSE generators - (which block on ``queue.get()``) break out of their loop immediately - instead of waiting up to the keepalive timeout. - """ - for _wfId, q in list(self._queues.items()): - try: - q.put_nowait(None) - except Exception: - pass - for wfId, task in list(self._cleanup_tasks.items()): - if not task.done(): - task.cancel() - self._cleanup_tasks.clear() - for wfId, task in list(self._agent_tasks.items()): - if not task.done(): - task.cancel() - self._agent_tasks.clear() - self._queues.clear() - logger.info("EventManager shutdown: all tasks cancelled, queues drained") - - -# Global event manager instance -_event_manager: Optional[EventManager] = None - - -def get_event_manager() -> EventManager: - """ - Get the global event manager instance. - - Returns: - EventManager instance - """ - global _event_manager - if _event_manager is None: - _event_manager = EventManager() - return _event_manager +from modules.shared.eventManager import ( # noqa: F401 + EventManager, + get_event_manager, +) diff --git a/modules/serviceCenter/serviceHub.py b/modules/serviceCenter/serviceHub.py new file mode 100644 index 00000000..a42f8d0e --- /dev/null +++ b/modules/serviceCenter/serviceHub.py @@ -0,0 +1,189 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Service Hub. +Consumer-facing aggregation layer for services, DB interfaces, and runtime state. + +Architecture: +- serviceHub delegates service resolution to serviceCenter (DI container) +- serviceHub owns DB interface initialization and runtime state +- serviceCenter knows nothing about serviceHub (one-way dependency) + +Import-Regelwerk: +- Zentrale Module (wie dieses) duerfen KEINE Feature-Container importieren +- Feature-spezifische Services werden dynamisch geladen +- Shared Services werden via serviceCenter resolved +""" + +import os +import importlib +import glob +from typing import Any, Optional, TYPE_CHECKING +import logging + +from modules.datamodels.datamodelUam import User + +if TYPE_CHECKING: + from modules.datamodels.datamodelChat import ChatWorkflow + +logger = logging.getLogger(__name__) + +_FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features") + + +class PublicService: + """Lightweight proxy exposing only public callable attributes of a target.""" + + def __init__(self, target: Any, functionsOnly: bool = True, nameFilter=None): + self._target = target + self._functionsOnly = functionsOnly + self._nameFilter = nameFilter + + def __getattr__(self, name: str): + if name.startswith('_'): + raise AttributeError(f"'{type(self._target).__name__}' attribute '{name}' is private") + if self._nameFilter and not self._nameFilter(name): + raise AttributeError(f"'{name}' not exposed by policy") + attr = getattr(self._target, name) + if self._functionsOnly and not callable(attr): + raise AttributeError(f"'{name}' is not a function") + return attr + + def __dir__(self): + return sorted([ + n for n in dir(self._target) + if not n.startswith('_') + and (not self._functionsOnly or callable(getattr(self._target, n, None))) + and (self._nameFilter(n) if self._nameFilter else True) + ]) + + +class ServiceHub: + """ + Consumer-facing aggregation of services, DB interfaces, and runtime state. + + Services are lazy-resolved via serviceCenter on first access. + DB interfaces and runtime state are initialized eagerly. + Feature services/interfaces are discovered dynamically from features/. + """ + + _SERVICE_CENTER_WRAPPING = { + "ai": {"functionsOnly": False}, + } + + def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + self.user: User = user + self.workflow = workflow + self.mandateId: Optional[str] = mandateId + self.featureInstanceId: Optional[str] = featureInstanceId + self.currentUserPrompt: str = "" + self.rawUserPrompt: str = "" + + from modules.serviceCenter.context import ServiceCenterContext + self._serviceCenterContext = ServiceCenterContext( + user=user, + workflow=workflow, + mandate_id=mandateId, + feature_instance_id=featureInstanceId, + ) + + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) + + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface + self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) + + self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None + + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + self.interfaceDbChat = getChatInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) + + self._loadFeatureInterfaces() + self._loadFeatureServices() + + def __getattr__(self, name: str): + """Lazy-resolve services via serviceCenter on first access.""" + if name.startswith('_'): + raise AttributeError(name) + try: + from modules.serviceCenter import getService + service = getService(name, self._serviceCenterContext) + wrapping = self._SERVICE_CENTER_WRAPPING.get(name, {}) + functionsOnly = wrapping.get("functionsOnly", True) + wrapped = PublicService(service, functionsOnly=functionsOnly) + setattr(self, name, wrapped) + return wrapped + except KeyError: + raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'") + + def _loadFeatureInterfaces(self): + """Dynamically load interfaces from feature containers by filename pattern.""" + pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py") + for filepath in glob.glob(pattern): + try: + featureDir = os.path.basename(os.path.dirname(filepath)) + filename = os.path.basename(filepath)[:-3] + + modulePath = f"modules.features.{featureDir}.{filename}" + module = importlib.import_module(modulePath) + + if hasattr(module, "getInterface"): + interface = module.getInterface(self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId) + attrName = filename.replace("interfaceFeature", "interfaceDb") + setattr(self, attrName, interface) + logger.debug(f"Loaded interface: {attrName} from {modulePath}") + except Exception as e: + logger.debug(f"Could not load interface from {filepath}: {e}") + + def _loadFeatureServices(self): + """Dynamically load services from feature containers by filename pattern.""" + pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py") + for filepath in glob.glob(pattern): + try: + serviceDir = os.path.basename(os.path.dirname(filepath)) + featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath))) + filename = os.path.basename(filepath)[:-3] + + modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}" + module = importlib.import_module(modulePath) + + serviceClass = None + for attrName in dir(module): + if attrName.endswith("Service") and not attrName.startswith("_"): + cls = getattr(module, attrName) + if isinstance(cls, type): + serviceClass = cls + break + + if serviceClass: + attrName = serviceDir.replace("service", "").lower() + if not attrName: + attrName = serviceDir.lower() + + functionsOnly = attrName != "ai" + + def _makeServiceResolver(hub): + def _resolver(depKey: str): + return getattr(hub, depKey) + return _resolver + + import inspect + sig = inspect.signature(serviceClass.__init__) + paramCount = len([p for p in sig.parameters if p != 'self']) + if paramCount >= 2: + serviceInstance = serviceClass(self, _makeServiceResolver(self)) + else: + serviceInstance = serviceClass(self) + setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly)) + logger.debug(f"Loaded service: {attrName} from {modulePath}") + except Exception as e: + logger.debug(f"Could not load service from {filepath}: {e}") + + +# Backward-compatible alias +Services = ServiceHub + + +def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> ServiceHub: + """Get ServiceHub instance for the given user, mandate, and feature instance context.""" + return ServiceHub(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId) diff --git a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py index dd9fac2c..f2418b4b 100644 --- a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py +++ b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py @@ -524,7 +524,7 @@ class ProviderNotAllowedException(Exception): super().__init__(self.message) -from modules.shared.serviceExceptions import BillingContextError +from modules.datamodels.serviceExceptions import BillingContextError # Expose exception classes on BillingService so consumers can use service.InsufficientBalanceException # instead of importing from this module diff --git a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py index da1194c4..21ba33d1 100644 --- a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py +++ b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py @@ -76,63 +76,7 @@ def enhancePlainTextWithMarkdownTables(body: str) -> str: return "\n\n".join(out_parts) -def parseInlineRuns(text: str) -> list: - """ - Parse inline markdown formatting into a list of InlineRun dicts. - Handles: images, links, bold, italic, inline code, plain text. - Uses a regex-based tokenizer that processes tokens left-to-right. - """ - if not text: - return [{"type": "text", "value": ""}] - - # Pattern order matters: images before links, bold before italic - _TOKEN_RE = re.compile( - r'!\[(?P[^\]]*)\]\((?P[^)"]+)(?:\s+"(?P\d+)pt")?\)' # image - r'|\[(?P[^\]]+)\]\((?P[^)]+)\)' # link - r'|`(?P[^`]+)`' # inline code - r'|\*\*(?P.+?)\*\*' # bold - r'|(?.+?)\*(?!\w)' # italic *x* - r'|(?.+?)_(?!\w)' # italic _x_ - ) - - runs = [] - lastEnd = 0 - - for m in _TOKEN_RE.finditer(text): - # Plain text before this match - if m.start() > lastEnd: - runs.append({"type": "text", "value": text[lastEnd:m.start()]}) - - if m.group("imgAlt") is not None or m.group("imgSrc") is not None: - alt = (m.group("imgAlt") or "").strip() or "Image" - src = (m.group("imgSrc") or "").strip() - widthStr = m.group("imgWidth") - run = {"type": "image", "value": alt} - if src.startswith("file:"): - run["fileId"] = src[5:] - else: - run["href"] = src - if widthStr: - run["widthPt"] = int(widthStr) - runs.append(run) - elif m.group("linkText") is not None: - runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")}) - elif m.group("code") is not None: - runs.append({"type": "code", "value": m.group("code")}) - elif m.group("bold") is not None: - runs.append({"type": "bold", "value": m.group("bold")}) - elif m.group("italic1") is not None: - runs.append({"type": "italic", "value": m.group("italic1")}) - elif m.group("italic2") is not None: - runs.append({"type": "italic", "value": m.group("italic2")}) - - lastEnd = m.end() - - # Trailing plain text - if lastEnd < len(text): - runs.append({"type": "text", "value": text[lastEnd:]}) - - return runs if runs else [{"type": "text", "value": text}] +from modules.shared.documentUtils import parseInlineRuns # noqa: F401 — canonical source in shared/ def _imageRefToDict(token: str) -> Optional[Dict[str, Any]]: diff --git a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py index 1eaebf56..71dc4526 100644 --- a/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py +++ b/modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py @@ -1039,7 +1039,7 @@ def _buildCancelSummaryHtml(subRecord: Dict[str, Any], platformUrl: str = "") -> # Exception Classes (defined in shared, re-exported here for backward compat) # ============================================================================ -from modules.shared.serviceExceptions import ( +from modules.datamodels.serviceExceptions import ( SubscriptionInactiveException, SubscriptionCapacityException, SUBSCRIPTION_USER_ACTION_UPGRADE, diff --git a/modules/serviceHub/__init__.py b/modules/serviceHub/__init__.py index a42f8d0e..14021394 100644 --- a/modules/serviceHub/__init__.py +++ b/modules/serviceHub/__init__.py @@ -1,189 +1,7 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Service Hub. -Consumer-facing aggregation layer for services, DB interfaces, and runtime state. - -Architecture: -- serviceHub delegates service resolution to serviceCenter (DI container) -- serviceHub owns DB interface initialization and runtime state -- serviceCenter knows nothing about serviceHub (one-way dependency) - -Import-Regelwerk: -- Zentrale Module (wie dieses) duerfen KEINE Feature-Container importieren -- Feature-spezifische Services werden dynamisch geladen -- Shared Services werden via serviceCenter resolved -""" - -import os -import importlib -import glob -from typing import Any, Optional, TYPE_CHECKING -import logging - -from modules.datamodels.datamodelUam import User - -if TYPE_CHECKING: - from modules.datamodels.datamodelChat import ChatWorkflow - -logger = logging.getLogger(__name__) - -_FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features") - - -class PublicService: - """Lightweight proxy exposing only public callable attributes of a target.""" - - def __init__(self, target: Any, functionsOnly: bool = True, nameFilter=None): - self._target = target - self._functionsOnly = functionsOnly - self._nameFilter = nameFilter - - def __getattr__(self, name: str): - if name.startswith('_'): - raise AttributeError(f"'{type(self._target).__name__}' attribute '{name}' is private") - if self._nameFilter and not self._nameFilter(name): - raise AttributeError(f"'{name}' not exposed by policy") - attr = getattr(self._target, name) - if self._functionsOnly and not callable(attr): - raise AttributeError(f"'{name}' is not a function") - return attr - - def __dir__(self): - return sorted([ - n for n in dir(self._target) - if not n.startswith('_') - and (not self._functionsOnly or callable(getattr(self._target, n, None))) - and (self._nameFilter(n) if self._nameFilter else True) - ]) - - -class ServiceHub: - """ - Consumer-facing aggregation of services, DB interfaces, and runtime state. - - Services are lazy-resolved via serviceCenter on first access. - DB interfaces and runtime state are initialized eagerly. - Feature services/interfaces are discovered dynamically from features/. - """ - - _SERVICE_CENTER_WRAPPING = { - "ai": {"functionsOnly": False}, - } - - def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - self.user: User = user - self.workflow = workflow - self.mandateId: Optional[str] = mandateId - self.featureInstanceId: Optional[str] = featureInstanceId - self.currentUserPrompt: str = "" - self.rawUserPrompt: str = "" - - from modules.serviceCenter.context import ServiceCenterContext - self._serviceCenterContext = ServiceCenterContext( - user=user, - workflow=workflow, - mandate_id=mandateId, - feature_instance_id=featureInstanceId, - ) - - from modules.interfaces.interfaceDbApp import getInterface as getAppInterface - self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) - - from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface - self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) - - self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None - - from modules.interfaces.interfaceDbChat import getInterface as getChatInterface - self.interfaceDbChat = getChatInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) - - self._loadFeatureInterfaces() - self._loadFeatureServices() - - def __getattr__(self, name: str): - """Lazy-resolve services via serviceCenter on first access.""" - if name.startswith('_'): - raise AttributeError(name) - try: - from modules.serviceCenter import getService - service = getService(name, self._serviceCenterContext) - wrapping = self._SERVICE_CENTER_WRAPPING.get(name, {}) - functionsOnly = wrapping.get("functionsOnly", True) - wrapped = PublicService(service, functionsOnly=functionsOnly) - setattr(self, name, wrapped) - return wrapped - except KeyError: - raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'") - - def _loadFeatureInterfaces(self): - """Dynamically load interfaces from feature containers by filename pattern.""" - pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py") - for filepath in glob.glob(pattern): - try: - featureDir = os.path.basename(os.path.dirname(filepath)) - filename = os.path.basename(filepath)[:-3] - - modulePath = f"modules.features.{featureDir}.{filename}" - module = importlib.import_module(modulePath) - - if hasattr(module, "getInterface"): - interface = module.getInterface(self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId) - attrName = filename.replace("interfaceFeature", "interfaceDb") - setattr(self, attrName, interface) - logger.debug(f"Loaded interface: {attrName} from {modulePath}") - except Exception as e: - logger.debug(f"Could not load interface from {filepath}: {e}") - - def _loadFeatureServices(self): - """Dynamically load services from feature containers by filename pattern.""" - pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py") - for filepath in glob.glob(pattern): - try: - serviceDir = os.path.basename(os.path.dirname(filepath)) - featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath))) - filename = os.path.basename(filepath)[:-3] - - modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}" - module = importlib.import_module(modulePath) - - serviceClass = None - for attrName in dir(module): - if attrName.endswith("Service") and not attrName.startswith("_"): - cls = getattr(module, attrName) - if isinstance(cls, type): - serviceClass = cls - break - - if serviceClass: - attrName = serviceDir.replace("service", "").lower() - if not attrName: - attrName = serviceDir.lower() - - functionsOnly = attrName != "ai" - - def _makeServiceResolver(hub): - def _resolver(depKey: str): - return getattr(hub, depKey) - return _resolver - - import inspect - sig = inspect.signature(serviceClass.__init__) - paramCount = len([p for p in sig.parameters if p != 'self']) - if paramCount >= 2: - serviceInstance = serviceClass(self, _makeServiceResolver(self)) - else: - serviceInstance = serviceClass(self) - setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly)) - logger.debug(f"Loaded service: {attrName} from {modulePath}") - except Exception as e: - logger.debug(f"Could not load service from {filepath}: {e}") - - -# Backward-compatible alias -Services = ServiceHub - - -def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> ServiceHub: - """Get ServiceHub instance for the given user, mandate, and feature instance context.""" - return ServiceHub(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId) +# Re-export shim — canonical source: modules.serviceCenter.serviceHub +from modules.serviceCenter.serviceHub import ( # noqa: F401 + PublicService, + ServiceHub, + Services, + getInterface, +) diff --git a/modules/shared/documentUtils.py b/modules/shared/documentUtils.py new file mode 100644 index 00000000..cc08835c --- /dev/null +++ b/modules/shared/documentUtils.py @@ -0,0 +1,64 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Document utility functions (Layer L0 - shared). +Pure text-processing helpers with zero internal dependencies. +""" + +import re + + +def parseInlineRuns(text: str) -> list: + """ + Parse inline markdown formatting into a list of InlineRun dicts. + Handles: images, links, bold, italic, inline code, plain text. + Uses a regex-based tokenizer that processes tokens left-to-right. + """ + if not text: + return [{"type": "text", "value": ""}] + + _TOKEN_RE = re.compile( + r'!\[(?P[^\]]*)\]\((?P[^)"]+)(?:\s+"(?P\d+)pt")?\)' + r'|\[(?P[^\]]+)\]\((?P[^)]+)\)' + r'|`(?P[^`]+)`' + r'|\*\*(?P.+?)\*\*' + r'|(?.+?)\*(?!\w)' + r'|(?.+?)_(?!\w)' + ) + + runs = [] + lastEnd = 0 + + for m in _TOKEN_RE.finditer(text): + if m.start() > lastEnd: + runs.append({"type": "text", "value": text[lastEnd:m.start()]}) + + if m.group("imgAlt") is not None or m.group("imgSrc") is not None: + alt = (m.group("imgAlt") or "").strip() or "Image" + src = (m.group("imgSrc") or "").strip() + widthStr = m.group("imgWidth") + run = {"type": "image", "value": alt} + if src.startswith("file:"): + run["fileId"] = src[5:] + else: + run["href"] = src + if widthStr: + run["widthPt"] = int(widthStr) + runs.append(run) + elif m.group("linkText") is not None: + runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")}) + elif m.group("code") is not None: + runs.append({"type": "code", "value": m.group("code")}) + elif m.group("bold") is not None: + runs.append({"type": "bold", "value": m.group("bold")}) + elif m.group("italic1") is not None: + runs.append({"type": "italic", "value": m.group("italic1")}) + elif m.group("italic2") is not None: + runs.append({"type": "italic", "value": m.group("italic2")}) + + lastEnd = m.end() + + if lastEnd < len(text): + runs.append({"type": "text", "value": text[lastEnd:]}) + + return runs if runs else [{"type": "text", "value": text}] diff --git a/modules/shared/eventManager.py b/modules/shared/eventManager.py new file mode 100644 index 00000000..13b0b322 --- /dev/null +++ b/modules/shared/eventManager.py @@ -0,0 +1,167 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Event manager for SSE streaming (Layer L0 - shared). +Manages event queues for Server-Sent Events (SSE) streaming across features. +Generic pub/sub infrastructure with zero internal dependencies. +""" + +import logging +import asyncio +from typing import Dict, Optional, Any + +logger = logging.getLogger(__name__) + + +class EventManager: + """ + Manages event queues for SSE streaming. + Each workflow has its own async queue for events. + """ + + def __init__(self): + """Initialize the event manager.""" + self._queues: Dict[str, asyncio.Queue] = {} + self._cleanup_tasks: Dict[str, asyncio.Task] = {} + self._agent_tasks: Dict[str, asyncio.Task] = {} + self._cancelled: Dict[str, bool] = {} + + def create_queue(self, workflow_id: str) -> asyncio.Queue: + """Create an event queue for a workflow.""" + if workflow_id in self._cleanup_tasks: + self._cleanup_tasks[workflow_id].cancel() + del self._cleanup_tasks[workflow_id] + logger.debug(f"Cancelled pending cleanup for workflow {workflow_id}") + + if workflow_id not in self._queues: + self._queues[workflow_id] = asyncio.Queue() + logger.debug(f"Created event queue for workflow {workflow_id}") + else: + old = self._queues[workflow_id] + while not old.empty(): + try: + old.get_nowait() + except asyncio.QueueEmpty: + break + logger.debug(f"Reusing event queue for workflow {workflow_id} (drained stale events)") + return self._queues[workflow_id] + + def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]: + """Get the event queue for a workflow.""" + return self._queues.get(workflow_id) + + def has_queue(self, workflow_id: str) -> bool: + """Check if a queue exists for a workflow.""" + return workflow_id in self._queues + + def register_agent_task(self, workflow_id: str, task: asyncio.Task) -> None: + """Register the asyncio Task running the agent for a workflow.""" + self._agent_tasks[workflow_id] = task + self._cancelled.pop(workflow_id, None) + + def is_cancelled(self, workflow_id: str) -> bool: + """Check if a workflow has been cancelled.""" + return self._cancelled.get(workflow_id, False) + + async def cancel_agent(self, workflow_id: str) -> bool: + """Cancel the running agent task for a workflow. Returns True if cancelled.""" + self._cancelled[workflow_id] = True + task = self._agent_tasks.pop(workflow_id, None) + if task and not task.done(): + task.cancel() + logger.info(f"Cancelled agent task for workflow {workflow_id}") + return True + logger.debug(f"No running agent task found for workflow {workflow_id}") + return False + + def _unregister_agent_task(self, workflow_id: str) -> None: + """Remove the agent task reference after completion.""" + self._agent_tasks.pop(workflow_id, None) + self._cancelled.pop(workflow_id, None) + + async def emit_event( + self, + context_id: str, + event_type: str, + data: Dict[str, Any], + event_category: str = "chat", + message: Optional[str] = None, + step: Optional[str] = None + ) -> None: + """Emit an event to the queue for a workflow.""" + queue = self._queues.get(context_id) + if not queue: + return + + event = { + "type": event_type, + "data": data, + "category": event_category, + "message": message, + "step": step + } + + try: + await queue.put(event) + if event_type not in ("chunk",): + logger.debug(f"Emitted {event_type} event for workflow {context_id}") + except Exception as e: + logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True) + + async def cleanup(self, workflow_id: str, delay: float = 60.0) -> None: + """Schedule cleanup of a queue after a delay.""" + if workflow_id in self._cleanup_tasks: + self._cleanup_tasks[workflow_id].cancel() + + async def _cleanup(): + try: + await asyncio.sleep(delay) + if workflow_id in self._queues: + queue = self._queues[workflow_id] + while not queue.empty(): + try: + queue.get_nowait() + except asyncio.QueueEmpty: + break + del self._queues[workflow_id] + logger.info(f"Cleaned up event queue for workflow {workflow_id}") + except asyncio.CancelledError: + logger.debug(f"Cleanup cancelled for workflow {workflow_id}") + except Exception as e: + logger.error(f"Error during cleanup for workflow {workflow_id}: {e}", exc_info=True) + finally: + if workflow_id in self._cleanup_tasks: + del self._cleanup_tasks[workflow_id] + + task = asyncio.create_task(_cleanup()) + self._cleanup_tasks[workflow_id] = task + + def shutdown(self) -> None: + """Cancel all pending cleanup and agent tasks for fast process exit.""" + for _wfId, q in list(self._queues.items()): + try: + q.put_nowait(None) + except Exception: + pass + for wfId, task in list(self._cleanup_tasks.items()): + if not task.done(): + task.cancel() + self._cleanup_tasks.clear() + for wfId, task in list(self._agent_tasks.items()): + if not task.done(): + task.cancel() + self._agent_tasks.clear() + self._queues.clear() + logger.info("EventManager shutdown: all tasks cancelled, queues drained") + + +# Global event manager instance +_event_manager: Optional[EventManager] = None + + +def get_event_manager() -> EventManager: + """Get the global event manager instance.""" + global _event_manager + if _event_manager is None: + _event_manager = EventManager() + return _event_manager diff --git a/modules/shared/featureDiscovery.py b/modules/shared/featureDiscovery.py new file mode 100644 index 00000000..0332e9c1 --- /dev/null +++ b/modules/shared/featureDiscovery.py @@ -0,0 +1,59 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Feature discovery utility (Layer L0 - shared). +Dynamically discovers and loads feature main modules from the features directory. +Zero internal dependencies — only os, glob, importlib, logging. +""" + +import os +import glob +import importlib +import logging +from typing import Dict, Any + +logger = logging.getLogger(__name__) + +FEATURES_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "features" +) + +_cachedMainModules = None + + +def loadFeatureMainModules() -> Dict[str, Any]: + """ + Dynamically load main modules from all discovered feature containers. + Results are cached after the first call. + """ + global _cachedMainModules + if _cachedMainModules is not None: + return _cachedMainModules + + mainModules = {} + pattern = os.path.join(FEATURES_DIR, "*", "main*.py") + + for filepath in glob.glob(pattern): + filename = os.path.basename(filepath) + if filename == "__init__.py": + continue + + featureDir = os.path.basename(os.path.dirname(filepath)) + if featureDir.startswith("_"): + continue + + if featureDir in mainModules: + continue + + mainFile = filename[:-3] + + try: + modulePath = f"modules.features.{featureDir}.{mainFile}" + module = importlib.import_module(modulePath) + mainModules[featureDir] = module + logger.debug(f"Loaded main module: {featureDir}") + except Exception as e: + logger.error(f"Failed to load main module from {featureDir}: {e}") + + _cachedMainModules = mainModules + return mainModules diff --git a/modules/system/gdprDeletion.py b/modules/system/gdprDeletion.py index c6f8d5ca..ab3a6e2b 100644 --- a/modules/system/gdprDeletion.py +++ b/modules/system/gdprDeletion.py @@ -561,39 +561,27 @@ def _deleteUserDataFromFeatureDatabases(userId: str, currentUser) -> Dict[str, A logger.info(f"Found {len(featureCodes)} feature types to process: {featureCodes}") - # Process each feature type + # Process each feature type via lifecycle hooks + from modules.shared.featureDiscovery import loadFeatureMainModules + featureModules = loadFeatureMainModules() + for featureCode in featureCodes: try: - dbName = f"poweron_{featureCode}" - - # Try to get feature interface - featureInterface = None - - if featureCode == "trustee": - from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface - featureInterface = getTrusteeInterface(currentUser) - elif featureCode == "realestate": - from modules.features.realestate.interfaceFeatureRealEstate import getInterface as getRealEstateInterface - featureInterface = getRealEstateInterface(currentUser) - elif featureCode == "neutralization": - from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutralizerInterface - featureInterface = getNeutralizerInterface(currentUser) - else: - logger.warning(f"No interface found for feature code: {featureCode}") + featureModule = featureModules.get(featureCode) + hook = getattr(featureModule, "onUserDelete", None) if featureModule else None + + if hook is None: + logger.warning(f"No onUserDelete hook for feature: {featureCode}") continue - - if featureInterface and hasattr(featureInterface, 'db'): - featureStats = deleteUserDataFromDatabase( - featureInterface.db, - userId, - dbName - ) + + featureStats = hook(userId, currentUser) + if featureStats: stats["databases"].append(featureStats) - stats["totalTablesProcessed"] += featureStats["tablesProcessed"] - stats["totalRecordsDeleted"] += featureStats["recordsDeleted"] - stats["totalRecordsAnonymized"] += featureStats["recordsAnonymized"] - stats["errors"].extend(featureStats["errors"]) - + stats["totalTablesProcessed"] += featureStats.get("tablesProcessed", 0) + stats["totalRecordsDeleted"] += featureStats.get("recordsDeleted", 0) + stats["totalRecordsAnonymized"] += featureStats.get("recordsAnonymized", 0) + stats["errors"].extend(featureStats.get("errors", [])) + except Exception as featureErr: errorMsg = f"Error processing feature {featureCode}: {featureErr}" logger.warning(errorMsg) diff --git a/modules/system/i18nBootSync.py b/modules/system/i18nBootSync.py index e60aa4d3..96f1b69d 100644 --- a/modules/system/i18nBootSync.py +++ b/modules/system/i18nBootSync.py @@ -202,19 +202,19 @@ def _registerRbacLabels(): logger.info("i18n rbac labels: %d new keys (rbac.* context)", added) -def _registerServiceCenterLabels(): - """Register service-center category labels and bootstrap role descriptions.""" +def _registerServiceCenterLabels(serviceLabels: list = None): + """Register service-center category labels and bootstrap role descriptions. + + serviceLabels is injected by app.py (Composition Root) to avoid + system(L4) → serviceCenter(L5) upward import. + """ added = 0 - try: - from modules.serviceCenter.registry import IMPORTABLE_SERVICES - for svc in IMPORTABLE_SERVICES.values(): - key = _extractRegistrySourceText(svc.get("label")) - if key and key not in _REGISTRY: - _REGISTRY[key] = _I18nRegistryEntry(context="service", value="") - added += 1 - except ImportError: - pass + for label in (serviceLabels or []): + key = _extractRegistrySourceText(label) + if key and key not in _REGISTRY: + _REGISTRY[key] = _I18nRegistryEntry(context="service", value="") + added += 1 _bootstrapRoleDescriptions = [ "Administrator - Benutzer und Ressourcen im Mandanten verwalten", @@ -288,38 +288,28 @@ def _registerNodeLabels(): logger.info("i18n node labels: %d new keys (node.*/port.* context)", added) -def _registerAccountingConnectorLabels(): - """Register all accounting connector configField labels at boot time.""" +def _registerAccountingConnectorLabels(accountingLabels: list = None): + """Register accounting connector configField labels at boot time. + + Args: + accountingLabels: List of dicts with keys 'label' and 'connectorType', + injected from app.py to avoid features-import. + """ + if not accountingLabels: + return + added = 0 - try: - from modules.features.trustee.accounting.accountingRegistry import getAccountingRegistry - except ImportError: - logger.debug("i18n accounting connectors: registry not importable") - return - - try: - registry = getAccountingRegistry() - except Exception as e: - logger.warning("i18n accounting connectors: registry init failed: %s", e) - return - - for connectorType, connector in (registry._connectors or {}).items(): - try: - for field in connector.getRequiredConfigFields(): - key = getattr(field, "label", "") or "" - if not isinstance(key, str) or not key: - continue - if key not in _REGISTRY: - _REGISTRY[key] = _I18nRegistryEntry( - context=f"connector.accounting.{connectorType}", - value="", - ) - added += 1 - except Exception as e: - logger.warning( - "i18n accounting connector %s: failed to read fields: %s", - connectorType, e, + for entry in accountingLabels: + key = entry.get("label", "") + connectorType = entry.get("connectorType", "unknown") + if not isinstance(key, str) or not key: + continue + if key not in _REGISTRY: + _REGISTRY[key] = _I18nRegistryEntry( + context=f"connector.accounting.{connectorType}", + value="", ) + added += 1 logger.info("i18n accounting connector labels: %d new keys", added) @@ -385,16 +375,21 @@ def _registerDatamodelOptionLabels(): # Public boot API (called by app.py) # --------------------------------------------------------------------------- -async def syncRegistryToDb(): - """Boot hook: discover all i18n keys and write them into UiLanguageSet(xx).""" +async def syncRegistryToDb(serviceLabels: list = None, accountingLabels: list = None): + """Boot hook: discover all i18n keys and write them into UiLanguageSet(xx). + + Args: + serviceLabels: Service label strings injected from app.py (avoids upward import). + accountingLabels: Accounting connector field labels injected from app.py. + """ _scanRouteApiMsgKeys() _registerNavLabels() _registerFeatureUiLabels() _registerRbacLabels() - _registerServiceCenterLabels() + _registerServiceCenterLabels(serviceLabels) _registerNodeLabels() _registerDatamodelOptionLabels() - _registerAccountingConnectorLabels() + _registerAccountingConnectorLabels(accountingLabels) if not _REGISTRY: logger.info("i18n registry: no keys to sync (empty registry)") diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index 21d0cbee..bbdffbbd 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -20,351 +20,7 @@ FEATURE_CODE = "system" FEATURE_LABEL = "System" FEATURE_ICON = "mdi-cog" -# ============================================================================= -# Navigation Structure (Single Source of Truth) -# ============================================================================= -# -# Block Order (gemäss Navigation-API-Konzept): -# - System: 10 -# - : 15 (wird in routeSystem.py eingefügt) -# - Basisdaten: 30 -# - Administration: 200 -# -# NOTE: Workflows and Migrate sections removed - now handled as features -# -# Item Order: Default-Abstand 10 pro Item -# uiComponent: Abgeleitet von objectKey (ui.system.home -> page.system.home) -# icon: Wird intern gehalten aber NICHT in der API Response zurückgegeben - -NAVIGATION_SECTIONS = [ - # ─── Meine Sicht (with top-level item + subgroups) ─── - { - "id": "system", - "title": t("Meine Sicht"), - "order": 10, - "items": [ - { - "id": "home", - "objectKey": "ui.system.home", - "label": t("Start"), - "icon": "FaHome", - "path": "/", - "order": 10, - "public": True, - }, - ], - "subgroups": [ - # ── Übersichten ── - { - "id": "system-overviews", - "title": t("Übersichten"), - "order": 15, - "items": [ - { - "id": "integrations", - "objectKey": "ui.system.integrations", - "label": t("Integrationen"), - "icon": "FaProjectDiagram", - "path": "/integrations", - "order": 10, - "public": True, - }, - { - "id": "compliance-audit", - "objectKey": "ui.system.complianceAudit", - "label": t("Compliance & Audit"), - "icon": "FaShieldAlt", - "path": "/compliance-audit", - "order": 20, - }, - ], - }, - # ── Basisdaten ── - { - "id": "system-basedata", - "title": t("Basisdaten"), - "order": 20, - "items": [ - { - "id": "connections", - "objectKey": "ui.system.connections", - "label": t("Verbindungen"), - "icon": "FaLink", - "path": "/basedata/connections", - "order": 10, - "public": True, - }, - { - "id": "files", - "objectKey": "ui.system.files", - "label": t("Dateien"), - "icon": "FaRegFileAlt", - "path": "/basedata/files", - "order": 20, - "public": True, - }, - { - "id": "prompts", - "objectKey": "ui.system.prompts", - "label": t("Prompts"), - "icon": "FaLightbulb", - "path": "/basedata/prompts", - "order": 30, - "public": True, - }, - ], - }, - # ── Nutzung ── - { - "id": "system-usage", - "title": t("Nutzung"), - "order": 30, - "items": [ - { - "id": "billing-admin", - "objectKey": "ui.system.billingAdmin", - "label": t("Abrechnung"), - "icon": "FaMoneyBillAlt", - "path": "/billing/admin", - "order": 10, - }, - { - "id": "statistics", - "objectKey": "ui.system.statistics", - "label": t("Statistiken"), - "icon": "FaChartBar", - "path": "/billing/transactions", - "order": 20, - }, - { - "id": "automations", - "objectKey": "ui.system.automations", - "label": t("Automations"), - "icon": "FaRobot", - "path": "/automations", - "order": 30, - }, - { - "id": "rag-inventory", - "objectKey": "ui.system.ragInventory", - "label": t("RAG-Inventar"), - "icon": "FaDatabase", - "path": "/rag-inventory", - "order": 35, - }, - { - "id": "store", - "objectKey": "ui.system.store", - "label": t("Store"), - "icon": "FaStore", - "path": "/store", - "order": 40, - "public": True, - }, - { - "id": "settings", - "objectKey": "ui.system.settings", - "label": t("Einstellungen"), - "icon": "FaCog", - "path": "/settings", - "order": 50, - "public": True, - }, - ], - }, - ], - }, - # ─── Administration (with subgroups) ─── - { - "id": "admin", - "title": t("Administration"), - "order": 200, - "subgroups": [ - # ── Wizards ── - { - "id": "admin-wizards", - "title": t("Wizards"), - "order": 10, - "items": [ - { - "id": "admin-mandate-wizard", - "objectKey": "ui.admin.mandateWizard", - "label": t("Mandanten-Wizard"), - "icon": "FaMagic", - "path": "/admin/mandate-wizard", - "order": 10, - "adminOnly": True, - }, - { - "id": "admin-invitation-wizard", - "objectKey": "ui.admin.invitationWizard", - "label": t("Einladungs-Wizard"), - "icon": "FaEnvelopeOpenText", - "path": "/admin/invitation-wizard", - "order": 20, - "adminOnly": True, - }, - ], - }, - # ── Users ── - { - "id": "admin-users-group", - "title": t("Benutzer"), - "order": 20, - "items": [ - { - "id": "admin-users", - "objectKey": "ui.admin.users", - "label": t("Benutzer"), - "icon": "FaUsers", - "path": "/admin/users", - "order": 10, - "adminOnly": True, - }, - { - "id": "admin-invitations", - "objectKey": "ui.admin.invitations", - "label": t("Benutzer-Einladungen"), - "icon": "FaEnvelopeOpenText", - "path": "/admin/invitations", - "order": 20, - "adminOnly": True, - }, - { - "id": "admin-user-access-overview", - "objectKey": "ui.admin.userAccessOverview", - "label": t("Benutzer-Zugriffsübersicht"), - "icon": "FaClipboardList", - "path": "/admin/user-access-overview", - "order": 30, - "adminOnly": True, - }, - { - "id": "admin-subscriptions", - "objectKey": "ui.admin.subscriptions", - "label": t("Abonnements"), - "icon": "FaFileContract", - "path": "/admin/subscriptions", - "order": 40, - "adminOnly": True, - }, - ], - }, - # ── System ── - { - "id": "admin-system-group", - "title": t("System"), - "order": 30, - "items": [ - { - "id": "admin-roles", - "objectKey": "ui.admin.roles", - "label": t("Rollen"), - "icon": "FaUserTag", - "path": "/admin/mandate-roles", - "order": 10, - "adminOnly": True, - }, - { - "id": "admin-mandate-role-permissions", - "objectKey": "ui.admin.mandateRolePermissions", - "label": t("Rollen-Berechtigungen"), - "icon": "FaKey", - "path": "/admin/mandate-role-permissions", - "order": 20, - "adminOnly": True, - }, - { - "id": "admin-mandates", - "objectKey": "ui.admin.mandates", - "label": t("Mandanten"), - "icon": "FaBuilding", - "path": "/admin/mandates", - "order": 30, - "adminOnly": True, - }, - { - "id": "admin-user-mandates", - "objectKey": "ui.admin.userMandates", - "label": t("Mandanten-Mitglieder"), - "icon": "FaUserFriends", - "path": "/admin/user-mandates", - "order": 40, - "adminOnly": True, - }, - { - "id": "admin-access", - "objectKey": "ui.admin.access", - "label": t("Zugriffsverwaltung"), - "icon": "FaBuilding", - "path": "/admin/access", - "order": 50, - "adminOnly": True, - }, - { - "id": "admin-feature-instances", - "objectKey": "ui.admin.featureInstances", - "label": t("Feature-Instanzen"), - "icon": "FaCubes", - "path": "/admin/feature-instances", - "order": 60, - "adminOnly": True, - }, - { - "id": "admin-feature-roles", - "objectKey": "ui.admin.featureRoles", - "label": t("Features Rollen-Vorlagen"), - "icon": "FaShieldAlt", - "path": "/admin/feature-roles", - "order": 70, - "adminOnly": True, - "sysAdminOnly": True, - }, - { - "id": "admin-logs", - "objectKey": "ui.admin.logs", - "label": t("Logs"), - "icon": "FaFileAlt", - "path": "/admin/logs", - "order": 90, - "adminOnly": True, - "sysAdminOnly": True, - }, - { - "id": "admin-languages", - "objectKey": "ui.admin.languages", - "label": t("UI-Sprachen"), - "icon": "FaGlobe", - "path": "/admin/languages", - "order": 95, - "adminOnly": True, - "sysAdminOnly": True, - }, - { - "id": "admin-database-health", - "objectKey": "ui.admin.databaseHealth", - "label": t("Datenbank-Gesundheit"), - "icon": "FaDatabase", - "path": "/admin/database-health", - "order": 98, - "adminOnly": True, - "sysAdminOnly": True, - }, - { - "id": "admin-demo-config", - "objectKey": "ui.admin.demoConfig", - "label": t("Demo Config"), - "icon": "FaCubes", - "path": "/admin/demo-config", - "order": 100, - "adminOnly": True, - "sysAdminOnly": True, - }, - ], - }, - ], - }, -] +from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS # noqa: F401 — canonical source def objectKeyToUiComponent(objectKey: str) -> str: diff --git a/modules/system/registry.py b/modules/system/registry.py index 67f3d28b..1e2dffb4 100644 --- a/modules/system/registry.py +++ b/modules/system/registry.py @@ -89,45 +89,7 @@ def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: return results -_cachedMainModules = None - -def loadFeatureMainModules() -> Dict[str, Any]: - """ - Dynamically load main modules from all discovered feature containers. - Results are cached after the first call. - """ - global _cachedMainModules - if _cachedMainModules is not None: - return _cachedMainModules - - mainModules = {} - pattern = os.path.join(FEATURES_DIR, "*", "main*.py") - - for filepath in glob.glob(pattern): - filename = os.path.basename(filepath) - if filename == "__init__.py": - continue - - featureDir = os.path.basename(os.path.dirname(filepath)) - if featureDir.startswith("_"): - continue - - # Skip if this feature already has a main module loaded (avoid duplicates) - if featureDir in mainModules: - continue - - mainFile = filename[:-3] # Remove .py - - try: - modulePath = f"modules.features.{featureDir}.{mainFile}" - module = importlib.import_module(modulePath) - mainModules[featureDir] = module - logger.debug(f"Loaded main module: {featureDir}") - except Exception as e: - logger.error(f"Failed to load main module from {featureDir}: {e}") - - _cachedMainModules = mainModules - return mainModules +from modules.shared.featureDiscovery import loadFeatureMainModules # noqa: F401 — re-export def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: @@ -150,16 +112,8 @@ def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]: logger.error(f"Error registering system RBAC objects: {e}") results["system"] = False - # Register service center RBAC objects (service.web, service.extraction, etc.) - try: - from modules.serviceCenter import registerServiceObjects - success = registerServiceObjects(catalogService) - results["servicecenter"] = success - except ImportError as e: - logger.warning(f"Service center not found, skipping service RBAC registration: {e}") - except Exception as e: - logger.error(f"Error registering service RBAC objects: {e}") - results["servicecenter"] = False + # Service center RBAC objects are registered by app.py (Composition Root) + # to avoid system(L4) → serviceCenter(L5) upward import. # Register feature modules mainModules = loadFeatureMainModules() diff --git a/modules/workflows/automation2/executionEngine.py b/modules/workflows/automation2/executionEngine.py index f8d95f1c..d5fdbd0e 100644 --- a/modules/workflows/automation2/executionEngine.py +++ b/modules/workflows/automation2/executionEngine.py @@ -31,7 +31,7 @@ from modules.workflows.automation2.executors import ( ) from modules.features.graphicalEditor.portTypes import normalizeToSchema, wrapTransit, unwrapTransit from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES -from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError from modules.workflows.automation2.graphicalEditorRunFileLogger import ( GraphicalEditorRunFileLogger, graphical_editor_run_file_logging_enabled, @@ -252,7 +252,7 @@ def _merge_node_parameters_into_snap( def _emitStepEvent(runId: str, stepData: Dict[str, Any]) -> None: """Emit a step-log SSE event to any listening client for this run.""" try: - from modules.serviceCenter.core.serviceStreaming.eventManager import get_event_manager + from modules.shared.eventManager import get_event_manager em = get_event_manager() queueId = f"run-trace-{runId}" if not em.has_queue(queueId): diff --git a/modules/workflows/automation2/executors/actionNodeExecutor.py b/modules/workflows/automation2/executors/actionNodeExecutor.py index 20fed58a..9af626d4 100644 --- a/modules/workflows/automation2/executors/actionNodeExecutor.py +++ b/modules/workflows/automation2/executors/actionNodeExecutor.py @@ -19,7 +19,7 @@ from modules.features.graphicalEditor.portTypes import ( _normalizeError, normalizeToSchema, ) -from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError from modules.workflows.automation2.executors.inputExecutor import PauseForHumanTaskError from modules.workflows.methods.methodContext.actions.extractContent import ( PRESENTATION_KIND, diff --git a/modules/workflows/methods/methodAi/actions/consolidate.py b/modules/workflows/methods/methodAi/actions/consolidate.py index 0dced074..70d345cd 100644 --- a/modules/workflows/methods/methodAi/actions/consolidate.py +++ b/modules/workflows/methods/methodAi/actions/consolidate.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum from modules.datamodels.datamodelChat import ActionResult -from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py index 66a1d0bf..7a13e4a1 100644 --- a/modules/workflows/methods/methodAi/actions/generateCode.py +++ b/modules/workflows/methods/methodAi/actions/generateCode.py @@ -9,7 +9,7 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData -from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 2006ba96..20b82042 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -9,7 +9,7 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData -from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py index 47774eb1..04046f39 100644 --- a/modules/workflows/methods/methodAi/actions/process.py +++ b/modules/workflows/methods/methodAi/actions/process.py @@ -10,7 +10,7 @@ from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ActionResult, ActionDocument from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart -from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodAi/actions/webResearch.py b/modules/workflows/methods/methodAi/actions/webResearch.py index 0dfdeeab..778faf11 100644 --- a/modules/workflows/methods/methodAi/actions/webResearch.py +++ b/modules/workflows/methods/methodAi/actions/webResearch.py @@ -7,8 +7,7 @@ import re import json from typing import Dict, Any from modules.datamodels.datamodelChat import ActionResult, ActionDocument -from modules.serviceCenter import ServiceCenterContext, getService, can_access_service -from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError +from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError logger = logging.getLogger(__name__) @@ -45,6 +44,8 @@ def _build_research_prompt(parameters: Dict[str, Any]) -> str: async def webResearch(self, parameters: Dict[str, Any]) -> ActionResult: + from modules.serviceCenter import ServiceCenterContext, getService, can_access_service + operationId = None try: prompt = _build_research_prompt(parameters) diff --git a/modules/workflows/methods/methodContext/actions/extractContent.py b/modules/workflows/methods/methodContext/actions/extractContent.py index 44309888..2c1a2f9c 100644 --- a/modules/workflows/methods/methodContext/actions/extractContent.py +++ b/modules/workflows/methods/methodContext/actions/extractContent.py @@ -1399,7 +1399,7 @@ def _load_image_bytes_by_file_id(services: Any, file_id: str) -> Optional[bytes] def _inline_runs_from_presentation_lines(lines: List[Any]) -> List[Dict[str, Any]]: """Map presentation ``lines`` to inline runs, preserving line order with explicit breaks.""" - from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import parseInlineRuns + from modules.shared.documentUtils import parseInlineRuns runs: List[Dict[str, Any]] = [] first = True @@ -1537,7 +1537,7 @@ def presentation_envelopes_to_document_json( services: Any = None, ) -> Dict[str, Any]: """Map presentation envelope(s) to ``renderReport`` ``extractedContent`` (documents/sections).""" - from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import parseInlineRuns + from modules.shared.documentUtils import parseInlineRuns envelopes = normalize_presentation_envelopes(raw) if not envelopes: diff --git a/modules/workflows/processing/shared/methodDiscovery.py b/modules/workflows/processing/shared/methodDiscovery.py index 213dee83..d778ba39 100644 --- a/modules/workflows/processing/shared/methodDiscovery.py +++ b/modules/workflows/processing/shared/methodDiscovery.py @@ -34,57 +34,78 @@ def _collectActionsUnfiltered(methodInstance) -> Dict[str, Dict[str, Any]]: return result +def _registerMethodClasses(serviceCenter, modulePath: str, uniqueCount: int) -> int: + """Import a method module and register all MethodBase subclasses found in it.""" + try: + module = importlib.import_module(modulePath) + except Exception as e: + logger.error(f"Error importing method module {modulePath}: {e}") + return uniqueCount + + for itemName, item in inspect.getmembers(module): + if (inspect.isclass(item) and + issubclass(item, MethodBase) and + item != MethodBase): + + if itemName in methods: + continue + + shortName = itemName.replace('Method', '').lower() + methodInstance = item(serviceCenter) + actions = _collectActionsUnfiltered(methodInstance) + + methodInfo = { + 'instance': methodInstance, + 'actions': actions, + 'description': item.__doc__ or f"Method {itemName}" + } + + methods[itemName] = methodInfo + methods[shortName] = methodInfo + uniqueCount += 1 + logger.info(f"Discovered method {itemName} (short: {shortName}) with {len(actions)} actions") + + return uniqueCount + + def discoverMethods(serviceCenter): - """Dynamically discover all method classes and their actions in modules methods package. - - Always creates fresh method instances bound to the given serviceCenter, - preventing stale or cross-workflow service references. + """Dynamically discover all method classes and their actions. + + Scans two locations: + 1. modules.workflows.methods (core methods) + 2. modules.features.*/workflows/ (feature-owned methods) """ global methods try: methodsPackage = importlib.import_module('modules.workflows.methods') - - # Clear and rebuild to prevent cross-workflow state contamination + methods.clear() uniqueCount = 0 - + for _, name, isPkg in pkgutil.iter_modules(methodsPackage.__path__): if name.startswith('method'): - try: - module = importlib.import_module(f'modules.workflows.methods.{name}') - - for itemName, item in inspect.getmembers(module): - if (inspect.isclass(item) and - issubclass(item, MethodBase) and - item != MethodBase): - - shortName = itemName.replace('Method', '').lower() - - # Skip if already processed (via another module path) - if itemName in methods: - continue - - methodInstance = item(serviceCenter) - actions = _collectActionsUnfiltered(methodInstance) - - methodInfo = { - 'instance': methodInstance, - 'actions': actions, - 'description': item.__doc__ or f"Method {itemName}" - } - - methods[itemName] = methodInfo - methods[shortName] = methodInfo - uniqueCount += 1 - - logger.info(f"Discovered method {itemName} (short: {shortName}) with {len(actions)} actions") - - except Exception as e: - logger.error(f"Error discovering method {name}: {str(e)}") - continue - + uniqueCount = _registerMethodClasses( + serviceCenter, f'modules.workflows.methods.{name}', uniqueCount + ) + + # Feature-owned methods (e.g. features/trustee/workflows/methodTrustee) + import os + import glob as _glob + featuresDir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "features" + ) + for wfInit in _glob.glob(os.path.join(featuresDir, "*", "workflows", "__init__.py")): + wfDir = os.path.dirname(wfInit) + featureName = os.path.basename(os.path.dirname(wfDir)) + for entry in os.listdir(wfDir): + entryPath = os.path.join(wfDir, entry) + if os.path.isdir(entryPath) and entry.startswith("method"): + modulePath = f"modules.features.{featureName}.workflows.{entry}" + uniqueCount = _registerMethodClasses(serviceCenter, modulePath, uniqueCount) + logger.info(f"Discovered {uniqueCount} unique methods ({len(methods)} entries with aliases)") - + except Exception as e: logger.error(f"Error discovering methods: {str(e)}") diff --git a/modules/workflows/scheduler/mainScheduler.py b/modules/workflows/scheduler/mainScheduler.py index ef89f821..9af9889f 100644 --- a/modules/workflows/scheduler/mainScheduler.py +++ b/modules/workflows/scheduler/mainScheduler.py @@ -396,48 +396,29 @@ def _createRunFailedNotification( logger.warning("Failed to create in-app run.failed notification: %s", e) +_onRunFailedCallback = None + + +def setOnRunFailedCallback(callback) -> None: + """Set the callback for run failure notifications (injected by app.py).""" + global _onRunFailedCallback + _onRunFailedCallback = callback + + def _triggerRunFailedSubscription( workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None ) -> None: - """Trigger the messaging subscription for run failures (email notifications).""" + """Trigger the messaging subscription for run failures via injected callback.""" + if _onRunFailedCallback is None: + return try: - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelMessaging import MessagingEventParameters - - rootInterface = getRootInterface() - if not rootInterface: - return - eventUser = rootInterface.getUserByUsername("event") - if not eventUser: - return - - ctx = ServiceCenterContext( - user=eventUser, - mandate_id=mandateId or "", - feature_instance_id="", - feature_code="graphicalEditor", + _onRunFailedCallback( + workflowId=workflowId, + runId=runId, + error=error, + mandateId=mandateId, + workflowLabel=workflowLabel, ) - messagingService = getService("messaging", ctx) - - subscriptionId = "GraphicalEditorRunFailed" - eventParams = MessagingEventParameters(triggerData={ - "workflowId": workflowId, - "workflowLabel": workflowLabel or workflowId, - "runId": runId, - "error": error, - "mandateId": mandateId or "", - }) - result = messagingService.executeSubscription(subscriptionId, eventParams) - logger.info( - "Triggered run.failed subscription: sent=%d success=%s", - result.messagesSent, result.success, - ) - except FileNotFoundError: - logger.debug("Subscription function GraphicalEditorRunFailed not found (not yet registered)") - except ValueError as e: - logger.debug("Subscription GraphicalEditorRunFailed: %s", e) except Exception as e: logger.warning("Failed to trigger run.failed subscription: %s", e)