PowerOn/platform-core
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: architecture cleanup + fix scheduler Automation2Workflow error
Some checks failed
Deploy Plattform-Core (Int) / test (push) Failing after 13s
Details
Deploy Plattform-Core (Int) / deploy (push) Has been skipped
Details

Browse source 
Fix: add missing Automation2Workflow/Automation2WorkflowRun imports to interfaceFeatureGraphicalEditor.py (caused scheduler crash on boot)
Refactor: gdprDeletion via onUserDelete lifecycle hooks
Refactor: i18nBootSync accounting labels via app.py parameter injection
Refactor: serviceHub moved to serviceCenter/serviceHub.py
Split: teamsbot/service.py, realEstate/main, routeTrustee, routeBilling
Cleanup: remove obsolete methodTrustee, serviceExceptions shim
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG 2026-06-07 07:59:31 +02:00
parent bc7c6fe27c
commit cf0233f193
59 changed files with 7662 additions and 8339 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
app.py
View file

@ -329,6 +329,14 @@ async def lifespan(app: FastAPI):
catalogService = getCatalogService()
registerAllFeaturesInCatalog(catalogService)
logger.info("Feature catalog registration completed")
# Register service center RBAC objects (Composition Root — avoids system→serviceCenter import)
try:
from modules.serviceCenter import registerServiceObjects
registerServiceObjects(catalogService)
except Exception as e:
logger.warning(f"Service center RBAC registration failed: {e}")
# Persist the in-memory feature registry into the Feature DB-table so
# the FeatureInstance.featureCode FK has real targets. Without this
# every FeatureInstance row would be flagged as orphan by the
@ -343,7 +351,22 @@ async def lifespan(app: FastAPI):
# Sync gateway i18n registry to DB and load translation cache
try:
from modules.system.i18nBootSync import syncRegistryToDb, loadCache
await syncRegistryToDb()
from modules.serviceCenter.registry import IMPORTABLE_SERVICES
serviceLabels = [svc.get("label") for svc in IMPORTABLE_SERVICES.values()]
accountingLabels = []
try:
from modules.features.trustee.accounting.accountingRegistry import getAccountingRegistry
registry = getAccountingRegistry()
for connectorType, connector in (registry._connectors or {}).items():
for field in connector.getRequiredConfigFields():
label = getattr(field, "label", "") or ""
if label:
accountingLabels.append({"label": label, "connectorType": connectorType})
except Exception:
pass
await syncRegistryToDb(serviceLabels=serviceLabels, accountingLabels=accountingLabels)
await loadCache()
logger.info("i18n registry sync + cache load completed")
except Exception as e:
@ -409,9 +432,41 @@ async def lifespan(app: FastAPI):
try:
main_loop = asyncio.get_running_loop()
eventManager.set_event_loop(main_loop)
from modules.workflows.scheduler.mainScheduler import setMainLoop as setSchedulerMainLoop
from modules.workflows.scheduler.mainScheduler import setMainLoop as setSchedulerMainLoop, setOnRunFailedCallback
setSchedulerMainLoop(main_loop)
# Inject run-failed notification callback (Composition Root — avoids workflows→serviceCenter import)
def _onRunFailed(workflowId, runId, error, mandateId=None, workflowLabel=None):
from modules.serviceCenter import getService
from modules.serviceCenter.context import ServiceCenterContext
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelMessaging import MessagingEventParameters
rootInterface = getRootInterface()
if not rootInterface:
return
eventUser = rootInterface.getUserByUsername("event")
if not eventUser:
return
ctx = ServiceCenterContext(
user=eventUser,
mandate_id=mandateId or "",
feature_instance_id="",
feature_code="graphicalEditor",
)
messagingService = getService("messaging", ctx)
subscriptionId = "GraphicalEditorRunFailed"
eventParams = MessagingEventParameters(triggerData={
"workflowId": workflowId,
"workflowLabel": workflowLabel or workflowId,
"runId": runId,
"error": error,
"mandateId": mandateId or "",
})
messagingService.executeSubscription(subscriptionId, eventParams)
setOnRunFailedCallback(_onRunFailed)
# Suppress noisy ConnectionResetError from ProactorEventLoop on Windows
# when clients (browsers) close connections abruptly. This is a known
# asyncio issue on Windows: https://bugs.python.org/issue39010

348
modules/datamodels/datamodelNavigation.py Normal file
View file

@ -0,0 +1,348 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Navigation structure data (Layer L1 - datamodels).
Single source of truth for UI navigation sections used by RBAC and frontend.
"""
from modules.shared.i18nRegistry import t
# =============================================================================
# Navigation Structure (Single Source of Truth)
# =============================================================================
#
# Block Order (gemaess Navigation-API-Konzept):
# - System: 10
# - <dynamic/features>: 15 (wird in routeSystem.py eingefuegt)
# - Basisdaten: 30
# - Administration: 200
#
# NOTE: Workflows and Migrate sections removed - now handled as features
#
# Item Order: Default-Abstand 10 pro Item
# uiComponent: Abgeleitet von objectKey (ui.system.home -> page.system.home)
# icon: Wird intern gehalten aber NICHT in der API Response zurueckgegeben
NAVIGATION_SECTIONS = [
# --- Meine Sicht (with top-level item + subgroups) ---
{
"id": "system",
"title": t("Meine Sicht"),
"order": 10,
"items": [
{
"id": "home",
"objectKey": "ui.system.home",
"label": t("Start"),
"icon": "FaHome",
"path": "/",
"order": 10,
"public": True,
},
],
"subgroups": [
{
"id": "system-overviews",
"title": t("Übersichten"),
"order": 15,
"items": [
{
"id": "integrations",
"objectKey": "ui.system.integrations",
"label": t("Integrationen"),
"icon": "FaProjectDiagram",
"path": "/integrations",
"order": 10,
"public": True,
},
{
"id": "compliance-audit",
"objectKey": "ui.system.complianceAudit",
"label": t("Compliance & Audit"),
"icon": "FaShieldAlt",
"path": "/compliance-audit",
"order": 20,
},
],
},
{
"id": "system-basedata",
"title": t("Basisdaten"),
"order": 20,
"items": [
{
"id": "connections",
"objectKey": "ui.system.connections",
"label": t("Verbindungen"),
"icon": "FaLink",
"path": "/basedata/connections",
"order": 10,
"public": True,
},
{
"id": "files",
"objectKey": "ui.system.files",
"label": t("Dateien"),
"icon": "FaRegFileAlt",
"path": "/basedata/files",
"order": 20,
"public": True,
},
{
"id": "prompts",
"objectKey": "ui.system.prompts",
"label": t("Prompts"),
"icon": "FaLightbulb",
"path": "/basedata/prompts",
"order": 30,
"public": True,
},
],
},
{
"id": "system-usage",
"title": t("Nutzung"),
"order": 30,
"items": [
{
"id": "billing-admin",
"objectKey": "ui.system.billingAdmin",
"label": t("Abrechnung"),
"icon": "FaMoneyBillAlt",
"path": "/billing/admin",
"order": 10,
},
{
"id": "statistics",
"objectKey": "ui.system.statistics",
"label": t("Statistiken"),
"icon": "FaChartBar",
"path": "/billing/transactions",
"order": 20,
},
{
"id": "automations",
"objectKey": "ui.system.automations",
"label": t("Automations"),
"icon": "FaRobot",
"path": "/automations",
"order": 30,
},
{
"id": "rag-inventory",
"objectKey": "ui.system.ragInventory",
"label": t("RAG-Inventar"),
"icon": "FaDatabase",
"path": "/rag-inventory",
"order": 35,
},
{
"id": "store",
"objectKey": "ui.system.store",
"label": t("Store"),
"icon": "FaStore",
"path": "/store",
"order": 40,
"public": True,
},
{
"id": "settings",
"objectKey": "ui.system.settings",
"label": t("Einstellungen"),
"icon": "FaCog",
"path": "/settings",
"order": 50,
"public": True,
},
],
},
],
},
# --- Administration (with subgroups) ---
{
"id": "admin",
"title": t("Administration"),
"order": 200,
"subgroups": [
{
"id": "admin-wizards",
"title": t("Wizards"),
"order": 10,
"items": [
{
"id": "admin-mandate-wizard",
"objectKey": "ui.admin.mandateWizard",
"label": t("Mandanten-Wizard"),
"icon": "FaMagic",
"path": "/admin/mandate-wizard",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-invitation-wizard",
"objectKey": "ui.admin.invitationWizard",
"label": t("Einladungs-Wizard"),
"icon": "FaEnvelopeOpenText",
"path": "/admin/invitation-wizard",
"order": 20,
"adminOnly": True,
},
],
},
{
"id": "admin-users-group",
"title": t("Benutzer"),
"order": 20,
"items": [
{
"id": "admin-users",
"objectKey": "ui.admin.users",
"label": t("Benutzer"),
"icon": "FaUsers",
"path": "/admin/users",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-invitations",
"objectKey": "ui.admin.invitations",
"label": t("Benutzer-Einladungen"),
"icon": "FaEnvelopeOpenText",
"path": "/admin/invitations",
"order": 20,
"adminOnly": True,
},
{
"id": "admin-user-access-overview",
"objectKey": "ui.admin.userAccessOverview",
"label": t("Benutzer-Zugriffsübersicht"),
"icon": "FaClipboardList",
"path": "/admin/user-access-overview",
"order": 30,
"adminOnly": True,
},
{
"id": "admin-subscriptions",
"objectKey": "ui.admin.subscriptions",
"label": t("Abonnements"),
"icon": "FaFileContract",
"path": "/admin/subscriptions",
"order": 40,
"adminOnly": True,
},
],
},
{
"id": "admin-system-group",
"title": t("System"),
"order": 30,
"items": [
{
"id": "admin-roles",
"objectKey": "ui.admin.roles",
"label": t("Rollen"),
"icon": "FaUserTag",
"path": "/admin/mandate-roles",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-mandate-role-permissions",
"objectKey": "ui.admin.mandateRolePermissions",
"label": t("Rollen-Berechtigungen"),
"icon": "FaKey",
"path": "/admin/mandate-role-permissions",
"order": 20,
"adminOnly": True,
},
{
"id": "admin-mandates",
"objectKey": "ui.admin.mandates",
"label": t("Mandanten"),
"icon": "FaBuilding",
"path": "/admin/mandates",
"order": 30,
"adminOnly": True,
},
{
"id": "admin-user-mandates",
"objectKey": "ui.admin.userMandates",
"label": t("Mandanten-Mitglieder"),
"icon": "FaUserFriends",
"path": "/admin/user-mandates",
"order": 40,
"adminOnly": True,
},
{
"id": "admin-access",
"objectKey": "ui.admin.access",
"label": t("Zugriffsverwaltung"),
"icon": "FaBuilding",
"path": "/admin/access",
"order": 50,
"adminOnly": True,
},
{
"id": "admin-feature-instances",
"objectKey": "ui.admin.featureInstances",
"label": t("Feature-Instanzen"),
"icon": "FaCubes",
"path": "/admin/feature-instances",
"order": 60,
"adminOnly": True,
},
{
"id": "admin-feature-roles",
"objectKey": "ui.admin.featureRoles",
"label": t("Features Rollen-Vorlagen"),
"icon": "FaShieldAlt",
"path": "/admin/feature-roles",
"order": 70,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-logs",
"objectKey": "ui.admin.logs",
"label": t("Logs"),
"icon": "FaFileAlt",
"path": "/admin/logs",
"order": 90,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-languages",
"objectKey": "ui.admin.languages",
"label": t("UI-Sprachen"),
"icon": "FaGlobe",
"path": "/admin/languages",
"order": 95,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-database-health",
"objectKey": "ui.admin.databaseHealth",
"label": t("Datenbank-Gesundheit"),
"icon": "FaDatabase",
"path": "/admin/database-health",
"order": 98,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-demo-config",
"objectKey": "ui.admin.demoConfig",
"label": t("Demo Config"),
"icon": "FaCubes",
"path": "/admin/demo-config",
"order": 100,
"adminOnly": True,
"sysAdminOnly": True,
},
],
},
],
},
]

0
modules/shared/serviceExceptions.py → modules/datamodels/serviceExceptions.py
View file

2
modules/features/graphicalEditor/interfaceFeatureGraphicalEditor.py
View file

@ -46,6 +46,8 @@ from modules.datamodels.datamodelWorkflowAutomation import (
AutoRun,
AutoStepLog,
AutoTask,
Automation2Workflow,
Automation2WorkflowRun,
)
from modules.features.graphicalEditor.entryPoints import invocations_synced_with_graph
from modules.connectors.connectorDbPostgre import DatabaseConnector

24
modules/features/neutralization/mainNeutralization.py
View file

@ -279,3 +279,27 @@ def onMandateDelete(mandateId: str, instances: list) -> None:
except Exception as e:
logger.warning(f"Failed to cascade-delete neutralization data for mandate {mandateId}: {e}")
def onUserDelete(userId: str, currentUser) -> dict:
"""Delete/anonymize user data from the neutralization database (GDPR)."""
from modules.system.gdprDeletion import deleteUserDataFromDatabase
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.shared.configuration import APP_CONFIG
dbName = "poweron_neutralization"
try:
db = DatabaseConnector(
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
dbDatabase=dbName,
dbUser=APP_CONFIG.get("DB_USER"),
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"),
dbPort=int(APP_CONFIG.get("DB_PORT", 5432)),
userId=None,
)
stats = deleteUserDataFromDatabase(db, userId, dbName)
db.close()
return stats
except Exception as e:
logger.warning(f"onUserDelete neutralization failed: {e}")
return {"database": dbName, "tablesProcessed": 0, "recordsDeleted": 0, "recordsAnonymized": 0, "errors": [str(e)]}

2
modules/features/neutralization/neutralizePlayground.py
View file

@ -9,7 +9,7 @@ from urllib.parse import urlparse, unquote
from modules.datamodels.datamodelUam import User
from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig, DataNeutralizationSnapshot
from .interfaceFeatureNeutralizer import getInterface as _getNeutralizerInterface
from modules.serviceHub import getInterface as getServices
from modules.serviceCenter.serviceHub import getInterface as getServices
logger = logging.getLogger(__name__)

949
modules/features/realEstate/handlerRealEstate.py Normal file
View file

@ -0,0 +1,949 @@
"""
Handler functions for Real Estate feature routes.
Contains extracted business logic from route handlers.
"""
import json
import logging
import re
import aiohttp
from typing import Optional, Dict, Any, List
from fastapi import HTTPException, status
from modules.datamodels.datamodelPagination import (
PaginationParams,
PaginatedResponse,
PaginationMetadata,
)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.interfaces.interfaceFeatures import getFeatureInterface
from .datamodelFeatureRealEstate import (
Projekt,
Parzelle,
Dokument,
DokumentTyp,
Gemeinde,
Kanton,
Land,
Kontext,
StatusProzess,
)
from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface
from .mainRealEstate import (
create_project_with_parcel_data,
extract_bzo_information,
)
from .parcelSelectionService import is_parcel_adjacent_to_selection
from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector
from modules.connectors.connectorOerebWfs import OerebWfsConnector
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeFeatureRealEstate")
logger = logging.getLogger(__name__)
# ============================================================================
# GEMEINDE / BZO HANDLERS
# ============================================================================
async def processGemeindenSync(interface, instanceId: str, mandateId: str, onlyCurrent: bool = True) -> Dict[str, Any]:
"""
Fetch all Gemeinden from Swiss Topo and save to DB for an instance.
Creates Kantone as needed.
"""
try:
oerebConnector = OerebWfsConnector()
connector = SwissTopoMapServerConnector(oereb_connector=oerebConnector)
gemeindenData = await connector.get_all_gemeinden(only_current=onlyCurrent)
except Exception as e:
logger.error(f"Error fetching Gemeinden from Swiss Topo: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=f"Error fetching Gemeinden: {str(e)}")
gemeindenCreated = 0
gemeindenSkipped = 0
kantoneCreated = 0
errors: List[str] = []
kantonCache: Dict[str, str] = {}
def _findGemeindeByBfsNummer(bfsNummer: str) -> Optional[Any]:
try:
gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId})
for g in gemeinden:
for k in (g.kontextInformationen or []):
try:
data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt
if isinstance(data, dict) and str(data.get("bfs_nummer")) == str(bfsNummer):
return g
except (json.JSONDecodeError, AttributeError):
continue
except Exception as ex:
logger.error(f"Error finding Gemeinde by BFS {bfsNummer}: {ex}", exc_info=True)
return None
def _getOrCreateKanton(kantonAbk: str) -> Optional[str]:
nonlocal kantoneCreated, errors
if not kantonAbk:
return None
if kantonAbk in kantonCache:
return kantonCache[kantonAbk]
kantone = interface.getKantone(recordFilter={"mandateId": mandateId, "abk": kantonAbk})
if kantone:
kantonCache[kantonAbk] = kantone[0].id
return kantone[0].id
kantonNames = {
"AG": "Aargau", "AI": "Appenzell Innerrhoden", "AR": "Appenzell Ausserrhoden",
"BE": "Bern", "BL": "Basel-Landschaft", "BS": "Basel-Stadt",
"FR": "Freiburg", "GE": "Genf", "GL": "Glarus", "GR": "Graubünden",
"JU": "Jura", "LU": "Luzern", "NE": "Neuenburg", "NW": "Nidwalden",
"OW": "Obwalden", "SG": "St. Gallen", "SH": "Schaffhausen", "SO": "Solothurn",
"SZ": "Schwyz", "TG": "Thurgau", "TI": "Tessin", "UR": "Uri",
"VD": "Waadt", "VS": "Wallis", "ZG": "Zug", "ZH": "Zürich",
}
try:
kantonLabel = kantonNames.get(kantonAbk, kantonAbk)
kanton = Kanton(
mandateId=mandateId,
featureInstanceId=instanceId,
label=kantonLabel,
abk=kantonAbk,
)
created = interface.createKanton(kanton)
if created and created.id:
kantonCache[kantonAbk] = created.id
kantoneCreated += 1
return created.id
except Exception as ex:
errors.append(f"Error creating Kanton {kantonAbk}: {ex}")
return None
savedGemeinden: List[Dict[str, Any]] = []
for gd in gemeindenData:
try:
gemeindeName = gd.get("name")
bfsNummer = gd.get("bfs_nummer")
kantonAbk = gd.get("kanton")
if not gemeindeName or bfsNummer is None:
gemeindenSkipped += 1
continue
existing = _findGemeindeByBfsNummer(str(bfsNummer))
if existing:
gemeindenSkipped += 1
savedGemeinden.append(existing.model_dump() if hasattr(existing, "model_dump") else existing)
continue
kantonId = _getOrCreateKanton(kantonAbk) if kantonAbk else None
gemeinde = Gemeinde(
mandateId=mandateId,
featureInstanceId=instanceId,
label=gemeindeName,
id_kanton=kantonId,
kontextInformationen=[
Kontext(thema="BFS Nummer", inhalt=json.dumps({"bfs_nummer": bfsNummer}, ensure_ascii=False))
],
)
created = interface.createGemeinde(gemeinde)
if created and created.id:
gemeindenCreated += 1
savedGemeinden.append(created.model_dump() if hasattr(created, "model_dump") else created)
else:
errors.append(f"Failed to create Gemeinde {gemeindeName}")
gemeindenSkipped += 1
except Exception as ex:
errors.append(f"Error processing {gd.get('name', 'Unknown')}: {str(ex)}")
gemeindenSkipped += 1
return {
"gemeinden": savedGemeinden,
"count": len(savedGemeinden),
"stats": {
"gemeinden_created": gemeindenCreated,
"gemeinden_skipped": gemeindenSkipped,
"kantone_created": kantoneCreated,
"error_count": len(errors),
"errors": errors[:10],
},
}
async def processBzoDocumentsFetch(interface, componentInterface, mandateId: str, instanceId: str) -> Dict[str, Any]:
"""Search for and download BZO documents for all Gemeinden of an instance."""
from modules.features.realEstate.realEstateGemeindeService import fetch_bzo_for_gemeinde
gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId})
stats = {"gemeinden_processed": 0, "documents_created": 0, "documents_skipped": 0, "errors": []}
results: List[Dict[str, Any]] = []
for gemeinde in gemeinden:
gr = {"gemeinde_id": gemeinde.id, "gemeinde_label": gemeinde.label, "status": None, "dokument_ids": [], "error": None}
try:
stats["gemeinden_processed"] += 1
fetched = await fetch_bzo_for_gemeinde(
interface, componentInterface, gemeinde, mandateId, instanceId
)
if fetched:
gr["status"] = "created"
stats["documents_created"] += 1
refreshed = interface.getGemeinde(gemeinde.id)
if refreshed and refreshed.dokumente:
for doc in refreshed.dokumente:
docId = getattr(doc, "id", None) or (doc.get("id") if isinstance(doc, dict) else None)
if docId:
gr["dokument_ids"].append(docId)
else:
gr["status"] = "skipped"
stats["documents_skipped"] += 1
except Exception as ex:
gr["status"] = "error"
gr["error"] = str(ex)
stats["errors"].append(f"{gemeinde.label}: {str(ex)}")
results.append(gr)
return {"success": True, "stats": stats, "results": results}
async def processParcelDocuments(interface, componentInterface, gemeindeName: str, bauzone: str, mandateId: str, instanceId: str) -> Dict[str, Any]:
"""
Ensure BZO document exists for Gemeinde, return documents for parcel info display.
Creates Gemeinde (Swiss Topo) and BZO (Tavily) if not in DB.
"""
from modules.features.realEstate.realEstateGemeindeService import (
ensure_single_gemeinde,
fetch_bzo_for_gemeinde,
)
gemeindeObj = None
byLabel = interface.getGemeinden(recordFilter={"label": gemeindeName, "mandateId": mandateId})
gemeindeObj = byLabel[0] if byLabel else None
if not gemeindeObj:
allG = interface.getGemeinden(recordFilter={"mandateId": mandateId})
gNorm = gemeindeName.strip().lower()
for g in allG:
gl = (g.label or "").strip().lower()
if gl == gNorm or gNorm in gl or gl in gNorm:
gemeindeObj = g
logger.debug(f"parcel-documents: Found Gemeinde by label match '{gemeindeName}' -> '{g.label}'")
break
if gemeindeObj:
logger.debug(f"parcel-documents: Gemeinde '{gemeindeName}' resolved: {gemeindeObj.id}")
if not gemeindeObj:
logger.info(f"parcel-documents: No Gemeinde for label '{gemeindeName}', ensuring via Swiss Topo...")
gemeindeObj = await ensure_single_gemeinde(interface, mandateId, instanceId, gemeinde_name=gemeindeName)
if not gemeindeObj:
logger.warning(f"parcel-documents: Gemeinde '{gemeindeName}' nicht gefunden (mandateId={mandateId[:8]}...)")
return {"documents": [], "error": f"Gemeinde '{gemeindeName}' nicht gefunden"}
bzoDocs = []
if gemeindeObj.dokumente:
for doc in gemeindeObj.dokumente:
typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None)
if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION] or str(typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision"]:
docId = doc.id if hasattr(doc, "id") else doc.get("id")
if docId:
full = interface.getDokument(docId)
if full and full.dokumentReferenz:
bzoDocs.append(full)
if not bzoDocs:
logger.info(f"parcel-documents: No BZO for {gemeindeName}, fetching...")
fetched = await fetch_bzo_for_gemeinde(interface, componentInterface, gemeindeObj, mandateId, instanceId)
if fetched:
gemeindeObj = interface.getGemeinde(gemeindeObj.id)
if gemeindeObj and gemeindeObj.dokumente:
for doc in gemeindeObj.dokumente:
typ = getattr(doc, "dokumentTyp", None) or (doc.get("dokumentTyp") if isinstance(doc, dict) else None)
if typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION]:
docId = doc.id if hasattr(doc, "id") else doc.get("id")
if docId:
full = interface.getDokument(docId)
if full and full.dokumentReferenz:
bzoDocs.append(full)
result = []
for d in bzoDocs:
result.append({
"id": d.id,
"label": d.label,
"fileId": d.dokumentReferenz,
"fileName": (d.label or "BZO") + ".pdf",
"mimeType": d.mimeType or "application/pdf",
})
return {"documents": result, "gemeinde": gemeindeName, "bauzone": bauzone}
# ============================================================================
# LEGACY TABLE HANDLERS
# ============================================================================
def processTableData(user, mandateId: Optional[str], table: str, pagination: Optional[str]) -> PaginatedResponse:
"""Fetch and paginate table data for a real estate entity table."""
tableMapping = {
"Projekt": (Projekt, "getProjekte"),
"Parzelle": (Parzelle, "getParzellen"),
"Dokument": (Dokument, "getDokumente"),
"Gemeinde": (Gemeinde, "getGemeinden"),
"Kanton": (Kanton, "getKantone"),
"Land": (Land, "getLaender"),
}
if table not in tableMapping:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid table name '{table}'. Available tables: {', '.join(tableMapping.keys())}"
)
realEstateInterface = getRealEstateInterface(user, mandateId=mandateId)
modelClass, methodName = tableMapping[table]
getterMethod = getattr(realEstateInterface, methodName)
items = getterMethod(recordFilter=None)
paginationParams = None
if pagination:
try:
paginationDict = json.loads(pagination)
paginationParams = PaginationParams(**paginationDict) if paginationDict else None
except (json.JSONDecodeError, ValueError) as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid pagination parameter: {str(e)}"
)
if paginationParams:
if paginationParams.sort:
for sortField in reversed(paginationParams.sort):
fieldName = sortField.field
direction = sortField.direction.lower()
def _sortKey(item, _fieldName=fieldName):
value = getattr(item, _fieldName, None)
if value is None:
return (1, None)
return (0, value)
items.sort(key=_sortKey, reverse=(direction == "desc"))
totalItems = len(items)
totalPages = (totalItems + paginationParams.pageSize - 1) // paginationParams.pageSize
startIdx = (paginationParams.page - 1) * paginationParams.pageSize
endIdx = startIdx + paginationParams.pageSize
paginatedItems = items[startIdx:endIdx]
return PaginatedResponse(
items=paginatedItems,
pagination=PaginationMetadata(
currentPage=paginationParams.page,
pageSize=paginationParams.pageSize,
totalItems=totalItems,
totalPages=totalPages,
sort=paginationParams.sort,
filters=paginationParams.filters
)
)
return PaginatedResponse(items=items, pagination=None)
async def processCreateTableRecord(user, mandateId: Optional[str], table: str, data: Dict[str, Any]) -> Dict[str, Any]:
"""Create a record in a real estate table, with special handling for Projekt+parcel."""
if table == "Projekt" and ("parzelle" in data or "parzellen" in data):
logger.info(f"Creating Projekt with parcel data for user {user.id}")
label = data.get("label")
if not label:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=routeApiMsg("label is required")
)
statusProzess = data.get("statusProzess", "Eingang")
parzellenData = []
if "parzellen" in data:
parzellenData = data.get("parzellen", [])
if not isinstance(parzellenData, list):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=routeApiMsg("parzellen must be an array")
)
elif "parzelle" in data:
parzelleData = data.get("parzelle")
if parzelleData:
parzellenData = [parzelleData]
if not parzellenData:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=routeApiMsg("parzelle or parzellen data is required")
)
result = await create_project_with_parcel_data(
currentUser=user,
mandateId=mandateId,
projekt_label=label,
parzellen_data=parzellenData,
status_prozess=statusProzess,
)
return result.get("projekt", {})
tableMapping = {
"Projekt": (Projekt, "createProjekt"),
"Parzelle": (Parzelle, "createParzelle"),
"Dokument": (Dokument, "createDokument"),
"Gemeinde": (Gemeinde, "createGemeinde"),
"Kanton": (Kanton, "createKanton"),
"Land": (Land, "createLand"),
}
if table not in tableMapping:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid table name '{table}'. Available tables: {', '.join(tableMapping.keys())}"
)
realEstateInterface = getRealEstateInterface(user, mandateId=mandateId)
modelClass, methodName = tableMapping[table]
createMethod = getattr(realEstateInterface, methodName)
if "mandateId" not in data:
data["mandateId"] = mandateId
try:
modelInstance = modelClass(**data)
except Exception as e:
logger.error(f"Error creating {table} model instance: {str(e)}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid data for {table}: {str(e)}"
)
createdRecord = createMethod(modelInstance)
if hasattr(createdRecord, 'model_dump'):
return createdRecord.model_dump()
return createdRecord
# ============================================================================
# PARCEL SEARCH HANDLER
# ============================================================================
async def processParcelSearch(user, mandateId: Optional[str], location: str, includeBauzone: bool, includeAdjacent: bool) -> Dict[str, Any]:
"""
Search for parcel information by address or coordinates.
Resolves address, calculates geometry, optionally fetches adjacent parcels and bauzone.
"""
connector = SwissTopoMapServerConnector()
parcelData = await connector.search_parcel(location)
if not parcelData:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"No parcel found for location: {location}"
)
extractedAttributes = connector.extract_parcel_attributes(parcelData)
attributes = parcelData.get("attributes", {})
geometry = parcelData.get("geometry", {})
areaM2 = None
centroid = None
if extractedAttributes.get("perimeter"):
perimeter = extractedAttributes["perimeter"]
points = perimeter.get("punkte", [])
if len(points) >= 3:
area = 0
for i in range(len(points)):
j = (i + 1) % len(points)
area += points[i]["x"] * points[j]["y"]
area -= points[j]["x"] * points[i]["y"]
areaM2 = abs(area / 2)
sumX = sum(p["x"] for p in points)
sumY = sum(p["y"] for p in points)
centroid = {"x": sumX / len(points), "y": sumY / len(points)}
canton = attributes.get("ak", "")
municipalityName = None
fullAddress = None
plz = None
geocodedAddress = parcelData.get('geocoded_address')
if geocodedAddress:
fullAddress = geocodedAddress.get('full_address')
plz = geocodedAddress.get('plz')
municipalityName = geocodedAddress.get('municipality')
logger.debug(f"Using geocoded address: {fullAddress}")
queryCoords = parcelData.get('query_coordinates')
addressQueryCoords = queryCoords if queryCoords else centroid
if not fullAddress and addressQueryCoords:
queryX = addressQueryCoords['x']
queryY = addressQueryCoords['y']
logger.debug(f"Querying address layer at query coordinates: ({queryX}, {queryY})")
isCoordinateSearch = ',' in location and not any(c.isalpha() for c in location.split(',')[0])
buildingTolerance = 1 if isCoordinateSearch else 10
buildingResult = await connector._query_building_layer(queryX, queryY, tolerance=buildingTolerance, buffer=25)
if buildingResult:
addrAttrs = buildingResult.get("attributes", {})
logger.debug(f"Address layer attributes: {addrAttrs}")
addressInfo = connector._extract_address_from_building_attrs(addrAttrs)
fullAddress = addressInfo.get('full_address')
plz = addressInfo.get('plz')
municipalityName = addressInfo.get('municipality')
if fullAddress:
logger.debug(f"Constructed address: {fullAddress}")
if not fullAddress:
if location and any(c.isalpha() for c in location) and "CH" not in location:
fullAddress = location
logger.debug(f"Using location as address: {fullAddress}")
if not municipalityName and fullAddress:
plzMunicipalityMatch = re.search(r"\b(\d{4})\s+([A-ZÄÖÜ][a-zäöüß\s-]+)", fullAddress)
if plzMunicipalityMatch:
extractedMunicipality = plzMunicipalityMatch.group(2).strip()
extractedMunicipality = re.sub(r"[,;\.]+$", "", extractedMunicipality).strip()
if extractedMunicipality:
municipalityName = extractedMunicipality
if not plz:
plz = plzMunicipalityMatch.group(1)
logger.debug(f"Extracted municipality from address: {municipalityName}")
bfsnr = attributes.get("bfsnr")
if not municipalityName and bfsnr and canton and mandateId:
try:
interface = getRealEstateInterface(user, mandateId=mandateId, featureInstanceId=None)
gemeinden = interface.getGemeinden(recordFilter={"mandateId": mandateId})
for g in gemeinden:
for k in (g.kontextInformationen or []):
try:
data = json.loads(k.inhalt) if isinstance(k.inhalt, str) else k.inhalt
if isinstance(data, dict):
bfs = data.get("bfs_nummer") or data.get("bfsnr") or data.get("municipality_code")
if str(bfs) == str(bfsnr):
municipalityName = g.label
logger.debug(f"Found Gemeinde by BFS {bfsnr} in DB: {municipalityName}")
break
except (json.JSONDecodeError, AttributeError):
continue
if municipalityName:
break
except Exception as e:
logger.debug(f"Error querying Gemeinde by BFS: {e}")
if not municipalityName and centroid and canton:
try:
geocodeUrl = "https://api3.geo.admin.ch/rest/services/api/MapServer/identify"
params = {
"geometry": f"{centroid['x']},{centroid['y']}",
"geometryType": "esriGeometryPoint",
"layers": "all:ch.swisstopo.swissboundaries3d-gemeinde-flaeche.fill",
"tolerance": "0",
"returnGeometry": "false",
"sr": "2056",
"f": "json",
}
async with aiohttp.ClientSession() as session:
async with session.get(geocodeUrl, params=params) as resp:
if resp.status == 200:
data = await resp.json()
results = data.get("results", [])
if results:
attrs = results[0].get("attributes", {})
geoName = attrs.get("name") or attrs.get("gemeindename") or attrs.get("label")
if geoName:
municipalityName = connector._clean_municipality_name(str(geoName))
logger.debug(f"Found municipality via Swiss Topo geocoding: {municipalityName}")
except Exception as e:
logger.debug(f"Error querying Swiss Topo geocoding: {e}")
if not municipalityName and bfsnr:
commonMunicipalities = {
261: "Zürich", 198: "Pfäffikon", 191: "Uster", 3203: "Winterthur",
351: "Bern", 2701: "Basel", 6621: "Genève", 5586: "Lausanne",
1061: "Luzern", 230: "St. Gallen", 5192: "Lugano", 1367: "Schwyz",
}
if bfsnr in commonMunicipalities:
municipalityName = commonMunicipalities[bfsnr]
logger.debug(f"Looked up municipality from common list: {municipalityName}")
elif canton and bfsnr:
municipalityName = f"{canton}-{bfsnr}"
logger.debug(f"Using fallback municipality: {municipalityName}")
if fullAddress and fullAddress.startswith("CH") and len(fullAddress) == 14 and fullAddress[2:].isdigit():
fullAddress = None
logger.debug("Removed EGRID from address field")
bauzone = None
hasGeometry = geometry and (geometry.get("rings") or geometry.get("coordinates"))
if includeBauzone and canton and hasGeometry and centroid:
try:
logger.debug(f"Querying zone information for parcel {attributes.get('label')} in canton {canton}")
oerebConnector = OerebWfsConnector()
zoneResults = await oerebConnector.query_zone_layer(
egrid=attributes.get("egris_egrid", "") or "",
x=centroid["x"],
y=centroid["y"],
canton=canton,
geometry=geometry,
)
if zoneResults and len(zoneResults) > 0:
zoneAttrs = zoneResults[0].get("attributes", {})
typGdeAbkuerzung = zoneAttrs.get("typ_gde_abkuerzung")
if typGdeAbkuerzung:
bauzone = typGdeAbkuerzung
logger.debug(f"Found bauzone: {bauzone} for parcel {attributes.get('label')}")
except Exception as e:
logger.warning(f"Error querying zone information: {e}", exc_info=True)
parcelInfo = {
"id": attributes.get("label") or attributes.get("number"),
"egrid": attributes.get("egris_egrid"),
"number": attributes.get("number"),
"name": attributes.get("name"),
"identnd": attributes.get("identnd"),
"canton": attributes.get("ak"),
"municipality_code": attributes.get("bfsnr"),
"municipality_name": municipalityName,
"address": fullAddress,
"plz": plz,
"perimeter": extractedAttributes.get("perimeter"),
"area_m2": areaM2,
"centroid": centroid,
"geoportal_url": attributes.get("geoportal_url"),
"realestate_type": attributes.get("realestate_type"),
"bauzone": bauzone,
}
bbox = parcelData.get("bbox", [])
mapView = {
"center": centroid,
"zoom_bounds": {
"min_x": bbox[0] if len(bbox) >= 4 else None,
"min_y": bbox[1] if len(bbox) >= 4 else None,
"max_x": bbox[2] if len(bbox) >= 4 else None,
"max_y": bbox[3] if len(bbox) >= 4 else None
},
"geometry_geojson": {
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [
[[p["x"], p["y"]] for p in extractedAttributes["perimeter"]["punkte"]]
] if extractedAttributes.get("perimeter") else []
},
"properties": {
"id": parcelInfo["id"],
"egrid": parcelInfo["egrid"],
"number": parcelInfo["number"]
}
}
}
responseData = {
"parcel": parcelInfo,
"map_view": mapView
}
if includeAdjacent and parcelData and parcelData.get("geometry"):
try:
selectedParcelId = parcelInfo["id"]
adjacentParcelsRaw = await connector.find_neighboring_parcels(
parcel_data=parcelData,
selected_parcel_id=selectedParcelId,
sample_distance=20.0,
max_sample_points=30,
max_neighbors=15,
max_concurrent=50,
)
adjacentParcels = [_convertParcelGeometry(adjParcel) for adjParcel in adjacentParcelsRaw]
responseData["adjacent_parcels"] = adjacentParcels
logger.info(f"Found {len(adjacentParcels)} neighboring parcels for parcel {selectedParcelId}")
except Exception as e:
logger.warning(f"Error fetching adjacent parcels: {e}", exc_info=True)
responseData["adjacent_parcels"] = []
return responseData
def _convertParcelGeometry(adjParcel: Dict[str, Any]) -> Dict[str, Any]:
"""Convert an adjacent parcel to include GeoJSON geometry."""
adjParcelWithGeo = {
"id": adjParcel["id"],
"egrid": adjParcel.get("egrid"),
"number": adjParcel.get("number"),
"perimeter": adjParcel.get("perimeter")
}
adjGeometry = adjParcel.get("geometry")
adjPerimeter = adjParcel.get("perimeter")
if adjGeometry:
if "rings" in adjGeometry and adjGeometry["rings"]:
ring = adjGeometry["rings"][0]
coordinates = [[[p[0], p[1]] for p in ring]]
adjParcelWithGeo["geometry_geojson"] = {
"type": "Feature",
"geometry": {"type": "Polygon", "coordinates": coordinates},
"properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")}
}
elif adjGeometry.get("type") == "Polygon":
adjParcelWithGeo["geometry_geojson"] = {
"type": "Feature",
"geometry": adjGeometry,
"properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")}
}
if "geometry_geojson" not in adjParcelWithGeo and adjPerimeter and adjPerimeter.get("punkte"):
punkte = adjPerimeter["punkte"]
coordinates = [[[p["x"], p["y"]] for p in punkte]]
adjParcelWithGeo["geometry_geojson"] = {
"type": "Feature",
"geometry": {"type": "Polygon", "coordinates": coordinates},
"properties": {"id": adjParcel["id"], "egrid": adjParcel.get("egrid"), "number": adjParcel.get("number")}
}
return adjParcelWithGeo
# ============================================================================
# ADD ADJACENT PARCEL HANDLER
# ============================================================================
async def processAddAdjacentParcel(location: Dict[str, Any], selectedParcels: List[Dict[str, Any]]) -> Dict[str, Any]:
"""
Add an adjacent parcel to the selection. Validates adjacency.
Returns parcel response with geometry.
"""
locStr = f"{location['x']},{location['y']}"
connector = SwissTopoMapServerConnector()
parcelData = await connector.search_parcel(locStr)
if not parcelData:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=routeApiMsg("No parcel found at this location")
)
extracted = connector.extract_parcel_attributes(parcelData)
attributes = parcelData.get("attributes", {})
geometry = parcelData.get("geometry", {})
areaM2 = None
centroid = None
if extracted.get("perimeter"):
perimeter = extracted["perimeter"]
points = perimeter.get("punkte", [])
if len(points) >= 3:
area = 0
for i in range(len(points)):
j = (i + 1) % len(points)
area += points[i]["x"] * points[j]["y"]
area -= points[j]["x"] * points[i]["y"]
areaM2 = abs(area / 2)
sumX = sum(p["x"] for p in points)
sumY = sum(p["y"] for p in points)
centroid = {"x": sumX / len(points), "y": sumY / len(points)}
parcelInfo = {
"id": attributes.get("label") or attributes.get("number"),
"egrid": attributes.get("egris_egrid"),
"number": attributes.get("number"),
"name": attributes.get("name"),
"identnd": attributes.get("identnd"),
"canton": attributes.get("ak"),
"municipality_code": attributes.get("bfsnr"),
"municipality_name": None,
"address": None,
"plz": None,
"perimeter": extracted.get("perimeter"),
"area_m2": areaM2,
"centroid": centroid,
"geoportal_url": attributes.get("geoportal_url"),
"realestate_type": attributes.get("realestate_type"),
"bauzone": None,
}
mapView = {
"center": centroid,
"zoom_bounds": parcelData.get("bbox", []) and {
"min_x": parcelData["bbox"][0],
"min_y": parcelData["bbox"][1],
"max_x": parcelData["bbox"][2],
"max_y": parcelData["bbox"][3],
} or None,
"geometry_geojson": _buildGeometryGeojson(extracted, parcelInfo),
}
newParcelResponse = {"parcel": parcelInfo, "map_view": mapView}
if not is_parcel_adjacent_to_selection(newParcelResponse, selectedParcels):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=routeApiMsg("Nur angrenzende Parzellen können hinzugefügt werden")
)
bbox = parcelData.get("bbox", [])
mapView["zoom_bounds"] = {
"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3]
} if len(bbox) >= 4 else None
geocodedAddress = parcelData.get("geocoded_address")
if geocodedAddress:
parcelInfo["municipality_name"] = geocodedAddress.get("municipality")
parcelInfo["address"] = geocodedAddress.get("full_address")
parcelInfo["plz"] = geocodedAddress.get("plz")
if centroid and attributes.get("ak"):
try:
oereb = OerebWfsConnector()
zoneResults = await oereb.query_zone_layer(
egrid=attributes.get("egris_egrid", "") or "",
x=centroid["x"], y=centroid["y"],
canton=attributes.get("ak"),
geometry=geometry,
)
if zoneResults and len(zoneResults) > 0:
parcelInfo["bauzone"] = zoneResults[0].get("attributes", {}).get("typ_gde_abkuerzung")
except Exception as oe:
logger.debug(f"ÖREB zone query failed: {oe}")
return newParcelResponse
def _buildGeometryGeojson(extracted: Dict[str, Any], parcelInfo: Dict[str, Any]) -> Dict[str, Any]:
"""Build geometry_geojson from extracted perimeter."""
coords = []
if extracted.get("perimeter", {}).get("punkte"):
coords = [[[p["x"], p["y"]] for p in extracted["perimeter"]["punkte"]]]
return {
"type": "Feature",
"geometry": {"type": "Polygon", "coordinates": coords},
"properties": {"id": parcelInfo["id"], "egrid": parcelInfo["egrid"], "number": parcelInfo["number"]},
}
# ============================================================================
# ADD PARCEL TO PROJECT HANDLER
# ============================================================================
async def processAddParcelToProject(user, mandateId: Optional[str], projektId: str, body: Dict[str, Any]) -> Dict[str, Any]:
"""
Add a parcel to an existing project.
Supports linking existing, creating from location, or creating from custom data.
"""
realEstateInterface = getRealEstateInterface(user, mandateId=mandateId)
recordFilter = {"id": projektId}
if mandateId:
recordFilter["mandateId"] = mandateId
projekte = realEstateInterface.getProjekte(recordFilter=recordFilter)
if not projekte:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Projekt {projektId} not found"
)
projekt = projekte[0]
parcelId = body.get("parcelId")
location = body.get("location")
parcelDataDict = body.get("parcelData")
parzelle = None
if parcelId:
logger.info(f"Linking existing parcel {parcelId}")
parcelFilter = {"id": parcelId}
if mandateId:
parcelFilter["mandateId"] = mandateId
parcels = realEstateInterface.getParzellen(recordFilter=parcelFilter)
if not parcels:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Parzelle {parcelId} not found"
)
parzelle = parcels[0]
elif location:
logger.info(f"Creating parcel from location: {location}")
connector = SwissTopoMapServerConnector()
parcelData = await connector.search_parcel(location)
if not parcelData:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"No parcel found at location: {location}"
)
extractedAttributes = connector.extract_parcel_attributes(parcelData)
attributes = parcelData.get("attributes", {})
parzelleCreateData = {
"mandateId": mandateId,
"label": extractedAttributes.get("label") or attributes.get("number") or "Unknown",
"parzellenAliasTags": [attributes.get("egris_egrid")] if attributes.get("egris_egrid") else [],
"eigentuemerschaft": None,
"strasseNr": location if not location.replace(",", "").replace(".", "").replace(" ", "").isdigit() else None,
"plz": None,
"perimeter": extractedAttributes.get("perimeter"),
"baulinie": None,
"kontextGemeinde": None,
"bauzone": None,
"az": None,
"bz": None,
"vollgeschossZahl": None,
"anrechenbarDachgeschoss": None,
"anrechenbarUntergeschoss": None,
"gebaeudehoeheMax": None,
"regelnGrenzabstand": [],
"regelnMehrlaengenzuschlag": [],
"regelnMehrhoehenzuschlag": [],
"parzelleBebaut": None,
"parzelleErschlossen": None,
"parzelleHanglage": None,
"laermschutzzone": None,
"hochwasserschutzzone": None,
"grundwasserschutzzone": None,
"parzellenNachbarschaft": [],
"dokumente": [],
"kontextInformationen": [
Kontext(
thema="Swiss Topo Data",
inhalt=json.dumps({
"egrid": attributes.get("egris_egrid"),
"identnd": attributes.get("identnd"),
"canton": attributes.get("ak"),
"municipality_code": attributes.get("bfsnr"),
"geoportal_url": attributes.get("geoportal_url")
}, ensure_ascii=False)
)
]
}
parzelleInstance = Parzelle(**parzelleCreateData)
parzelle = realEstateInterface.createParzelle(parzelleInstance)
elif parcelDataDict:
logger.info(f"Creating parcel from custom data")
parcelDataDict["mandateId"] = mandateId
parzelleInstance = Parzelle(**parcelDataDict)
parzelle = realEstateInterface.createParzelle(parzelleInstance)
else:
raise ValueError("One of 'parcelId', 'location', or 'parcelData' is required")
if parzelle not in projekt.parzellen:
projekt.parzellen.append(parzelle)
if not projekt.perimeter and parzelle.perimeter:
projekt.perimeter = parzelle.perimeter
updatedProjekt = realEstateInterface.updateProjekt(projekt)
logger.info(f"Added Parzelle {parzelle.id} to Projekt {projektId}")
return {
"projekt": updatedProjekt.model_dump(),
"parzelle": parzelle.model_dump()
}

2900
modules/features/realEstate/mainRealEstate.py
View file

File diff suppressed because it is too large Load diff

1369
modules/features/realEstate/routeFeatureRealEstate.py
View file

File diff suppressed because it is too large Load diff

1087
modules/features/realEstate/serviceAiIntent.py Normal file
View file

File diff suppressed because it is too large Load diff

725
modules/features/realEstate/serviceBzo.py Normal file
View file

@ -0,0 +1,725 @@
"""
Real Estate feature — BZO (Bau- und Zonenordnung) information extraction.
Handles extraction of BZO information from PDF documents, filtering rules/zones/articles
by Bauzone, and generating AI summaries for building zone regulations.
"""
import logging
from typing import Optional, Dict, Any, List
from fastapi import HTTPException, status
from modules.datamodels.datamodelUam import User
from .datamodelFeatureRealEstate import DokumentTyp
from modules.serviceCenter.serviceHub import getInterface as getServices
from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
from modules.features.realEstate.bzoDocumentRetriever import BZODocumentRetriever
from modules.features.realEstate.bzoExtraction import run_extraction, run_bzo_params_extraction
from modules.features.realEstate.parcelSelectionService import compute_selection_summary
from modules.features.realEstate.realEstateGemeindeService import (
ensure_single_gemeinde,
fetch_bzo_for_gemeinde,
)
logger = logging.getLogger(__name__)
async def extract_bzo_information(
currentUser: User,
gemeinde: str,
bauzone: str,
mandateId: Optional[str] = None,
featureInstanceId: Optional[str] = None,
total_area_m2: Optional[float] = None,
parcels: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""
Extract BZO information from PDF documents for a specific Bauzone in a Gemeinde.
Retrieves BZO documents for the specified Gemeinde, extracts content using
the BZO extraction pipeline, filters by Bauzone, and uses AI to find relevant information.
When total_area_m2 or parcels are provided, runs Machbarkeitsstudie for structured output.
Args:
currentUser: Current authenticated user
gemeinde: Gemeinde name (e.g., "Zürich") or ID
bauzone: Bauzone code (e.g., "W3", "W2/30")
mandateId: Optional mandate ID for instance-scoped data (defaults to currentUser.mandateId)
featureInstanceId: Optional feature instance ID for instance-scoped data
total_area_m2: Optional total parcel area (m²) for Machbarkeitsstudie
parcels: Optional list of parcel dicts; total area computed via compute_selection_summary if not total_area_m2
Returns:
Dictionary containing:
- bauzone, gemeinde, extracted_content, ai_summary, relevant_rules, documents_processed
- machbarkeitsstudie: Structured Machbarkeitsstudie output when total_area_m2/parcels provided
"""
try:
_mandateId = mandateId or (str(currentUser.mandateId) if currentUser.mandateId else None)
logger.info(f"Extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}' (user: {currentUser.id}, mandate: {_mandateId})")
realEstateInterface = getRealEstateInterface(
currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId
)
componentInterface = getComponentInterface(
currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId
)
logger.debug(f"Attempting to retrieve Gemeinde '{gemeinde}' for mandate {_mandateId}")
gemeinde_obj = realEstateInterface.getGemeinde(gemeinde)
if not gemeinde_obj:
logger.debug(f"Gemeinde not found by ID, trying to search by label: {gemeinde}")
record_filter = {"label": gemeinde}
if _mandateId:
record_filter["mandateId"] = _mandateId
gemeinden_by_label = realEstateInterface.getGemeinden(
recordFilter=record_filter
)
if gemeinden_by_label and len(gemeinden_by_label) > 0:
gemeinde_obj = gemeinden_by_label[0]
logger.info(f"Found Gemeinde by label '{gemeinde}' with ID: {gemeinde_obj.id}")
if not gemeinde_obj and _mandateId and featureInstanceId:
logger.info(f"Gemeinde '{gemeinde}' not in DB - fetching from Swiss Topo (this Gemeinde only)")
gemeinde_obj = await ensure_single_gemeinde(
realEstateInterface, _mandateId, featureInstanceId, gemeinde_name=gemeinde
)
if not gemeinde_obj:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Gemeinde '{gemeinde}' not found or not accessible"
)
gemeinde_id = gemeinde_obj.id
bzo_documents = []
if gemeinde_obj.dokumente:
for doc in gemeinde_obj.dokumente:
if isinstance(doc, dict):
doc_id = doc.get("id")
doc_typ = doc.get("dokumentTyp")
else:
doc_id = doc.id if hasattr(doc, "id") else None
doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None
if doc_typ:
if isinstance(doc_typ, DokumentTyp):
is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION]
elif isinstance(doc_typ, str):
is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"]
else:
doc_typ_str = str(doc_typ)
is_bzo = doc_typ_str in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"]
if is_bzo:
if doc_id:
full_doc = realEstateInterface.getDokument(doc_id)
if full_doc:
bzo_documents.append(full_doc)
else:
logger.warning(f"Document {doc_id} referenced in Gemeinde but not found in database")
if not bzo_documents and _mandateId and featureInstanceId:
logger.info(f"No BZO documents for Gemeinde '{gemeinde_obj.label}' - fetching from web")
fetched = await fetch_bzo_for_gemeinde(
realEstateInterface, componentInterface, gemeinde_obj, _mandateId, featureInstanceId
)
if fetched:
gemeinde_obj = realEstateInterface.getGemeinde(gemeinde_obj.id)
bzo_documents = []
if gemeinde_obj and gemeinde_obj.dokumente:
for doc in gemeinde_obj.dokumente:
if isinstance(doc, dict):
doc_id = doc.get("id")
doc_typ = doc.get("dokumentTyp")
else:
doc_id = doc.id if hasattr(doc, "id") else None
doc_typ = doc.dokumentTyp if hasattr(doc, "dokumentTyp") else None
if doc_typ:
if isinstance(doc_typ, DokumentTyp):
is_bzo = doc_typ in [DokumentTyp.GEMEINDE_BZO_AKTUELL, DokumentTyp.GEMEINDE_BZO_REVISION]
elif isinstance(doc_typ, str):
is_bzo = doc_typ in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"]
else:
is_bzo = str(doc_typ) in ["gemeindeBzoAktuell", "gemeindeBzoRevision", "GEMEINDE_BZO_AKTUELL", "GEMEINDE_BZO_REVISION"]
if is_bzo and doc_id:
full_doc = realEstateInterface.getDokument(doc_id)
if full_doc:
bzo_documents.append(full_doc)
if not bzo_documents:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"No BZO documents found for Gemeinde '{gemeinde_obj.label}'"
)
logger.info(f"Found {len(bzo_documents)} BZO document(s) for Gemeinde '{gemeinde_obj.label}'")
document_retriever = BZODocumentRetriever(realEstateInterface, componentInterface)
all_extracted_content = {
"articles": [],
"zones": [],
"rules": [],
"zone_parameter_tables": [],
"errors": [],
"warnings": []
}
documents_processed = []
for dokument in bzo_documents:
try:
logger.info(f"Processing document {dokument.id}: {dokument.label}")
pdf_bytes = document_retriever.retrieve_pdf_content(dokument)
if not pdf_bytes:
logger.warning(f"Could not retrieve PDF content for dokument {dokument.id}")
all_extracted_content["warnings"].append(
f"Could not retrieve PDF content for document '{dokument.label}'"
)
continue
extraction_result = run_extraction(
pdf_bytes=pdf_bytes,
pdf_id=dokument.dokumentReferenz or f"dok_{dokument.id}",
dokument_id=dokument.id
)
all_extracted_content["articles"].extend(extraction_result.get("articles", []))
all_extracted_content["zones"].extend(extraction_result.get("zones", []))
all_extracted_content["rules"].extend(extraction_result.get("rules", []))
all_extracted_content["zone_parameter_tables"].extend(extraction_result.get("zone_parameter_tables", []))
all_extracted_content["errors"].extend(extraction_result.get("errors", []))
all_extracted_content["warnings"].extend(extraction_result.get("warnings", []))
documents_processed.append({
"id": dokument.id,
"label": dokument.label,
"dokumentTyp": dokument.dokumentTyp.value if dokument.dokumentTyp else None
})
except Exception as e:
logger.error(f"Error processing document {dokument.id}: {str(e)}", exc_info=True)
all_extracted_content["errors"].append(
f"Error processing document '{dokument.label}': {str(e)}"
)
continue
relevant_rules = filter_rules_by_bauzone(
all_extracted_content["rules"],
bauzone
)
logger.info(f"Extracting for Bauzone {bauzone}: {len(relevant_rules)} zone-specific rules, "
f"{len([t for t in all_extracted_content.get('zone_parameter_tables', []) if bauzone.upper() in str(t.get('zones', [])).upper()])} tables with zone data")
relevant_zones = filter_zones_by_bauzone(
all_extracted_content["zones"],
bauzone
)
relevant_articles = filter_articles_by_bauzone(
all_extracted_content.get("articles", []),
bauzone
)
_total_area_m2 = total_area_m2
if _total_area_m2 is None and parcels:
selection_summary = compute_selection_summary(parcels)
_total_area_m2 = selection_summary.get("total_area_m2") or 0.0
bzo_params_result = None
try:
services = getServices(
currentUser, workflow=None, mandateId=_mandateId, featureInstanceId=featureInstanceId
)
ai_service = services.ai
bzo_params_result = await run_bzo_params_extraction(
extracted_content=all_extracted_content,
bauzone=bauzone,
ai_service=ai_service,
gemeinde=gemeinde_obj.label,
relevant_rules=relevant_rules,
relevant_articles=relevant_articles,
total_area_m2=_total_area_m2,
)
except Exception as me:
logger.warning(f"BZO parameter extraction failed: {me}", exc_info=True)
all_extracted_content["warnings"] = all_extracted_content.get("warnings", []) + [
f"BZO-Parameter konnten nicht extrahiert werden: {str(me)}"
]
ai_summary = await generate_bauzone_ai_summary(
currentUser=currentUser,
bauzone=bauzone,
gemeinde=gemeinde_obj.label,
extracted_content=all_extracted_content,
relevant_rules=relevant_rules,
relevant_zones=relevant_zones,
mandateId=_mandateId,
featureInstanceId=featureInstanceId,
)
unified_summary = ai_summary
summary_lower = unified_summary.lower()
zones_mentioned = any(zone.get("zone_code", "").upper() in summary_lower for zone in relevant_zones)
if not zones_mentioned and relevant_zones:
unified_summary += "\n\n=== ZONENDEFINITIONEN ===\n"
for zone in relevant_zones:
zone_code = zone.get("zone_code", "")
zone_name = zone.get("zone_name", "")
zone_category = zone.get("zone_category", "")
geschosszahl = zone.get("geschosszahl")
gewerbeerleichterung = zone.get("gewerbeerleichterung", False)
page_num = zone.get("page", 0)
source_article = zone.get("source_article", "")
zone_info = f"{zone_code}: {zone_name}"
if zone_category:
zone_info += f"\nKategorie: {zone_category}"
if geschosszahl:
zone_info += f"\nGeschosszahl: {geschosszahl}"
if gewerbeerleichterung:
zone_info += "\nGewerbeerleichterung: Ja"
if source_article:
zone_info += f"\nQuelle: {source_article} (Seite {page_num})"
unified_summary += zone_info + "\n\n"
articles_mentioned = any(article.get("article_label", "") in summary_lower for article in relevant_articles)
if not articles_mentioned and relevant_articles:
unified_summary += "\n\n=== RELEVANTE ARTIKEL ===\n"
for article in relevant_articles:
article_label = article.get("article_label", "")
article_title = article.get("article_title", "")
article_text = article.get("text", "")
page_start = article.get("page_start", 0)
page_end = article.get("page_end", 0)
page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}"
unified_summary += f"{article_label}"
if article_title:
unified_summary += f": {article_title}"
unified_summary += f" ({page_range})\n"
if article_text:
preview = article_text[:500] + "..." if len(article_text) > 500 else article_text
unified_summary += f"{preview}\n\n"
return {
"bauzone": bauzone,
"gemeinde": {
"id": gemeinde_obj.id,
"label": gemeinde_obj.label,
"plz": gemeinde_obj.plz
},
"extracted_content": {
"zones": relevant_zones,
"rules": relevant_rules,
"articles": relevant_articles,
"zone_parameter_tables": _filter_tables_by_bauzone(
all_extracted_content.get("zone_parameter_tables", []),
bauzone
),
"total_zones": len(all_extracted_content.get("zones", [])),
"total_rules": len(all_extracted_content.get("rules", [])),
"total_articles": len(all_extracted_content.get("articles", [])),
"total_tables": len(all_extracted_content.get("zone_parameter_tables", []))
},
"ai_summary": unified_summary,
"relevant_rules": relevant_rules,
"documents_processed": documents_processed,
"errors": all_extracted_content.get("errors", []),
"warnings": all_extracted_content.get("warnings", []),
"machbarkeitsstudie": bzo_params_result,
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error extracting BZO information for Gemeinde '{gemeinde}', Bauzone '{bauzone}': {str(e)}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error extracting BZO information: {str(e)}"
)
def filter_rules_by_bauzone(rules: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]:
"""
Filter rules by Bauzone code. Only keeps rules from SINGLE-zone articles to avoid
wrong values (e.g. article with W2,W3,W5 has different values per zone - we cannot
associate a rule value with a specific zone from article text alone).
"""
relevant_rules = []
bauzone_upper = bauzone.upper()
def _zone_matches(z: str) -> bool:
zu = (z or "").upper().strip()
if not zu:
return False
if bauzone_upper in zu:
return True
if zu in bauzone_upper and len(zu) >= 2:
return True
return False
for rule in rules:
table_zones = rule.get("table_zones", []) or []
zone_raw = rule.get("zone_raw")
has_zone = bool(zone_raw) or bool(table_zones)
if not has_zone:
continue
if len(table_zones) > 1:
matches_all = all(_zone_matches(str(z)) for z in table_zones)
if not matches_all:
continue
matches = False
if zone_raw and _zone_matches(zone_raw):
matches = True
if not matches and table_zones:
for tz in table_zones:
if _zone_matches(str(tz)):
matches = True
break
if not matches:
ts = (rule.get("text_snippet") or "").upper()
if bauzone_upper in ts and len(table_zones) <= 1:
matches = True
if matches:
relevant_rules.append(rule)
logger.info(f"Filtered {len(relevant_rules)} rules for Bauzone {bauzone} from {len(rules)} total (multi-zone articles excluded)")
return relevant_rules
def filter_zones_by_bauzone(zones: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]:
"""
Filter zones by Bauzone code.
Args:
zones: List of zone dictionaries from extraction
bauzone: Bauzone code to filter by
Returns:
Filtered list of zones that match the Bauzone
"""
relevant_zones = []
bauzone_upper = bauzone.upper()
for zone in zones:
zone_code = zone.get("zone_code", "")
if bauzone_upper in zone_code.upper():
relevant_zones.append(zone)
logger.info(f"Filtered {len(relevant_zones)} zones for Bauzone {bauzone} from {len(zones)} total zones")
return relevant_zones
def filter_articles_by_bauzone(articles: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]:
"""
Filter articles that mention the Bauzone.
Args:
articles: List of article dictionaries from extraction
bauzone: Bauzone code to filter by
Returns:
Filtered list of articles that mention the Bauzone
"""
relevant_articles = []
bauzone_upper = bauzone.upper()
for article in articles:
text = article.get("text", "")
zone_raw = article.get("zone_raw")
text_matches = bauzone_upper in text.upper() if text else False
zone_matches = bauzone_upper in zone_raw.upper() if zone_raw else False
if text_matches or zone_matches:
relevant_articles.append(article)
logger.info(f"Filtered {len(relevant_articles)} articles for Bauzone {bauzone} from {len(articles)} total articles")
return relevant_articles
def _filter_tables_by_bauzone(tables: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]:
"""
Filter zone-parameter tables to include only those containing the specified Bauzone.
Args:
tables: List of zone-parameter table dictionaries
bauzone: Bauzone code to filter by
Returns:
Filtered list of tables containing the Bauzone
"""
relevant_tables = []
bauzone_upper = bauzone.upper()
for table in tables:
zones = table.get("zones", [])
matching_zones = [z for z in zones if bauzone_upper in str(z).upper()]
if matching_zones:
filtered_table = {
"page": table.get("page"),
"zones": matching_zones,
"parameters": []
}
for param in table.get("parameters", []):
values_by_zone = param.get("values_by_zone", {})
filtered_values = {
zone: values_by_zone[zone]
for zone in matching_zones
if zone in values_by_zone
}
if filtered_values:
filtered_table["parameters"].append({
"parameter": param.get("parameter"),
"values_by_zone": filtered_values
})
if filtered_table["parameters"]:
relevant_tables.append(filtered_table)
logger.info(f"Filtered {len(relevant_tables)} tables for Bauzone {bauzone} from {len(tables)} total tables")
return relevant_tables
async def generate_bauzone_ai_summary(
currentUser: User,
bauzone: str,
gemeinde: str,
extracted_content: Dict[str, Any],
relevant_rules: List[Dict[str, Any]],
relevant_zones: List[Dict[str, Any]],
mandateId: Optional[str] = None,
featureInstanceId: Optional[str] = None,
) -> str:
"""
Use AI to generate a summary of relevant information for a Bauzone.
Args:
currentUser: Current authenticated user
bauzone: Bauzone code
gemeinde: Gemeinde name
extracted_content: All extracted content from PDFs
relevant_rules: Rules filtered by Bauzone
relevant_zones: Zones filtered by Bauzone
Returns:
AI-generated summary string
"""
try:
services = getServices(
currentUser, workflow=None, mandateId=mandateId, featureInstanceId=featureInstanceId
)
aiService = services.ai
context_parts = []
zone_parameter_tables = extracted_content.get("zone_parameter_tables", [])
table_values_for_bauzone = []
if zone_parameter_tables:
context_parts.append("=== BUILDING REGULATIONS TABLE VALUES FOR BAUZONE (INCLUDE THESE EXACT VALUES IN YOUR SUMMARY) ===")
for table in zone_parameter_tables:
page_num = table.get("page", 0)
article_ref = table.get("article", "Unknown article")
zones_in_table = table.get("zones", [])
matching_zones = [z for z in zones_in_table if bauzone.upper() in str(z).upper()]
if matching_zones:
context_parts.append(f"\nTabelle aus {article_ref} (Seite {page_num}):")
for param in table.get("parameters", []):
param_name = param.get("parameter", "")
values_by_zone = param.get("values_by_zone", {})
for zone, values in values_by_zone.items():
if bauzone.upper() in zone.upper():
if isinstance(values, list) and len(values) > 0:
val_entry = values[0]
value = val_entry.get("value", "")
unit = val_entry.get("unit", "")
unit_str = f" {unit}" if unit else ""
formatted_param = param_name
if "Ausnützungsziffer" in param_name or "ausnützungsziffer" in param_name.lower():
formatted_param = "Ausnützungsziffer max."
elif "Vollgeschosse" in param_name or "vollgeschosse" in param_name.lower():
formatted_param = "Vollgeschosse max."
elif "Gebäudelänge" in param_name or "gebäudelänge" in param_name.lower():
formatted_param = "Gebäudelänge max."
elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Grundabstand" in param_name or "grundabstand" in param_name.lower()):
formatted_param = "Grenzabstand - Grundabstand min."
elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Mehrlängen" in param_name or "mehrlängen" in param_name.lower()):
formatted_param = "Grenzabstand - Mehrlängen-zuschlag"
elif ("Grenzabstand" in param_name or "grenzabstand" in param_name.lower()) and ("Höchstmass" in param_name or "höchstmass" in param_name.lower() or "Höchstmaß" in param_name):
formatted_param = "Grenzabstand - Höchstmass max."
elif "Fassadenhöhen" in param_name or "fassadenhöhen" in param_name.lower():
formatted_param = "Fassadenhöhen max."
elif "Dachgeschosse" in param_name or "dachgeschosse" in param_name.lower():
formatted_param = "anrechenbare Dachgeschosse max."
elif "Attikageschoss" in param_name or "attikageschoss" in param_name.lower():
formatted_param = "anrechenbares Attikageschoss max."
elif "Untergeschoss" in param_name or "untergeschoss" in param_name.lower():
formatted_param = "anrechenbares Untergeschoss max."
table_values_for_bauzone.append({
"parameter": formatted_param,
"value": value,
"unit": unit_str,
"article": article_ref,
"page": page_num
})
context_parts.append(f" • {formatted_param}: {value}{unit_str} (Quelle: {article_ref}, Seite {page_num})")
if len(values) > 1:
for idx, val_entry in enumerate(values[1:], 1):
value_extra = val_entry.get("value", "")
unit_extra = val_entry.get("unit", "")
unit_str_extra = f" {unit_extra}" if unit_extra else ""
context_parts.append(f" (Alternative: {value_extra}{unit_str_extra})")
if relevant_zones:
context_parts.append("\n=== ZONE DEFINITIONS ===")
for zone in relevant_zones:
zone_code = zone.get("zone_code", "")
zone_name = zone.get("zone_name", "")
zone_category = zone.get("zone_category", "")
geschosszahl = zone.get("geschosszahl")
gewerbeerleichterung = zone.get("gewerbeerleichterung", False)
page_num = zone.get("page", 0)
source_article = zone.get("source_article", "")
zone_info = f"- {zone_code}: {zone_name}"
if zone_category:
zone_info += f" (Kategorie: {zone_category})"
if geschosszahl:
zone_info += f", Geschosszahl: {geschosszahl}"
if gewerbeerleichterung:
zone_info += ", Gewerbeerleichterung: Ja"
if source_article:
zone_info += f" - Quelle: {source_article} (Seite {page_num})"
context_parts.append(zone_info)
relevant_articles = filter_articles_by_bauzone(extracted_content.get("articles", []), bauzone)
if relevant_articles:
context_parts.append("\n=== RELEVANT ARTICLES (full content) ===")
for article in relevant_articles:
article_label = article.get("article_label", "")
article_title = article.get("article_title", "")
article_text = article.get("text", "")
page_start = article.get("page_start", 0)
page_end = article.get("page_end", 0)
page_range = f"Seite {page_start}" if page_start == page_end else f"Seiten {page_start}-{page_end}"
context_parts.append(f"\n{article_label}: {article_title or 'Kein Titel'}")
context_parts.append(f"Lage: {page_range}")
if len(article_text) > 1000:
context_parts.append(f"Inhalt: {article_text[:1000]}...")
else:
context_parts.append(f"Inhalt: {article_text}")
if relevant_rules:
table_parameter_names = set()
for table in zone_parameter_tables:
for param in table.get("parameters", []):
param_name = param.get("parameter", "").lower()
table_parameter_names.add(param_name)
unique_rules = []
for rule in relevant_rules[:15]:
rule_type = rule.get("rule_type", "").lower()
if not any(tp in rule_type for tp in table_parameter_names):
unique_rules.append(rule)
if unique_rules:
context_parts.append("\n=== ADDITIONAL BUILDING REGULATIONS (from text) ===")
for rule in unique_rules[:8]:
rule_type = rule.get("rule_type", "")
value_numeric = rule.get("value_numeric")
value_text = rule.get("value_text", "")
unit = rule.get("unit", "")
page_num = rule.get("page", 0)
rule_desc = f"- {rule_type}: "
if value_numeric is not None:
rule_desc += f"{value_numeric}"
if unit:
rule_desc += f" {unit}"
else:
rule_desc += value_text
rule_desc += f" (Seite {page_num})"
context_parts.append(rule_desc)
context = "\n".join(context_parts)
prompt = f"""
Analyze the following building zone (Bauzone) information extracted from BZO (Bau- und Zonenordnung) documents for {gemeinde}, specifically for Bauzone {bauzone}.
Extracted Content:
{context}
CRITICAL INSTRUCTIONS:
1. You MUST include ALL actual values from the tables in your summary - do NOT just say "see tables on page X"
2. List ALL parameters with their actual values: Ausnützungsziffer, Vollgeschosse, Gebäudelänge, Grenzabstand (Grundabstand, Mehrlängen-zuschlag, Höchstmass), Fassadenhöhen, etc.
3. Integrate zone definitions and article information INTO the summary text - do NOT create separate sections
4. Always cite WHERE each piece of information was found (article number and page number)
5. Combine everything into ONE unified, flowing summary - no separate sections for zones/articles
6. Be comprehensive - include all relevant details from zones, articles, and tables
7. Format as a single, well-structured German text document
Please provide a comprehensive, unified summary that includes:
1. General description of Bauzone {bauzone}:
- Zone category (Wohnzonen, Zentrumszonen, etc.)
- Geschosszahl (number of full storeys)
- Gewerbeerleichterung status (Ja/Nein)
- Where defined (article and page number)
2. ALL building regulations with ACTUAL VALUES from tables (you MUST include the exact values):
- Ausnützungsziffer max.: [ACTUAL PERCENTAGE VALUE]% (from article, page)
- Vollgeschosse max.: [ACTUAL NUMBER] (from article, page)
- anrechenbare Dachgeschosse max.: [ACTUAL NUMBER] (from article, page)
- anrechenbares Attikageschoss max.: [ACTUAL NUMBER] (from article, page)
- anrechenbares Untergeschoss max.: [ACTUAL NUMBER] (from article, page)
- Gebäudelänge max.: [ACTUAL VALUE] m (from article, page)
- Grenzabstand - Grundabstand min.: [ACTUAL VALUE] m (from article, page)
- Grenzabstand - Mehrlängen-zuschlag: [ACTUAL FRACTION] (from article, page)
- Grenzabstand - Höchstmass max.: [ACTUAL VALUE] m (from article, page)
- Fassadenhöhen max.: [ACTUAL VALUE] m (from article, page, include footnote values if present)
3. Zone definitions: Integrate information about where this zone is defined (which articles mention it, with page numbers)
4. Relevant articles: Integrate key content from relevant articles naturally into the summary, citing article numbers and page numbers
5. Special conditions: Any special requirements or exceptions mentioned in articles
CRITICAL: You MUST include the actual numeric values from the tables in your summary. Do NOT say "see tables" - list the actual values. Format everything as ONE unified, flowing German text document without separate sections. Integrate zones and articles naturally into the narrative.
"""
logger.info(f"Generating AI summary for Bauzone {bauzone} in {gemeinde}")
ai_response = await aiService.callAiPlanning(
prompt=prompt,
debugType="bzo_summary"
)
return ai_response.strip()
except Exception as e:
logger.error(f"Error generating AI summary: {str(e)}", exc_info=True)
return f"Summary generation failed: {str(e)}. Found {len(relevant_rules)} relevant rules and {len(relevant_zones)} zones for Bauzone {bauzone}."

817
modules/features/realEstate/serviceGeometry.py Normal file
View file

@ -0,0 +1,817 @@
"""
Real Estate feature — Geometry utilities.
Handles conversion between GeoPolylinie and Shapely polygons, combining
parcel geometries, filtering neighbor parcels, fetching parcel polygons
from Swisstopo, creating projects with parcel data, and GeoJSON conversion.
"""
import logging
from typing import Optional, Dict, Any, List
from shapely.geometry import Polygon
from shapely.ops import unary_union
from .datamodelFeatureRealEstate import (
Projekt,
Parzelle,
StatusProzess,
GeoPolylinie,
GeoPunkt,
Kontext,
Gemeinde,
Kanton,
Land,
)
from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector
from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface
from modules.datamodels.datamodelUam import User
from fastapi import HTTPException, status
logger = logging.getLogger(__name__)
def geopolylinie_to_shapely_polygon(geopolylinie: GeoPolylinie) -> Polygon:
"""
Convert GeoPolylinie to Shapely Polygon.
Args:
geopolylinie: GeoPolylinie instance with punkte list
Returns:
Shapely Polygon object
"""
if not geopolylinie or not geopolylinie.punkte:
raise ValueError("GeoPolylinie must have at least one point")
coordinates = []
for punkt in geopolylinie.punkte:
coordinates.append((punkt.x, punkt.y))
if len(coordinates) < 3:
raise ValueError("Polygon must have at least 3 points")
if coordinates[0] != coordinates[-1]:
coordinates.append(coordinates[0])
return Polygon(coordinates)
def shapely_polygon_to_geopolylinie(polygon: Polygon) -> GeoPolylinie:
"""
Convert Shapely Polygon to GeoPolylinie.
Args:
polygon: Shapely Polygon object
Returns:
GeoPolylinie instance with LV95 coordinate system
"""
if not polygon or polygon.is_empty:
raise ValueError("Polygon must not be empty")
exterior_coords = list(polygon.exterior.coords)
if len(exterior_coords) > 1 and exterior_coords[0] == exterior_coords[-1]:
exterior_coords = exterior_coords[:-1]
punkte = []
for coord in exterior_coords:
punkt = GeoPunkt(
koordinatensystem="LV95",
x=float(coord[0]),
y=float(coord[1]),
z=None
)
punkte.append(punkt)
return GeoPolylinie(
closed=True,
punkte=punkte
)
def combine_parcel_geometries(geometries: List[GeoPolylinie]) -> GeoPolylinie:
"""
Combine multiple parcel geometries into a single outer outline.
Uses Shapely union operation to merge polygons and automatically
removes internal edges. The result is a clean outer boundary.
Args:
geometries: List of GeoPolylinie instances to combine
Returns:
Combined GeoPolylinie representing the outer outline
Raises:
ValueError: If geometries list is empty or invalid
"""
if not geometries or len(geometries) == 0:
raise ValueError("At least one geometry is required")
if len(geometries) == 1:
return geometries[0]
shapely_polygons = []
for geo in geometries:
try:
polygon = geopolylinie_to_shapely_polygon(geo)
if not polygon.is_empty:
shapely_polygons.append(polygon)
except Exception as e:
logger.warning(f"Error converting geometry to Shapely Polygon: {e}")
continue
if not shapely_polygons:
raise ValueError("No valid geometries to combine")
if len(shapely_polygons) == 1:
return shapely_polygon_to_geopolylinie(shapely_polygons[0])
try:
combined = unary_union(shapely_polygons)
if hasattr(combined, 'geoms'):
largest = max(combined.geoms, key=lambda p: p.area)
combined = largest
if combined.is_empty:
raise ValueError("Union resulted in empty geometry")
result = shapely_polygon_to_geopolylinie(combined)
logger.info(f"Combined {len(geometries)} geometries into single outline with {len(result.punkte)} points")
return result
except Exception as e:
logger.error(f"Error combining geometries: {e}", exc_info=True)
raise ValueError(f"Failed to combine geometries: {str(e)}")
def filter_neighbor_parcels(
neighbors: List[Dict[str, Any]],
selected_geometries: List[GeoPolylinie]
) -> List[Dict[str, Any]]:
"""
Filter neighbor parcels to exclude those that are part of the selected parcels.
Uses geometric comparison to check if neighbor parcels intersect or touch
any of the selected parcel geometries.
Args:
neighbors: List of neighbor parcel dictionaries (must have 'perimeter' or 'geometry_geojson')
selected_geometries: List of GeoPolylinie instances representing selected parcels
Returns:
Filtered list of neighbor parcels (excluding selected ones)
"""
if not neighbors or not selected_geometries:
return neighbors
selected_polygons = []
for geo in selected_geometries:
try:
polygon = geopolylinie_to_shapely_polygon(geo)
if not polygon.is_empty:
selected_polygons.append(polygon)
except Exception as e:
logger.warning(f"Error converting selected geometry for filtering: {e}")
continue
if not selected_polygons:
return neighbors
filtered_neighbors = []
for neighbor in neighbors:
try:
neighbor_geometry = None
if neighbor.get("perimeter"):
perimeter = neighbor["perimeter"]
if isinstance(perimeter, dict) and perimeter.get("punkte"):
punkte = []
for p in perimeter["punkte"]:
punkt = GeoPunkt(
koordinatensystem=p.get("koordinatensystem", "LV95"),
x=float(p.get("x", 0)),
y=float(p.get("y", 0)),
z=p.get("z")
)
punkte.append(punkt)
neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte)
elif neighbor.get("geometry_geojson"):
geo_json = neighbor["geometry_geojson"]
geometry = geo_json.get("geometry") if isinstance(geo_json, dict) else geo_json
if geometry and geometry.get("type") == "Polygon":
coordinates = geometry.get("coordinates", [])
if coordinates and len(coordinates) > 0:
ring = coordinates[0]
punkte = []
for coord in ring:
if len(coord) >= 2:
punkt = GeoPunkt(
koordinatensystem="LV95",
x=float(coord[0]),
y=float(coord[1]),
z=float(coord[2]) if len(coord) > 2 else None
)
punkte.append(punkt)
neighbor_geometry = GeoPolylinie(closed=True, punkte=punkte)
if not neighbor_geometry:
filtered_neighbors.append(neighbor)
continue
neighbor_polygon = geopolylinie_to_shapely_polygon(neighbor_geometry)
is_selected = False
for selected_polygon in selected_polygons:
if neighbor_polygon.intersects(selected_polygon) or neighbor_polygon.touches(selected_polygon):
area_diff = abs(neighbor_polygon.area - selected_polygon.area)
if area_diff < 1.0:
is_selected = True
break
if neighbor_polygon.contains(selected_polygon) or selected_polygon.contains(neighbor_polygon):
is_selected = True
break
if not is_selected:
filtered_neighbors.append(neighbor)
else:
logger.debug(f"Filtered out neighbor parcel {neighbor.get('id')} - part of selected parcels")
except Exception as e:
logger.warning(f"Error filtering neighbor parcel {neighbor.get('id')}: {e}")
filtered_neighbors.append(neighbor)
logger.info(f"Filtered {len(neighbors)} neighbors to {len(filtered_neighbors)} (removed {len(neighbors) - len(filtered_neighbors)} selected parcels)")
return filtered_neighbors
async def fetch_parcel_polygon_from_swisstopo(
gemeinde: str,
parzellen_nr: str,
sr: int = 2056
) -> Optional[Dict[str, Any]]:
"""
Holt die vollständige Polygon-Geometrie einer Parzelle von Swisstopo API.
Args:
gemeinde: Name der Gemeinde (z.B. "Bern")
parzellen_nr: Parzellennummer (z.B. "1234")
sr: Koordinatensystem (2056=LV95, 4326=WGS84)
Returns:
Dictionary mit GeoPolylinie-Format für perimeter-Feld, oder None wenn nicht gefunden
Format: {"closed": True, "punkte": [{"koordinatensystem": "LV95", "x": ..., "y": ..., "z": None}, ...]}
"""
try:
connector = SwissTopoMapServerConnector()
feature = await connector.get_parcel_polygon(gemeinde, parzellen_nr, sr)
if not feature:
logger.warning(f"Parzelle {gemeinde} {parzellen_nr} nicht gefunden in Swisstopo")
return None
geometry = feature.get("geometry", {})
if geometry.get("type") == "Polygon":
coordinates = geometry.get("coordinates", [])
if coordinates and len(coordinates) > 0:
ring = coordinates[0]
punkte = []
for coord in ring:
if len(coord) >= 2:
punkt = {
"koordinatensystem": "LV95" if sr == 2056 else "WGS84",
"x": coord[0],
"y": coord[1],
"z": coord[2] if len(coord) > 2 else None
}
punkte.append(punkt)
logger.info(f"Successfully fetched polygon with {len(punkte)} points for {gemeinde} {parzellen_nr}")
return {
"closed": True,
"punkte": punkte
}
logger.warning(f"Unexpected geometry type in Swisstopo response: {geometry.get('type')}")
return None
except Exception as e:
logger.error(f"Error fetching parcel polygon from Swisstopo: {e}", exc_info=True)
return None
def convert_geojson_to_geopolylinie(geometry_data: Dict[str, Any]) -> Optional[GeoPolylinie]:
"""Convert GeoJSON geometry to GeoPolylinie format."""
if not geometry_data:
return None
if "geometry" in geometry_data:
geometry_data = geometry_data["geometry"]
geometry_type = geometry_data.get("type")
coordinates = geometry_data.get("coordinates")
if not coordinates or geometry_type != "Polygon":
return None
if not coordinates or len(coordinates) == 0:
return None
ring = coordinates[0]
punkte = []
for coord in ring:
if len(coord) >= 2:
punkt = GeoPunkt(
koordinatensystem="LV95",
x=float(coord[0]),
y=float(coord[1]),
z=float(coord[2]) if len(coord) > 2 else None
)
punkte.append(punkt)
if not punkte:
return None
return GeoPolylinie(
closed=True,
punkte=punkte
)
async def create_project_with_parcel_data(
currentUser: User,
mandateId: str,
projekt_label: str,
parzellen_data: List[Dict[str, Any]],
status_prozess: Optional[str] = None,
) -> Dict[str, Any]:
"""
Create a Projekt with one or more Parzellen from provided parcel data.
Args:
currentUser: Current authenticated user
mandateId: Mandate context (from RequestContext / X-Mandate-Id header)
projekt_label: Label for the Projekt
parzellen_data: List of dictionaries containing parcel information from request
status_prozess: Optional project status (defaults to "Eingang")
Returns:
Dictionary containing created Projekt and list of Parzellen
Raises:
HTTPException: If Gemeinde or Kanton not found, or validation fails
"""
try:
logger.info(f"Creating project '{projekt_label}' with {len(parzellen_data)} parcel(s) for user {currentUser.id}")
realEstateInterface = getRealEstateInterface(currentUser, mandateId=mandateId)
if not projekt_label:
raise ValueError("Projekt label is required")
if not parzellen_data or len(parzellen_data) == 0:
raise ValueError("At least one Parzelle data is required")
for idx, parzelle_data in enumerate(parzellen_data):
if not parzelle_data.get("perimeter"):
raise ValueError(f"Parzelle {idx + 1} perimeter is required")
# First pass: Collect all parcel geometries for neighbor filtering
all_parcel_geometries = []
for parzelle_data in parzellen_data:
perimeter = parzelle_data.get("perimeter")
if perimeter:
if isinstance(perimeter, dict):
if "punkte" in perimeter and "closed" in perimeter:
try:
geo_perimeter = GeoPolylinie(**perimeter)
all_parcel_geometries.append(geo_perimeter)
except Exception as e:
logger.warning(f"Error converting perimeter to GeoPolylinie: {e}")
else:
converted = convert_geojson_to_geopolylinie(perimeter)
if converted:
all_parcel_geometries.append(converted)
elif isinstance(perimeter, GeoPolylinie):
all_parcel_geometries.append(perimeter)
created_parzellen = []
parcel_perimeters = []
for idx, parzelle_data in enumerate(parzellen_data):
logger.info(f"Processing Parzelle {idx + 1}/{len(parzellen_data)}")
parcel_label = parzelle_data.get("id") or parzelle_data.get("number") or parzelle_data.get("label") or "Unknown"
existing_parzellen = realEstateInterface.getParzellen(
recordFilter={"label": parcel_label, "mandateId": mandateId}
)
if existing_parzellen and len(existing_parzellen) > 0:
existing_parzelle = existing_parzellen[0]
logger.info(f"Parzelle with label '{parcel_label}' already exists (ID: {existing_parzelle.id}), reusing it")
if existing_parzelle.perimeter:
parcel_perimeters.append(existing_parzelle.perimeter)
created_parzellen.append(existing_parzelle)
continue
logger.info(f"Parzelle with label '{parcel_label}' does not exist, creating new one")
gemeinde_id = None
canton_abk = parzelle_data.get("canton")
municipality_name = parzelle_data.get("municipality_name")
logger.debug(f"Resolving Gemeinde/Kanton: canton='{canton_abk}', municipality='{municipality_name}'")
if municipality_name and canton_abk:
canton_names = {
"ZH": "Zürich", "BE": "Bern", "LU": "Luzern", "UR": "Uri", "SZ": "Schwyz",
"OW": "Obwalden", "NW": "Nidwalden", "GL": "Glarus", "ZG": "Zug", "FR": "Freiburg",
"SO": "Solothurn", "BS": "Basel-Stadt", "BL": "Basel-Landschaft", "SH": "Schaffhausen",
"AR": "Appenzell Ausserrhoden", "AI": "Appenzell Innerrhoden", "SG": "St. Gallen",
"GR": "Graubünden", "AG": "Aargau", "TG": "Thurgau", "TI": "Tessin",
"VD": "Waadt", "VS": "Wallis", "NE": "Neuenburg", "GE": "Genf", "JU": "Jura"
}
logger.debug("Ensuring Land 'Schweiz' exists")
laender = realEstateInterface.getLaender(recordFilter={"label": "Schweiz"})
if not laender:
logger.info("Creating Land 'Schweiz'")
land = Land(
mandateId=mandateId,
label="Schweiz",
abk="CH"
)
land = realEstateInterface.createLand(land)
logger.info(f"Created Land 'Schweiz' with ID: {land.id}")
else:
land = laender[0]
logger.debug(f"Found Land 'Schweiz' with ID: {land.id}")
logger.debug(f"Looking up Kanton with abk='{canton_abk}'")
kantone = realEstateInterface.getKantone(recordFilter={"abk": canton_abk})
logger.debug(f"Found {len(kantone)} Kanton(e) with abk='{canton_abk}'")
if not kantone:
logger.info(f"Kanton '{canton_abk}' not found, creating it")
kanton_label = canton_names.get(canton_abk, canton_abk)
kanton = Kanton(
mandateId=mandateId,
label=kanton_label,
abk=canton_abk,
id_land=land.id
)
kanton = realEstateInterface.createKanton(kanton)
logger.info(f"Created Kanton '{kanton_label}' ({canton_abk}) with ID: {kanton.id}")
else:
kanton = kantone[0]
logger.debug(f"Found Kanton: ID={kanton.id}, Label={kanton.label}, abk={kanton.abk}")
logger.debug(f"Looking up Gemeinde with label='{municipality_name}' and id_kanton='{kanton.id}'")
gemeinden = realEstateInterface.getGemeinden(
recordFilter={"label": municipality_name, "id_kanton": kanton.id}
)
logger.debug(f"Found {len(gemeinden)} Gemeinde(n) with label='{municipality_name}' and id_kanton='{kanton.id}'")
if not gemeinden:
logger.info(f"Gemeinde '{municipality_name}' in Kanton '{canton_abk}' not found, creating it")
gemeinde = Gemeinde(
mandateId=mandateId,
label=municipality_name,
id_kanton=kanton.id,
plz=parzelle_data.get("plz")
)
gemeinde = realEstateInterface.createGemeinde(gemeinde)
logger.info(f"Created Gemeinde '{municipality_name}' with ID: {gemeinde.id}")
else:
gemeinde = gemeinden[0]
logger.debug(f"Found Gemeinde: ID={gemeinde.id}, Label={gemeinde.label}")
gemeinde_id = gemeinde.id
logger.info(f"Resolved Gemeinde '{municipality_name}' to ID '{gemeinde_id}'")
else:
logger.warning(f"Missing Gemeinde/Kanton data: municipality_name={municipality_name}, canton={canton_abk}")
alias_tags = []
if parzelle_data.get("egrid"):
alias_tags.append(parzelle_data["egrid"])
if parzelle_data.get("number") and parzelle_data["number"] != parzelle_data.get("id"):
alias_tags.append(parzelle_data["number"])
strasse_nr = None
plz = None
address = parzelle_data.get("address")
if address:
parts = address.split(",")
if len(parts) >= 1:
strasse_nr = parts[0].strip()
plz = parzelle_data.get("plz")
logger.debug(f"Parzelle {idx + 1} address data: strasse_nr='{strasse_nr}', plz='{plz}', full_address='{address}'")
if not strasse_nr and not plz:
logger.warning(f"No address data found for Parzelle {idx + 1} (label: {parcel_label})")
kontext_items = []
if parzelle_data.get("egrid"):
kontext_items.append(Kontext(
thema="EGRID",
inhalt=parzelle_data["egrid"]
))
if parzelle_data.get("identnd"):
kontext_items.append(Kontext(
thema="IdentND",
inhalt=parzelle_data["identnd"]
))
if parzelle_data.get("area_m2"):
kontext_items.append(Kontext(
thema="Fläche",
inhalt=f"{parzelle_data['area_m2']} m²"
))
if parzelle_data.get("centroid"):
centroid = parzelle_data["centroid"]
kontext_items.append(Kontext(
thema="Zentrum (LV95)",
inhalt=f"X: {centroid.get('x')} m, Y: {centroid.get('y')} m (EPSG:2056)"
))
if parzelle_data.get("geoportal_url"):
kontext_items.append(Kontext(
thema="Geoportal URL",
inhalt=parzelle_data["geoportal_url"]
))
if parzelle_data.get("municipality_code"):
kontext_items.append(Kontext(
thema="BFS-Nummer",
inhalt=str(parzelle_data["municipality_code"])
))
adjacent_parcel_refs = []
if parzelle_data.get("adjacent_parcels"):
neighbors_to_filter = []
for adj_parcel in parzelle_data["adjacent_parcels"]:
if isinstance(adj_parcel, dict):
neighbors_to_filter.append(adj_parcel)
elif isinstance(adj_parcel, str):
neighbors_to_filter.append({"id": adj_parcel})
if all_parcel_geometries and neighbors_to_filter:
try:
filtered_neighbors = filter_neighbor_parcels(
neighbors_to_filter,
all_parcel_geometries
)
for filtered_neighbor in filtered_neighbors:
adj_id = filtered_neighbor.get("id")
if adj_id:
adjacent_parcel_refs.append({"id": adj_id})
except Exception as e:
logger.warning(f"Error filtering neighbor parcels: {e}, including all neighbors")
for adj_parcel in parzelle_data["adjacent_parcels"]:
if isinstance(adj_parcel, dict):
adj_id = adj_parcel.get("id")
if adj_id:
adjacent_parcel_refs.append({"id": adj_id})
elif isinstance(adj_parcel, str):
adjacent_parcel_refs.append({"id": adj_parcel})
else:
for adj_parcel in parzelle_data["adjacent_parcels"]:
if isinstance(adj_parcel, dict):
adj_id = adj_parcel.get("id")
if adj_id:
adjacent_parcel_refs.append({"id": adj_id})
elif isinstance(adj_parcel, str):
adjacent_parcel_refs.append({"id": adj_parcel})
perimeter = parzelle_data.get("perimeter")
if isinstance(perimeter, dict):
if "punkte" in perimeter and "closed" in perimeter:
try:
perimeter = GeoPolylinie(**perimeter)
except Exception as e:
raise ValueError(f"Invalid perimeter format: {str(e)}")
else:
converted = convert_geojson_to_geopolylinie(perimeter)
if converted:
perimeter = converted
else:
raise ValueError("Invalid perimeter format: cannot convert to GeoPolylinie")
elif isinstance(perimeter, GeoPolylinie):
pass
else:
raise ValueError("Invalid perimeter type: must be dict or GeoPolylinie")
baulinie = None
geometry = parzelle_data.get("geometry")
logger.debug(f"Geometry present: {geometry is not None}")
if geometry:
logger.debug(f"Geometry type: {type(geometry)}, keys: {list(geometry.keys()) if isinstance(geometry, dict) else 'not a dict'}")
baulinie = convert_geojson_to_geopolylinie(geometry)
if baulinie:
logger.info(f"Extracted baulinie from geometry with {len(baulinie.punkte)} points")
else:
logger.warning("Failed to extract baulinie from geometry")
else:
logger.warning("No geometry found in parzelle_data")
parzelle_create_data = {
"mandateId": mandateId,
"label": parcel_label,
"parzellenAliasTags": alias_tags,
"eigentuemerschaft": None,
"strasseNr": strasse_nr,
"plz": plz,
"perimeter": perimeter,
"baulinie": baulinie,
"kontextGemeinde": gemeinde_id,
"bauzone": None,
"az": None,
"bz": None,
"vollgeschossZahl": None,
"anrechenbarDachgeschoss": None,
"anrechenbarUntergeschoss": None,
"gebaeudehoeheMax": None,
"regelnGrenzabstand": [],
"regelnMehrlaengenzuschlag": [],
"regelnMehrhoehenzuschlag": [],
"parzelleBebaut": None,
"parzelleErschlossen": None,
"parzelleHanglage": None,
"laermschutzzone": None,
"hochwasserschutzzone": None,
"grundwasserschutzzone": None,
"parzellenNachbarschaft": adjacent_parcel_refs,
"dokumente": [],
"kontextInformationen": kontext_items,
}
logger.debug(f"Creating Parzelle with label: {parzelle_create_data.get('label')}")
logger.debug(f"Parzelle mandateId: {parzelle_create_data.get('mandateId')}")
logger.debug(f"Parzelle perimeter present: {parzelle_create_data.get('perimeter') is not None}")
try:
parzelle_instance = Parzelle(**parzelle_create_data)
logger.debug(f"Parzelle instance created successfully with ID: {parzelle_instance.id}")
except Exception as e:
logger.error(f"Error creating Parzelle instance: {str(e)}", exc_info=True)
raise
try:
logger.info(f"Calling createParzelle for Parzelle '{parzelle_instance.label}' (ID: {parzelle_instance.id})")
logger.debug(f"Parzelle instance before createParzelle: {parzelle_instance.model_dump(mode='json', exclude={'perimeter', 'baulinie', 'kontextInformationen'})}")
parzelle_dict = parzelle_instance.model_dump(mode='json')
logger.debug(f"Parzelle dict keys: {list(parzelle_dict.keys())}")
created_parzelle = realEstateInterface.createParzelle(parzelle_instance)
logger.info(f"createParzelle returned: ID={created_parzelle.id if created_parzelle else 'None'}, Label={created_parzelle.label if created_parzelle else 'None'}")
if not created_parzelle:
raise ValueError("Failed to create Parzelle - createParzelle returned None")
if not created_parzelle.id:
raise ValueError("Failed to create Parzelle - no ID returned")
logger.info(f"Parzelle created with ID: {created_parzelle.id}")
logger.debug(f"Verifying Parzelle {created_parzelle.id} exists in database...")
verify_parzelle = realEstateInterface.getParzelle(created_parzelle.id)
if not verify_parzelle:
logger.error(f"Parzelle {created_parzelle.id} was not found in database after creation")
all_parzellen = realEstateInterface.getParzellen(recordFilter=None)
logger.error(f"Total Parzellen in database: {len(all_parzellen)}")
if all_parzellen:
logger.error(f"Sample Parzelle IDs: {[p.id for p in all_parzellen[:5]]}")
raise ValueError(f"Parzelle {created_parzelle.id} was not found in database after creation")
logger.info(f"Verified Parzelle {created_parzelle.id} exists in database")
created_parzelle = verify_parzelle
if created_parzelle.perimeter:
parcel_perimeters.append(created_parzelle.perimeter)
created_parzellen.append(created_parzelle)
except Exception as e:
logger.error(f"Error creating Parzelle {idx + 1}: {str(e)}", exc_info=True)
raise
if not created_parzellen:
raise ValueError("No Parzellen were successfully created")
logger.info(f"Successfully created {len(created_parzellen)} Parzelle(n)")
project_baulinie = None
if len(parcel_perimeters) > 0:
try:
if len(parcel_perimeters) == 1:
project_baulinie = parcel_perimeters[0]
logger.info("Using single parcel perimeter as baulinie")
else:
logger.info(f"Combining {len(parcel_perimeters)} parcel geometries to create baulinie")
project_baulinie = combine_parcel_geometries(parcel_perimeters)
logger.info(f"Created combined baulinie with {len(project_baulinie.punkte)} points")
except Exception as e:
logger.error(f"Error combining parcel geometries for baulinie: {e}", exc_info=True)
if parcel_perimeters:
project_baulinie = parcel_perimeters[0]
logger.warning("Using first parcel perimeter as fallback baulinie")
status_prozess_enum = None
if status_prozess:
try:
if isinstance(status_prozess, str):
status_prozess_enum = StatusProzess(status_prozess)
elif isinstance(status_prozess, StatusProzess):
status_prozess_enum = status_prozess
except (ValueError, KeyError):
logger.warning(f"Invalid statusProzess '{status_prozess}', using default 'Eingang'")
status_prozess_enum = StatusProzess.EINGANG
else:
status_prozess_enum = StatusProzess.EINGANG
logger.debug(f"Preparing Projekt creation with baulinie: {project_baulinie is not None}")
if project_baulinie:
logger.debug(f"Baulinie has {len(project_baulinie.punkte)} points")
project_perimeter = created_parzellen[0].perimeter if created_parzellen else None
projekt_create_data = {
"mandateId": mandateId,
"label": projekt_label,
"statusProzess": status_prozess_enum,
"perimeter": project_perimeter,
"baulinie": project_baulinie,
"parzellen": created_parzellen,
"dokumente": [],
"kontextInformationen": [],
}
logger.debug(f"Projekt data prepared: label={projekt_label}, parzellen_count={len(projekt_create_data['parzellen'])}, baulinie={'present' if project_baulinie else 'None'}")
try:
projekt_instance = Projekt(**projekt_create_data)
logger.debug(f"Projekt instance created successfully with ID: {projekt_instance.id}")
except Exception as e:
logger.error(f"Error creating Projekt instance: {str(e)}", exc_info=True)
raise
logger.debug(f"Creating Projekt with {len(projekt_instance.parzellen)} Parzelle(n)")
if projekt_instance.parzellen:
for idx, p in enumerate(projekt_instance.parzellen):
logger.debug(f" Parzelle {idx}: ID={p.id}, Label={p.label}")
logger.debug(f"Projekt baulinie before save: {projekt_instance.baulinie is not None}")
if projekt_instance.baulinie:
logger.debug(f"Projekt baulinie has {len(projekt_instance.baulinie.punkte)} points")
try:
created_projekt = realEstateInterface.createProjekt(projekt_instance)
logger.info(f"Created Projekt '{created_projekt.label}' (ID: {created_projekt.id})")
logger.debug(f"Created Projekt baulinie: {created_projekt.baulinie is not None}")
except Exception as e:
logger.error(f"Error calling createProjekt: {str(e)}", exc_info=True)
raise
if not created_projekt or not created_projekt.id:
raise ValueError("Failed to create Projekt - no ID returned")
if not created_projekt.parzellen or len(created_projekt.parzellen) == 0:
logger.warning(f"Projekt {created_projekt.id} created but no Parzellen linked")
verify_projekt = realEstateInterface.getProjekt(created_projekt.id)
if verify_projekt and verify_projekt.parzellen:
logger.info(f"Parzellen found when fetching Projekt from database: {len(verify_projekt.parzellen)}")
created_projekt = verify_projekt
else:
raise ValueError(f"Projekt {created_projekt.id} has no Parzellen linked after creation")
else:
logger.info(f"Projekt {created_projekt.id} successfully linked to {len(created_projekt.parzellen)} Parzelle(n)")
for idx, p in enumerate(created_projekt.parzellen):
logger.debug(f" Linked Parzelle {idx}: ID={p.id if hasattr(p, 'id') else 'NO ID'}, Label={p.label if hasattr(p, 'label') else 'NO LABEL'}")
return {
"projekt": created_projekt.model_dump(),
"parzellen": [p.model_dump() for p in created_parzellen],
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error creating project with parcel data: {str(e)}", exc_info=True)
raise

2497
modules/features/teamsbot/service.py
View file

File diff suppressed because it is too large Load diff

305
modules/features/teamsbot/serviceCommands.py Normal file
View file

@ -0,0 +1,305 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Teamsbot Service — AI command execution logic.
Extracted from service.py. All functions accept `service` (a TeamsbotService
instance) as the first parameter so the class can delegate to them.
"""
import logging
import json
import asyncio
from datetime import datetime, timezone
from typing import List
from fastapi import WebSocket
from modules.shared.timeUtils import getUtcTimestamp
from .datamodelTeamsbot import (
TeamsbotTranscript,
TeamsbotCommand,
)
logger = logging.getLogger(__name__)
async def _executeCommands(
service,
sessionId: str,
commands: List[TeamsbotCommand],
voiceInterface,
websocket: WebSocket,
):
"""Execute structured commands returned by the AI."""
for cmd in commands:
action = cmd.action
params = cmd.params or {}
logger.info(f"Session {sessionId}: Executing command '{action}' with params {params}")
try:
if action == "toggleTranscript":
await _cmdToggleTranscript(service, sessionId, params, websocket)
elif action == "toggleChat":
await _cmdToggleChat(service, sessionId, params, websocket)
elif action == "sendChat":
await _cmdSendChat(service, sessionId, params, websocket)
elif action == "readChat":
await _cmdReadChat(service, sessionId, params, voiceInterface, websocket)
elif action == "readAloud":
await _cmdReadAloud(service, sessionId, params, voiceInterface, websocket)
elif action == "changeLanguage":
await _cmdChangeLanguage(service, sessionId, params)
elif action in ("toggleMic", "toggleCamera"):
await _cmdToggleMicOrCamera(service, sessionId, action, params, websocket)
elif action == "sendMail":
await _cmdSendMail(service, sessionId, params)
elif action == "storeDocument":
await _cmdStoreDocument(service, sessionId, params)
else:
logger.warning(f"Session {sessionId}: Unknown command '{action}'")
except Exception as cmdErr:
logger.warning(f"Session {sessionId}: Command '{action}' failed: {cmdErr}")
async def _cmdToggleTranscript(service, sessionId: str, params: dict, websocket: WebSocket):
"""Caption on/off - toggle Teams live transcript capture."""
enable = params.get("enable", True)
if websocket:
await websocket.send_text(json.dumps({
"type": "botCommand",
"sessionId": sessionId,
"command": "toggleTranscript",
"params": {"enable": enable},
}))
async def _cmdToggleChat(service, sessionId: str, params: dict, websocket: WebSocket):
"""Chat on/off - enable/disable meeting chat monitoring."""
enable = params.get("enable", True)
if websocket:
await websocket.send_text(json.dumps({
"type": "botCommand",
"sessionId": sessionId,
"command": "toggleChat",
"params": {"enable": enable},
}))
async def _cmdSendChat(service, sessionId: str, params: dict, websocket: WebSocket):
"""Send a message to the meeting chat and record it in transcript/SSE."""
from .service import _emitSessionEvent
chatText = params.get("text", "")
if not chatText:
return
if websocket:
await websocket.send_text(json.dumps({
"type": "sendChatMessage",
"sessionId": sessionId,
"text": chatText,
}))
logger.info(f"Chat command sent for session {sessionId}")
from . import interfaceFeatureTeamsbot as interfaceDb
interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId)
transcriptData = TeamsbotTranscript(
sessionId=sessionId,
speaker=service.config.botName,
text=chatText,
timestamp=getUtcTimestamp(),
confidence=1.0,
language=service.config.language,
isFinal=True,
source="chat",
).model_dump()
createdTranscript = interface.createTranscript(transcriptData)
import time
service._contextBuffer.append({
"speaker": service.config.botName,
"text": chatText,
"timestamp": getUtcTimestamp(),
"source": "chat",
})
service._lastTranscriptSpeaker = service.config.botName
service._lastTranscriptText = chatText
service._lastTranscriptId = createdTranscript.get("id")
service._lastBotResponseText = chatText.strip().lower()
service._lastBotResponseTs = time.time()
await _emitSessionEvent(sessionId, "transcript", {
"id": createdTranscript.get("id"),
"speaker": service.config.botName,
"text": chatText,
"confidence": 1.0,
"timestamp": getUtcTimestamp(),
"isContinuation": False,
"source": "chat",
"speakerResolvedFromHint": False,
})
async def _cmdReadChat(
service,
sessionId: str,
params: dict,
voiceInterface,
websocket: WebSocket,
):
"""Read chat messages (from DB) with optional fromdatetime/todatetime, then speak or send to chat."""
from .service import _speakTextChunked
from .serviceConversation import _summarizeForVoice
from . import interfaceFeatureTeamsbot as interfaceDb
interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId)
transcripts = interface.getTranscripts(sessionId)
fromDtRaw = params.get("fromdatetime") or params.get("fromDateTime")
toDtRaw = params.get("todatetime") or params.get("toDateTime")
fromTs = datetime.fromisoformat(fromDtRaw).replace(tzinfo=timezone.utc).timestamp() if fromDtRaw else None
toTs = datetime.fromisoformat(toDtRaw).replace(tzinfo=timezone.utc).timestamp() if toDtRaw else None
chatOnly = [t for t in transcripts if t.get("source") in ("chat", "chatHistory")]
if fromTs is not None:
chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) >= fromTs]
if toTs is not None:
chatOnly = [t for t in chatOnly if (t.get("timestamp") or 0) <= toTs]
summary = "\n".join(f"[{t.get('speaker', '?')}]: {t.get('text', '')}" for t in chatOnly[-20:])
if not summary:
summary = "Keine Chat-Nachrichten im angegebenen Zeitraum."
if voiceInterface and websocket:
spokenSummary = await _summarizeForVoice(service, sessionId, summary[:2000])
cancelHook = service._makeAnswerCancelHook()
async with service._meetingTtsLock:
await _speakTextChunked(
websocket=websocket,
voiceInterface=voiceInterface,
sessionId=sessionId,
voiceText=spokenSummary,
languageCode=service.config.language,
voiceName=service.config.voiceId,
isCancelled=cancelHook,
)
async def _cmdReadAloud(
service,
sessionId: str,
params: dict,
voiceInterface,
websocket: WebSocket,
):
"""Read text aloud via TTS and play in meeting."""
from .service import _speakTextChunked, _voiceFriendlyMeetingText
readText = params.get("text", "")
if readText and voiceInterface and websocket:
cancelHook = service._makeAnswerCancelHook()
async with service._meetingTtsLock:
await _speakTextChunked(
websocket=websocket,
voiceInterface=voiceInterface,
sessionId=sessionId,
voiceText=_voiceFriendlyMeetingText(readText),
languageCode=service.config.language,
voiceName=service.config.voiceId,
isCancelled=cancelHook,
)
async def _cmdChangeLanguage(service, sessionId: str, params: dict):
"""Change bot language."""
from .service import _emitSessionEvent
newLang = params.get("language", "")
if newLang:
service.config = service.config.model_copy(update={"language": newLang})
logger.info(f"Session {sessionId}: Language changed to '{newLang}'")
await _emitSessionEvent(sessionId, "languageChanged", {"language": newLang})
async def _cmdToggleMicOrCamera(
service,
sessionId: str,
action: str,
params: dict,
websocket: WebSocket,
):
"""Toggle mic or camera in the meeting."""
if websocket:
await websocket.send_text(json.dumps({
"type": "botCommand",
"sessionId": sessionId,
"command": action,
"params": params,
}))
async def _cmdSendMail(service, sessionId: str, params: dict):
"""Send email via Service Center MessagingService."""
recipient = params.get("recipient") or params.get("to", "")
subject = params.get("subject", "")
message = params.get("message") or params.get("body", "")
if not recipient or not subject:
logger.warning(f"Session {sessionId}: sendMail requires recipient and subject")
return
try:
from modules.serviceCenter import ServiceCenterContext, getService
ctx = ServiceCenterContext(
user=service.currentUser,
mandate_id=service.mandateId,
feature_instance_id=service.instanceId,
)
messaging = getService("messaging", ctx)
success = messaging.sendEmailDirect(
recipient=recipient,
subject=subject,
message=message,
userId=str(service.currentUser.id) if service.currentUser else None,
)
if success:
logger.info(f"Session {sessionId}: Email sent to {recipient}")
else:
logger.warning(f"Session {sessionId}: Email send failed for {recipient}")
except Exception as e:
logger.warning(f"Session {sessionId}: sendMail failed: {e}")
async def _cmdStoreDocument(service, sessionId: str, params: dict):
"""Store document via Service Center SharepointService."""
sitePath = params.get("sitePath") or params.get("site", "")
folderPath = params.get("folderPath") or params.get("folder", "")
fileName = params.get("fileName", "document.txt")
content = params.get("content", "")
if isinstance(content, str):
content = content.encode("utf-8")
if not sitePath or not folderPath:
logger.warning(f"Session {sessionId}: storeDocument requires sitePath and folderPath")
return
try:
from modules.serviceCenter import ServiceCenterContext, getService
ctx = ServiceCenterContext(
user=service.currentUser,
mandate_id=service.mandateId,
feature_instance_id=service.instanceId,
)
sharepoint = getService("sharepoint", ctx)
if not sharepoint.setAccessTokenFromConnection(service.currentUser):
logger.warning(f"Session {sessionId}: SharePoint connection not configured")
return
site = await sharepoint.getSiteByStandardPath(sitePath)
if not site:
logger.warning(f"Session {sessionId}: SharePoint site not found: {sitePath}")
return
result = await sharepoint.uploadFile(
siteId=site["id"],
folderPath=folderPath,
fileName=fileName,
content=content,
)
if "error" in result:
logger.warning(f"Session {sessionId}: storeDocument failed: {result['error']}")
else:
logger.info(f"Session {sessionId}: Document stored: {fileName}")
except Exception as e:
logger.warning(f"Session {sessionId}: storeDocument failed: {e}")

996
modules/features/teamsbot/serviceConversation.py Normal file
View file

@ -0,0 +1,996 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Teamsbot Service — Conversation & AI analysis logic.
Extracted from service.py. All functions accept `service` (a TeamsbotService
instance) as the first parameter so the class can delegate to them.
"""
import logging
import json
import re
import asyncio
import time
from typing import Optional, Dict, Any, List
from fastapi import WebSocket
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum
from modules.shared.timeUtils import getUtcTimestamp
from .datamodelTeamsbot import (
TeamsbotTranscript,
TeamsbotBotResponse,
TeamsbotResponseType,
TeamsbotResponseMode,
TeamsbotResponseChannel,
SpeechTeamsResponse,
TeamsbotDirectorPromptMode,
TeamsbotDirectorPromptStatus,
)
logger = logging.getLogger(__name__)
async def _analyzeAndRespond(
service,
sessionId: str,
interface,
voiceInterface,
websocket: WebSocket,
triggerTranscript: Dict[str, Any],
):
"""Run SPEECH_TEAMS AI analysis and respond if needed."""
from .service import (
_emitSessionEvent, createAiService, _speakTextChunked,
_voiceFriendlyMeetingText,
TEAMSBOT_AGENT_MAX_ROUNDS, TEAMSBOT_AGENT_MAX_COST_CHF,
)
if service._aiAnalysisInProgress:
logger.info(f"Session {sessionId}: AI analysis already in progress, skipping duplicate trigger")
return
if service._agentEscalationInFlight:
logger.info(
f"Session {sessionId}: Agent escalation still in flight — "
f"skipping new SPEECH_TEAMS trigger to prevent overlapping replies"
)
return
service._aiAnalysisInProgress = True
service._lastAiCallTime = time.time()
contextLines = []
for segment in service._contextBuffer:
speaker = segment.get("speaker", "Unknown")
text = segment.get("text", "")
segSource = segment.get("source", "caption")
prefix = "Chat" if segSource == "chat" else ""
if service._isBotSpeaker(speaker):
contextLines.append(f"[YOU ({service.config.botName})]: {text}")
elif prefix:
contextLines.append(f"[{prefix}: {speaker}]: {text}")
else:
contextLines.append(f"[{speaker}]: {text}")
sessionContextStr = ""
if service._sessionContext:
sessionContextStr = f"\nSESSION_CONTEXT (background knowledge provided by the user):\n{service._sessionContext}\n"
summaryStr = ""
if service._contextSummary:
summaryStr = f"\nEARLIER_CONVERSATION_SUMMARY:\n{service._contextSummary}\n"
directorStr = service._buildPersistentDirectorContext()
transcriptContext = f"BOT_NAME:{service.config.botName}{sessionContextStr}{summaryStr}{directorStr}\nRECENT_TRANSCRIPT:\n" + "\n".join(contextLines)
try:
aiService = createAiService(service.currentUser, service.mandateId, service.instanceId)
await aiService.ensureAiObjectsInitialized()
request = AiCallRequest(
prompt=service.config.aiSystemPrompt,
context=transcriptContext,
options=AiCallOptions(
operationType=OperationTypeEnum.SPEECH_TEAMS,
priority=PriorityEnum.SPEED,
)
)
response = await aiService.callAi(request)
try:
speechResult = SpeechTeamsResponse.model_validate_json(response.content)
except Exception:
try:
jsonStr = response.content
if "```json" in jsonStr:
jsonStr = jsonStr.split("```json")[1].split("```")[0]
elif "```" in jsonStr:
jsonStr = jsonStr.split("```")[1].split("```")[0]
speechResult = SpeechTeamsResponse.model_validate_json(jsonStr.strip())
except Exception as parseErr:
logger.warning(f"Failed to parse SPEECH_TEAMS response: {parseErr}")
speechResult = SpeechTeamsResponse(
shouldRespond=False,
reasoning=f"Parse error: {str(parseErr)[:100]}",
detectedIntent="none"
)
logger.info(
f"SPEECH_TEAMS result: shouldRespond={speechResult.shouldRespond}, "
f"intent={speechResult.detectedIntent}, "
f"reasoning={speechResult.reasoning[:80]}..."
)
await _emitSessionEvent(sessionId, "analysis", {
"shouldRespond": speechResult.shouldRespond,
"detectedIntent": speechResult.detectedIntent,
"reasoning": speechResult.reasoning,
"modelName": response.modelName,
"processingTime": response.processingTime,
"priceCHF": response.priceCHF,
"needsAgent": speechResult.needsAgent,
"agentReason": speechResult.agentReason,
})
if speechResult.needsAgent:
briefings = service._collectActiveDirectorBriefings()
briefingFileIds = service._collectDirectorFileIds()
briefingBlock = ""
if briefings:
parts = []
for b in briefings:
seg = f"- ({b.get('mode')}) {b.get('text', '')}".rstrip()
if b.get("fileIds"):
seg += f"\n attachedFileIds: {', '.join(b['fileIds'])}"
if b.get("note"):
note = b["note"]
seg += (
"\n priorAgentAnalysis: "
+ (note if len(note) <= 800 else note[:800] + "...")
)
parts.append(seg)
briefingBlock = (
"\n\nACTIVE_OPERATOR_BRIEFINGS (private; you may read the "
"attached files via summarizeContent / readFile / "
"readContentObjects to answer the user precisely; do NOT "
"quote the directive text itself):\n" + "\n".join(parts)
)
logger.info(
f"Session {sessionId}: SPEECH_TEAMS escalates to agent. "
f"Reason: {speechResult.agentReason or speechResult.reasoning} | "
f"briefings={len(briefings)}, fileIds={len(briefingFileIds)}"
)
taskBrief = (
(speechResult.agentReason
or speechResult.responseText
or "Verarbeite die juengste Sprecheranfrage und antworte ins Meeting.")
+ briefingBlock
)
service._agentEscalationInFlight = True
service._currentEscalationTask = asyncio.create_task(
_runEscalationAndRelease(
service,
sessionId=sessionId,
taskBrief=taskBrief,
briefingFileIds=briefingFileIds,
triggerTranscriptId=triggerTranscript.get("id"),
)
)
return
if speechResult.detectedIntent == "stop":
logger.info(f"Session {sessionId}: AI detected STOP intent: {speechResult.reasoning}")
if websocket:
try:
await websocket.send_text(json.dumps({
"type": "stopAudio",
"sessionId": sessionId,
}))
except Exception as stopErr:
logger.warning(f"Failed to send stop command: {stopErr}")
return
if speechResult.shouldRespond and speechResult.responseText:
if service.config.responseMode == TeamsbotResponseMode.MANUAL:
await _emitSessionEvent(sessionId, "suggestedResponse", {
"responseText": speechResult.responseText,
"detectedIntent": speechResult.detectedIntent,
"reasoning": speechResult.reasoning,
})
return
channels = speechResult.responseChannels
if channels and isinstance(channels, list):
channelStr = ",".join(str(c).lower().strip() for c in channels)
sendVoice = "voice" in channelStr
sendChat = "chat" in channelStr
logger.info(f"Response channel (from AI): voice={sendVoice}, chat={sendChat}")
else:
channelRaw = service.config.responseChannel
channelStr = (channelRaw.value if hasattr(channelRaw, 'value') else str(channelRaw)).lower().strip()
sendVoice = channelStr in ("voice", "both")
sendChat = channelStr in ("chat", "both")
logger.info(f"Response channel (from config): '{channelStr}'")
if sendVoice and sendChat:
responseType = TeamsbotResponseType.BOTH
elif sendVoice:
responseType = TeamsbotResponseType.AUDIO
else:
responseType = TeamsbotResponseType.CHAT
canonicalText = (
speechResult.responseText
or speechResult.responseTextForVoice
or speechResult.responseTextForChat
or ""
)
normalizedResponse = (canonicalText or "").strip().lower()
nowTs = time.time()
if (
normalizedResponse
and service._lastBotResponseText == normalizedResponse
and (nowTs - service._lastBotResponseTs) < 90
):
logger.info(f"Session {sessionId}: Suppressing duplicate bot response within 90s window")
await _emitSessionEvent(sessionId, "analysis", {
"shouldRespond": False,
"detectedIntent": speechResult.detectedIntent,
"reasoning": "Suppressed duplicate response within 90s",
"modelName": response.modelName,
"processingTime": response.processingTime,
"priceCHF": response.priceCHF,
})
return
textForVoice = speechResult.responseTextForVoice or speechResult.responseText
textForChat = speechResult.responseTextForChat or speechResult.responseText
storedText = textForChat or textForVoice or speechResult.responseText
if sendVoice and textForVoice:
await _emitSessionEvent(sessionId, "ttsDeliveryStatus", {
"status": "requested",
"hasWebSocket": websocket is not None,
"message": "TTS generation requested",
"timestamp": getUtcTimestamp(),
})
logger.info(
f"Session {sessionId}: TTS requested (websocket_available={websocket is not None})"
)
if not websocket:
logger.warning(
f"Session {sessionId}: TTS skipped (bot websocket unavailable, likely fallback mode)"
)
await _emitSessionEvent(sessionId, "ttsDeliveryStatus", {
"status": "unavailable",
"hasWebSocket": False,
"message": "TTS skipped — bot websocket unavailable",
"timestamp": getUtcTimestamp(),
})
if not sendChat:
sendChat = True
else:
spokenText = await _summarizeForVoice(service, sessionId, textForVoice)
cancelHook = service._makeAnswerCancelHook()
async with service._meetingTtsLock:
ttsOutcome = await _speakTextChunked(
websocket=websocket,
voiceInterface=voiceInterface,
sessionId=sessionId,
voiceText=spokenText,
languageCode=service.config.language,
voiceName=service.config.voiceId,
isCancelled=cancelHook,
)
if ttsOutcome.get("success"):
logger.info(
f"Session {sessionId}: TTS audio dispatched to bot "
f"(chunks={ttsOutcome.get('chunks')}, played={ttsOutcome.get('played')})"
)
await _emitSessionEvent(sessionId, "ttsDeliveryStatus", {
"status": "dispatched",
"hasWebSocket": True,
"chunks": ttsOutcome.get("chunks"),
"played": ttsOutcome.get("played"),
"timestamp": getUtcTimestamp(),
})
else:
logger.warning(
f"TTS failed for session {sessionId}: {ttsOutcome.get('error')}"
)
await _emitSessionEvent(sessionId, "ttsDeliveryStatus", {
"status": "failed",
"hasWebSocket": True,
"chunks": ttsOutcome.get("chunks"),
"played": ttsOutcome.get("played"),
"message": ttsOutcome.get("error"),
"timestamp": getUtcTimestamp(),
})
if not sendChat:
sendChat = True
if sendChat and textForChat:
try:
if websocket:
await websocket.send_text(json.dumps({
"type": "sendChatMessage",
"sessionId": sessionId,
"text": textForChat,
}))
logger.info(f"Chat response sent for session {sessionId}")
except Exception as chatErr:
logger.warning(f"Chat message send failed for session {sessionId}: {chatErr}")
botResponseData = TeamsbotBotResponse(
sessionId=sessionId,
responseText=storedText,
responseType=responseType,
detectedIntent=speechResult.detectedIntent,
reasoning=speechResult.reasoning,
triggeredByTranscriptId=triggerTranscript.get("id"),
modelName=response.modelName,
processingTime=response.processingTime,
priceCHF=response.priceCHF,
timestamp=getUtcTimestamp(),
).model_dump()
createdResponse = interface.createBotResponse(botResponseData)
await _emitSessionEvent(sessionId, "botResponse", {
"id": createdResponse.get("id"),
"responseText": storedText,
"responseType": responseType.value,
"detectedIntent": speechResult.detectedIntent,
"reasoning": speechResult.reasoning,
"modelName": response.modelName,
"processingTime": response.processingTime,
"priceCHF": response.priceCHF,
"timestamp": botResponseData.get("timestamp"),
})
session = interface.getSession(sessionId)
if session:
count = session.get("botResponseCount", 0) + 1
interface.updateSession(sessionId, {"botResponseCount": count})
service._lastBotResponseText = normalizedResponse
service._lastBotResponseTs = nowTs
botTranscriptData = TeamsbotTranscript(
sessionId=sessionId,
speaker=service.config.botName,
text=storedText,
timestamp=getUtcTimestamp(),
confidence=1.0,
language=service.config.language,
isFinal=True,
).model_dump()
botTranscript = interface.createTranscript(botTranscriptData)
service._contextBuffer.append({
"speaker": service.config.botName,
"text": storedText,
"timestamp": getUtcTimestamp(),
"source": "botResponse",
})
await _emitSessionEvent(sessionId, "transcript", {
"id": botTranscript.get("id"),
"speaker": service.config.botName,
"text": storedText,
"confidence": 1.0,
"timestamp": getUtcTimestamp(),
"isContinuation": False,
"source": "botResponse",
"speakerResolvedFromHint": False,
})
service._lastTranscriptSpeaker = service.config.botName
service._lastTranscriptText = storedText
service._lastTranscriptId = botTranscript.get("id")
service._followUpWindowEnd = time.time() + 15.0
logger.info(f"Bot responded in session {sessionId}: intent={speechResult.detectedIntent}, follow-up window open for 15s")
if speechResult.commands:
from .serviceCommands import _executeCommands
await _executeCommands(service, sessionId, speechResult.commands, voiceInterface, websocket)
if speechResult.shouldRespond and not speechResult.responseText:
cmdTexts = [
c.params.get("text", "") for c in speechResult.commands
if c.action == "sendChat" and c.params and c.params.get("text")
]
combinedText = " ".join(cmdTexts) if cmdTexts else None
if combinedText:
botResponseData = TeamsbotBotResponse(
sessionId=sessionId,
responseText=combinedText,
responseType=TeamsbotResponseType.CHAT,
detectedIntent=speechResult.detectedIntent,
reasoning=speechResult.reasoning,
triggeredByTranscriptId=triggerTranscript.get("id"),
modelName=response.modelName,
processingTime=response.processingTime,
priceCHF=response.priceCHF,
timestamp=getUtcTimestamp(),
).model_dump()
createdResponse = interface.createBotResponse(botResponseData)
await _emitSessionEvent(sessionId, "botResponse", {
"id": createdResponse.get("id"),
"responseText": combinedText,
"responseType": TeamsbotResponseType.CHAT.value,
"detectedIntent": speechResult.detectedIntent,
"reasoning": speechResult.reasoning,
"modelName": response.modelName,
"processingTime": response.processingTime,
"priceCHF": response.priceCHF,
"timestamp": botResponseData.get("timestamp"),
})
session = interface.getSession(sessionId)
if session:
count = session.get("botResponseCount", 0) + 1
interface.updateSession(sessionId, {"botResponseCount": count})
service._followUpWindowEnd = time.time() + 15.0
logger.info(
f"Bot responded via commands in session {sessionId}: "
f"intent={speechResult.detectedIntent}, follow-up window open for 15s"
)
except Exception as e:
logger.error(f"SPEECH_TEAMS analysis failed for session {sessionId}: {type(e).__name__}: {e}", exc_info=True)
await _emitSessionEvent(sessionId, "error", {"message": f"AI analysis failed: {type(e).__name__}: {str(e)}"})
finally:
service._aiAnalysisInProgress = False
async def _processTranscript(
service,
sessionId: str,
speaker: str,
text: str,
isFinal: bool,
interface,
voiceInterface,
websocket: WebSocket,
source: str = "caption",
speakerResolvedFromHint: Optional[bool] = None,
):
"""Process a transcript segment from captions or chat messages."""
from .service import _emitSessionEvent
text = text.strip()
if not text:
return
if source in ("caption", "speakerHint"):
service._registerSpeakerHint(speaker, text, sessionId)
if (
source == "speakerHint"
and isFinal
and not service._isBotSpeaker(speaker)
and service.config.responseMode != TeamsbotResponseMode.TRANSCRIBE_ONLY
and service._detectBotName(text)
):
triggerTranscript = {"id": None, "speaker": speaker, "text": text, "source": source}
isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, triggerTranscript)
if isNew:
logger.info(f"Session {sessionId}: Bot name in caption, debounce trigger started")
asyncio.create_task(_checkPendingNameTrigger(service))
service._currentQuickAckTask = asyncio.create_task(
_runQuickAck(service, sessionId)
)
return
if source == "chatHistory":
transcriptData = TeamsbotTranscript(
sessionId=sessionId,
speaker=speaker,
text=text,
timestamp=getUtcTimestamp(),
confidence=1.0,
language=service.config.language,
isFinal=True,
source="chatHistory",
).model_dump()
createdTranscript = interface.createTranscript(transcriptData)
await _emitSessionEvent(sessionId, "transcript", {
"id": createdTranscript.get("id"),
"speaker": speaker,
"text": text,
"confidence": 1.0,
"timestamp": getUtcTimestamp(),
"isContinuation": False,
"source": "chatHistory",
"isHistory": True,
})
logger.debug(f"Session {sessionId}: Chat history stored (no AI trigger): [{speaker}] {text[:60]}")
return
isBotSpeaker = service._isBotSpeaker(speaker)
if isBotSpeaker and source != "chat":
logger.debug(f"Session {sessionId}: Ignoring own bot caption from: [{speaker}] {text[:80]}...")
return
sttPauseThreshold = 5.0
isMerge = (
source == "audioCapture"
and service._lastTranscriptSpeaker == speaker
and service._lastTranscriptText is not None
and service._lastTranscriptId is not None
and (time.time() - service._lastSttTime) < sttPauseThreshold
)
if isMerge:
mergedText = f"{service._lastTranscriptText} {text}"
interface.updateTranscript(service._lastTranscriptId, {
"text": mergedText,
"isFinal": isFinal,
})
service._lastTranscriptText = mergedText
createdTranscript = {"id": service._lastTranscriptId}
if service._contextBuffer and service._contextBuffer[-1].get("speaker") == speaker:
service._contextBuffer[-1]["text"] = mergedText
else:
transcriptData = TeamsbotTranscript(
sessionId=sessionId,
speaker=speaker,
text=text,
timestamp=getUtcTimestamp(),
confidence=1.0,
language=service.config.language,
isFinal=isFinal,
source=source,
).model_dump()
createdTranscript = interface.createTranscript(transcriptData)
service._lastTranscriptSpeaker = speaker
service._lastTranscriptText = text
service._lastTranscriptId = createdTranscript.get("id")
if source == "audioCapture" and speaker == "Unknown":
service._unattributedTranscriptIds.append(createdTranscript.get("id"))
service._contextBuffer.append({
"speaker": speaker or "Unknown",
"text": text,
"timestamp": getUtcTimestamp(),
"source": source,
})
maxSegments = service.config.contextWindowSegments
if len(service._contextBuffer) > maxSegments:
if not service._contextSummary and len(service._contextBuffer) > maxSegments * 1.5:
asyncio.create_task(service._summarizeContextBuffer(sessionId))
service._contextBuffer = service._contextBuffer[-maxSegments:]
session = interface.getSession(sessionId)
if session:
count = session.get("transcriptSegmentCount", 0) + 1
interface.updateSession(sessionId, {"transcriptSegmentCount": count})
if source == "audioCapture":
service._lastSttTime = time.time()
displayText = service._lastTranscriptText if isMerge else text
await _emitSessionEvent(sessionId, "transcript", {
"id": createdTranscript.get("id"),
"speaker": speaker,
"text": displayText,
"confidence": 1.0,
"timestamp": getUtcTimestamp(),
"isContinuation": isMerge,
"source": source,
"speakerResolvedFromHint": (
speakerResolvedFromHint
if speakerResolvedFromHint is not None
else False
),
})
if not isFinal:
return
if service.config.responseMode == TeamsbotResponseMode.TRANSCRIBE_ONLY:
return
if source == "chat" and isBotSpeaker:
return
if service._isStopPhrase(text):
logger.info(
f"Session {sessionId}: Stop phrase detected ('{text.strip()[:60]}'), "
f"hard-cancelling in-flight speech immediately"
)
from .serviceWebSocket import _cancelInFlightSpeech
await _cancelInFlightSpeech(
service,
sessionId=sessionId,
websocket=websocket,
reason="userStopPhrase",
)
return
if service._pendingNameTrigger:
service._pendingNameTrigger["lastActivity"] = time.time()
if service._detectBotName(text):
isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript)
if isNew:
asyncio.create_task(_checkPendingNameTrigger(service))
service._currentQuickAckTask = asyncio.create_task(
_runQuickAck(service, sessionId)
)
return
if (
source == "audioCapture"
and not service._isBotSpeaker(speaker)
and time.time() < service._followUpWindowEnd
and not service._pendingNameTrigger
):
isNew = service._setPendingNameTrigger(sessionId, interface, voiceInterface, websocket, createdTranscript)
if isNew:
logger.info(f"Session {sessionId}: Follow-up window trigger (no name needed)")
asyncio.create_task(_checkPendingNameTrigger(service))
return
if not service._pendingNameTrigger:
shouldTrigger = service._shouldTriggerAnalysis(text)
if shouldTrigger:
logger.info(f"Session {sessionId}: Periodic trigger (buffer: {len(service._contextBuffer)} segments)")
await _analyzeAndRespond(service, sessionId, interface, voiceInterface, websocket, createdTranscript)
async def _summarizeForVoice(
service,
sessionId: str,
rawAnswer: str,
) -> str:
"""Return a SHORT, naturally-spoken paraphrase of ``rawAnswer`` for TTS."""
from .service import _voiceFriendlyMeetingText, createAiService
if not rawAnswer or not rawAnswer.strip():
return ""
sanitised = _voiceFriendlyMeetingText(rawAnswer)
if (
len(sanitised) <= service._VOICE_DIRECT_MAX_CHARS
and not service._looksLikeStructuredText(rawAnswer)
):
return sanitised
targetLang = (service.config.language or "de-DE").strip()
botName = (service.config.botName or "").strip() or "the assistant"
persona = (service.config.aiSystemPrompt or "").strip()
personaBlock = (
f"\n\nBOT PERSONA / TONE:\n{persona}\n"
if persona else ""
)
prompt = (
f"You are condensing a long written answer into a SHORT spoken "
f"paraphrase that the assistant '{botName}' will say out loud "
f"into a Microsoft Teams meeting. The full written answer is "
f"already in the meeting chat — your job is to summarise it for "
f"the EAR, not the eye.\n\n"
f"STRICT REQUIREMENTS:\n"
f"1. Output language: BCP-47 '{targetLang}'. No other language.\n"
f"2. 1 to 3 sentences, max ~{service._VOICE_SUMMARY_MAX_CHARS} characters total.\n"
f"3. Natural spoken style — no headings, no bullet points, no "
f"tables, no markdown, no emojis, no enumerations like 'Erstens... "
f"Zweitens...' unless that genuinely flows in speech.\n"
f"4. Capture the essence and the most important conclusion. Do "
f"NOT try to fit every detail. Listeners can read the chat for "
f"the full version.\n"
f"5. End by gently pointing the audience to the chat for details, "
f"e.g. 'Details stehen im Chat.' (adapted to the target language).\n"
f"6. Output ONLY the spoken text. No JSON, no quotes around it, "
f"no preamble like 'Here is the summary:'.\n"
f"{personaBlock}\n"
f"FULL WRITTEN ANSWER (markdown-formatted, sometimes long):\n"
f"---\n{rawAnswer.strip()[:6000]}\n---\n"
)
try:
aiService = createAiService(
service.currentUser, service.mandateId, service.instanceId
)
await aiService.ensureAiObjectsInitialized()
request = AiCallRequest(
prompt=prompt,
context="",
options=AiCallOptions(
operationType=OperationTypeEnum.DATA_ANALYSE,
priority=PriorityEnum.SPEED,
),
)
response = await aiService.callAi(request)
except Exception as aiErr:
logger.warning(
f"Session {sessionId}: Voice summary AI call failed: {aiErr}"
)
return sanitised[: service._VOICE_DIRECT_MAX_CHARS]
if not response or response.errorCount != 0 or not response.content:
logger.warning(
f"Session {sessionId}: Voice summary returned empty/error"
)
return sanitised[: service._VOICE_DIRECT_MAX_CHARS]
spoken = response.content.strip()
spoken = _voiceFriendlyMeetingText(spoken)
if not spoken:
return sanitised[: service._VOICE_DIRECT_MAX_CHARS]
logger.info(
f"Session {sessionId}: Voice summary generated "
f"(orig={len(rawAnswer)} chars, sanitised={len(sanitised)}, "
f"spoken={len(spoken)})"
)
return spoken
async def _pickQuickAckText(service) -> Optional[str]:
"""Return a short ack text in the bot's configured language."""
return await _pickEphemeralPhrase(service, "quickAck")
async def _pickEphemeralPhrase(
service,
kind: str,
substitutions: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
"""Round-robin selector over the cached phrase pool for ``kind``."""
variants = await _getEphemeralPhrases(service, kind)
if not variants:
return None
idx = service._phrasePoolIdx.get(kind, 0) % len(variants)
service._phrasePoolIdx[kind] = (idx + 1) % len(variants)
chosen = variants[idx]
if substitutions:
try:
chosen = chosen.format(**substitutions)
except (KeyError, IndexError, ValueError) as fmtErr:
logger.debug(
f"Ephemeral phrase substitution failed for kind={kind}: {fmtErr}"
)
return chosen
async def _getEphemeralPhrases(service, kind: str) -> List[str]:
"""Return the cached pool of AI-generated variants for ``kind``."""
cached = service._phrasePool.get(kind)
if cached:
return cached
async with service._phrasePoolLock:
cached = service._phrasePool.get(kind)
if cached:
return cached
phrases = await _generateEphemeralPhrases(service, kind, 4)
if phrases:
service._phrasePool[kind] = phrases
return phrases
async def _generateEphemeralPhrases(
service, kind: str, count: int
) -> List[str]:
"""Ask the AI to produce ``count`` short utterances for ``kind``."""
from .service import createAiService, _EPHEMERAL_PHRASE_INTENTS
intent = _EPHEMERAL_PHRASE_INTENTS.get(kind)
if not intent:
logger.warning(f"Unknown ephemeral phrase kind requested: {kind}")
return []
targetLang = (service.config.language or "").strip() or "en-US"
botName = (service.config.botName or "the assistant").strip()
persona = (service.config.aiSystemPrompt or "").strip()
prompt = (
f"You are localizing short SPOKEN-LANGUAGE utterances for a "
f"meeting assistant named '{botName}'.\n\n"
f"Persona / style guide for the assistant:\n"
f"{persona or '(no persona configured — use a neutral, polite, professional tone)'}\n\n"
f"Target spoken language (BCP-47 code): {targetLang}\n\n"
f"Utterance intent:\n{intent}\n\n"
f"Generate {count} DIFFERENT variants matching this intent, in "
f"the target language. Variants should feel natural when spoken "
f"aloud, not robotic. Do NOT include the assistant's name in "
f"the variants.\n\n"
f"Output STRICTLY a JSON array of {count} plain-text strings, "
f"with no markdown fences, no commentary, no surrounding "
f"quotation marks beyond the JSON syntax itself. Example "
f"format: [\"...\", \"...\", \"...\", \"...\"]"
)
try:
aiService = createAiService(
service.currentUser, service.mandateId, service.instanceId
)
await aiService.ensureAiObjectsInitialized()
request = AiCallRequest(
prompt=prompt,
context="",
options=AiCallOptions(
operationType=OperationTypeEnum.DATA_ANALYSE,
priority=PriorityEnum.SPEED,
),
)
response = await aiService.callAi(request)
except Exception as aiErr:
logger.warning(
f"Ephemeral phrase generation failed (kind={kind}, lang={targetLang}): {aiErr}"
)
return []
if not response or response.errorCount != 0 or not response.content:
logger.warning(
f"Ephemeral phrase generation returned empty/error "
f"(kind={kind}, lang={targetLang})"
)
return []
raw = response.content.strip()
raw = re.sub(r"^```(?:json)?\s*", "", raw)
raw = re.sub(r"\s*```\s*$", "", raw)
try:
arr = json.loads(raw)
except json.JSONDecodeError as parseErr:
logger.warning(
f"Ephemeral phrase generation: could not parse JSON "
f"(kind={kind}, lang={targetLang}): {parseErr} "
f"raw={raw[:200]}"
)
return []
if not isinstance(arr, list):
return []
cleaned = [
str(v).strip()
for v in arr
if isinstance(v, str) and str(v).strip()
]
cleaned = cleaned[:count]
if cleaned:
logger.info(
f"Ephemeral phrase pool generated (kind={kind}, "
f"lang={targetLang}, count={len(cleaned)})"
)
return cleaned
async def _runQuickAck(service, sessionId: str) -> None:
"""Background task: speak a short ack into the meeting via TTS."""
from .service import _emitSessionEvent, _speakTextChunked
websocket = service._websocket
voiceInterface = service._voiceInterface
if websocket is None or voiceInterface is None:
return
if not service._shouldFireQuickAck():
return
ackText = await _pickQuickAckText(service)
if not ackText:
return
service._lastQuickAckTs = time.time()
try:
await _emitSessionEvent(sessionId, "quickAck", {
"text": ackText,
"timestamp": getUtcTimestamp(),
})
cancelHook = service._makeAnswerCancelHook()
async with service._meetingTtsLock:
outcome = await _speakTextChunked(
websocket=websocket,
voiceInterface=voiceInterface,
sessionId=sessionId,
voiceText=ackText,
languageCode=service.config.language,
voiceName=service.config.voiceId,
isCancelled=cancelHook,
)
if not outcome.get("success"):
logger.info(
f"Session {sessionId}: Quick ack TTS failed silently "
f"({outcome.get('error')}) — main response will still go through"
)
except asyncio.CancelledError:
logger.info(f"Session {sessionId}: Quick ack cancelled by stop signal")
except Exception as ackErr:
logger.warning(f"Session {sessionId}: Quick ack failed: {ackErr}")
finally:
service._currentQuickAckTask = None
async def _checkPendingNameTrigger(service, delaySec: float = 3.0):
"""Async loop: fire the pending name trigger once the speaker is quiet."""
await asyncio.sleep(delaySec)
if not service._pendingNameTrigger:
return
now = time.time()
lastActivity = service._pendingNameTrigger.get("lastActivity", 0)
detectedAt = service._pendingNameTrigger.get("detectedAt", 0)
quietSec = now - lastActivity
totalWaitSec = now - detectedAt
if quietSec >= 3.0 or totalWaitSec >= 15.0:
trigger = service._pendingNameTrigger
service._pendingNameTrigger = None
logger.info(
f"Session {trigger['sessionId']}: Debounced name trigger fires "
f"(quiet={quietSec:.1f}s, totalWait={totalWaitSec:.1f}s)"
)
await _analyzeAndRespond(
service,
trigger["sessionId"],
trigger["interface"],
trigger["voiceInterface"],
trigger["websocket"],
trigger["triggerTranscript"],
)
else:
remaining = max(0.5, 3.0 - quietSec)
asyncio.create_task(_checkPendingNameTrigger(service, remaining))
async def _warmEphemeralPhrasePool(service, sessionId: str) -> None:
"""Fire-and-forget: generate ephemeral phrase pool for all kinds."""
from .service import _EPHEMERAL_PHRASE_INTENTS
try:
for kind in _EPHEMERAL_PHRASE_INTENTS:
try:
await _getEphemeralPhrases(service, kind)
except Exception as innerErr:
logger.warning(
f"Session {sessionId}: Phrase pool warmup failed for "
f"kind={kind}: {innerErr}"
)
except Exception as warmErr:
logger.warning(
f"Session {sessionId}: Phrase pool warmup task crashed: {warmErr}"
)
async def _runEscalationAndRelease(
service,
sessionId: str,
taskBrief: str,
briefingFileIds: List[str],
triggerTranscriptId: Optional[str],
) -> None:
"""Background wrapper for ``_runAgentForMeeting`` that holds the
``_agentEscalationInFlight`` flag for the duration of the agent run."""
try:
await service._runAgentForMeeting(
sessionId=sessionId,
taskText=taskBrief,
fileIds=briefingFileIds,
sourceLabel="speechEscalation",
triggerTranscriptId=triggerTranscriptId,
)
except asyncio.CancelledError:
logger.info(
f"Session {sessionId}: Escalation agent task cancelled by stop signal"
)
except Exception as escErr:
logger.error(
f"Session {sessionId}: Escalation agent task failed: "
f"{type(escErr).__name__}: {escErr}",
exc_info=True,
)
finally:
service._agentEscalationInFlight = False
service._currentEscalationTask = None

545
modules/features/teamsbot/serviceWebSocket.py Normal file
View file

@ -0,0 +1,545 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Teamsbot Service — WebSocket handler & audio chunk processing.
Extracted from service.py. All functions accept `service` (a TeamsbotService
instance) as the first parameter so the class can delegate to them.
"""
import logging
import json
import asyncio
import time
import base64
from typing import Optional, Dict, Any
from fastapi import WebSocket
from modules.shared.timeUtils import getUtcTimestamp
logger = logging.getLogger(__name__)
async def handleBotWebSocket(service, websocket: WebSocket, sessionId: str):
"""Main WebSocket handler for Browser Bot communication."""
from . import interfaceFeatureTeamsbot as interfaceDb
from modules.interfaces.interfaceVoiceObjects import getVoiceInterface
from .service import _activeServices, _emitSessionEvent, sessionEvents
from .serviceConversation import _processTranscript, _warmEphemeralPhrasePool
interface = interfaceDb.getInterface(service.currentUser, service.mandateId, service.instanceId)
voiceInterface = getVoiceInterface(service.currentUser, service.mandateId)
session = interface.getSession(sessionId)
if session:
rawContext = session.get("sessionContext")
if rawContext and len(rawContext) > 500:
logger.info(f"Session {sessionId}: Summarizing long session context ({len(rawContext)} chars)...")
service._sessionContext = await service._summarizeSessionContext(sessionId, rawContext)
elif rawContext:
service._sessionContext = rawContext
if service._sessionContext:
logger.info(f"Session {sessionId}: Session context ready ({len(service._sessionContext)} chars)")
try:
systemBot = interface.getActiveSystemBot(service.mandateId)
service._botAccountEmail = systemBot.get("email") if systemBot else None
if service._botAccountEmail:
logger.info(f"Session {sessionId}: Bot account email resolved: {service._botAccountEmail}")
except Exception:
service._botAccountEmail = None
service._activeSessionId = sessionId
service._websocket = websocket
service._voiceInterface = voiceInterface
_activeServices[sessionId] = service
try:
await _emitSessionEvent(sessionId, "botConnectionState", {
"connected": True,
"timestamp": getUtcTimestamp(),
})
except Exception:
pass
try:
service._activePersistentPrompts = interface.getActivePersistentPrompts(sessionId) or []
if service._activePersistentPrompts:
logger.info(
f"Session {sessionId}: Loaded {len(service._activePersistentPrompts)} active persistent director prompt(s)"
)
except Exception as restoreErr:
logger.warning(f"Session {sessionId}: Could not restore persistent director prompts: {restoreErr}")
service._activePersistentPrompts = []
asyncio.create_task(_warmEphemeralPhrasePool(service, sessionId))
logger.info(f"[WS] Handler started for session {sessionId}")
try:
msgCount = 0
while True:
data = await websocket.receive()
msgCount += 1
if "text" not in data:
logger.debug(f"[WS] session={sessionId} msg #{msgCount}: non-text data (keys: {list(data.keys())})")
continue
message = json.loads(data["text"])
msgType = message.get("type")
if msgType not in ("audioChunk", "ping"):
logger.info(f"[WS] session={sessionId} msg #{msgCount}: type={msgType}")
if msgType == "transcript":
transcript = message.get("transcript", {})
source = transcript.get("source", "caption")
speaker = transcript.get("speaker", "Unknown")
textPreview = (transcript.get("text", "") or "")[:60]
logger.info(f"[WS] Transcript (source={source}, speaker={speaker}): {textPreview}...")
await _processTranscript(
service,
sessionId=sessionId,
speaker=transcript.get("speaker", "Unknown"),
text=transcript.get("text", ""),
isFinal=transcript.get("isFinal", True),
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
source=source,
)
elif msgType == "chatMessage":
chat = message.get("chat", {})
isHistory = chat.get("isHistory", False)
source = "chatHistory" if isHistory else "chat"
logger.info(
f"[WS] Chat{'[HISTORY]' if isHistory else ''}: "
f"speaker={chat.get('speaker')}, text={chat.get('text', '')[:60]}..."
)
await _processTranscript(
service,
sessionId=sessionId,
speaker=chat.get("speaker", "Unknown"),
text=chat.get("text", ""),
isFinal=True,
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
source=source,
)
elif msgType == "status":
status = message.get("status")
errorMessage = message.get("message")
logger.info(f"[WS] Status: status={status}, message={errorMessage}")
await _handleBotStatus(service, sessionId, status, errorMessage, interface)
elif msgType == "audioChunk":
audioData = message.get("audio", {})
audioBase64 = audioData.get("data", "")
sampleRate = audioData.get("sampleRate", 16000)
captureDiagnostics = audioData.get("captureDiagnostics") or {}
if audioBase64:
await _processAudioChunk(
service,
sessionId=sessionId,
audioBase64=audioBase64,
sampleRate=sampleRate,
captureDiagnostics=captureDiagnostics,
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
)
elif msgType == "voiceGreeting":
greetingText = message.get("text", "")
greetingLang = message.get("language", service.config.language)
logger.info(
f"[WS] Voice greeting (legacy): text={greetingText[:60]}..., language={greetingLang}"
)
if greetingText and voiceInterface:
await service._dispatchGreetingToMeeting(
sessionId=sessionId,
greetingText=greetingText,
greetingLang=greetingLang,
sendToChat=False,
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
)
elif msgType == "requestGreeting":
requestedLang = (
message.get("language") or service.config.language or ""
).strip() or "en-US"
botNameHint = (
message.get("botName") or service.config.botName or ""
).strip() or service.config.botName
logger.info(
f"[WS] Greeting request from bot: language={requestedLang}, name={botNameHint}"
)
if voiceInterface:
try:
greetingText = await service._generateGreetingText(
requestedLang
)
except Exception as genErr:
logger.warning(
f"Greeting generation failed for session {sessionId}: {genErr}"
)
greetingText = ""
if greetingText:
await service._dispatchGreetingToMeeting(
sessionId=sessionId,
greetingText=greetingText,
greetingLang=requestedLang,
sendToChat=True,
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
)
else:
logger.warning(
f"Session {sessionId}: Skipping greeting — AI generation produced no text"
)
elif msgType == "ping":
await websocket.send_text(json.dumps({"type": "pong"}))
elif msgType == "ttsPlaybackAck":
playback = message.get("playback", {}) or {}
status = playback.get("status", "unknown")
ackMessage = playback.get("message") or "Bot playback status update"
logger.info(
f"[WS] TTS playback ack: status={status}, format={playback.get('format')}, "
f"bytesBase64={playback.get('bytesBase64')}"
)
await _emitSessionEvent(sessionId, "ttsDeliveryStatus", {
"status": f"playback_{status}",
"hasWebSocket": True,
"message": ackMessage,
"timestamp": playback.get("timestamp") or getUtcTimestamp(),
"format": playback.get("format"),
"bytesBase64": playback.get("bytesBase64"),
})
elif msgType == "mfaChallenge":
mfaData = message.get("mfa", {})
mfaType = mfaData.get("type", "unknown")
displayNumber = mfaData.get("displayNumber")
prompt = mfaData.get("prompt", "")
logger.info(f"[WS] MFA challenge: type={mfaType}, number={displayNumber}, prompt={prompt[:60]}")
await _emitSessionEvent(sessionId, "mfaChallenge", {
"mfaType": mfaType,
"displayNumber": displayNumber,
"prompt": prompt,
"timestamp": getUtcTimestamp(),
})
from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks
mfaQueue = asyncio.Queue()
mfaCodeQueues[sessionId] = mfaQueue
mfaWaitTasks[sessionId] = asyncio.create_task(
_waitAndForwardMfa(sessionId, mfaQueue, websocket)
)
elif msgType == "chatSendFailed":
errorData = message.get("error", {})
reason = errorData.get("reason", "unknown")
failedText = errorData.get("text", "")
logger.warning(
f"[WS] Chat send failed for session {sessionId}: "
f"reason={reason}, text={failedText[:60]}"
)
await _emitSessionEvent(sessionId, "chatSendFailed", {
"reason": reason,
"message": errorData.get("message", "Chat message could not be sent"),
"text": failedText,
"timestamp": getUtcTimestamp(),
})
elif msgType == "mfaResolved":
success = message.get("success", False)
logger.info(f"[WS] MFA resolved: success={success}")
from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks
task = mfaWaitTasks.pop(sessionId, None)
if task and not task.done():
task.cancel()
mfaCodeQueues.pop(sessionId, None)
await _emitSessionEvent(sessionId, "mfaResolved", {
"success": success,
"timestamp": getUtcTimestamp(),
})
except Exception as e:
if "disconnect" not in str(e).lower():
logger.error(f"[WS] Error for session {sessionId}: {type(e).__name__}: {e}")
finally:
if _activeServices.get(sessionId) is service:
_activeServices.pop(sessionId, None)
service._websocket = None
service._voiceInterface = None
service._activeSessionId = None
try:
await _emitSessionEvent(sessionId, "botConnectionState", {
"connected": False,
"timestamp": getUtcTimestamp(),
})
except Exception:
pass
logger.info(f"[WS] Handler ended for session {sessionId} after {msgCount} messages")
async def _waitAndForwardMfa(sid: str, queue: asyncio.Queue, ws: WebSocket):
"""Wait for an MFA code from the operator and forward it to the bot."""
from .service import _emitSessionEvent
from .routeFeatureTeamsbot import mfaCodeQueues, mfaWaitTasks
try:
mfaResponse = await asyncio.wait_for(queue.get(), timeout=120.0)
logger.info(f"[WS] MFA response received for session {sid}: action={mfaResponse.get('action')}")
await ws.send_text(json.dumps({
"type": "mfaResponse",
"sessionId": sid,
"mfa": mfaResponse,
}))
except asyncio.TimeoutError:
logger.warning(f"[WS] MFA response timeout for session {sid}")
await ws.send_text(json.dumps({
"type": "mfaResponse",
"sessionId": sid,
"mfa": {"action": "timeout"},
}))
await _emitSessionEvent(sid, "mfaChallenge", {
"mfaType": "timeout",
"prompt": "MFA-Zeitlimit ueberschritten. Bitte erneut versuchen.",
})
except asyncio.CancelledError:
logger.info(f"[WS] MFA wait cancelled for session {sid} (resolved via page)")
finally:
mfaCodeQueues.pop(sid, None)
mfaWaitTasks.pop(sid, None)
async def _handleBotStatus(
service,
sessionId: str,
status: str,
errorMessage: Optional[str],
interface,
):
"""Handle status updates from the browser bot."""
from .service import _emitSessionEvent
from .datamodelTeamsbot import TeamsbotSessionStatus
logger.info(f"Bot status update for session {sessionId}: {status}")
statusMap = {
"connecting": TeamsbotSessionStatus.JOINING.value,
"launching": TeamsbotSessionStatus.JOINING.value,
"navigating": TeamsbotSessionStatus.JOINING.value,
"in_lobby": TeamsbotSessionStatus.JOINING.value,
"joined": TeamsbotSessionStatus.ACTIVE.value,
"in_meeting": TeamsbotSessionStatus.ACTIVE.value,
"left": TeamsbotSessionStatus.ENDED.value,
"error": TeamsbotSessionStatus.ERROR.value,
}
dbStatus = statusMap.get(status, TeamsbotSessionStatus.ACTIVE.value)
updates = {"status": dbStatus}
if errorMessage:
updates["errorMessage"] = errorMessage
if dbStatus == TeamsbotSessionStatus.ACTIVE.value:
updates["startedAt"] = getUtcTimestamp()
elif dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]:
updates["endedAt"] = getUtcTimestamp()
interface.updateSession(sessionId, updates)
await _emitSessionEvent(sessionId, "statusChange", {"status": status, "errorMessage": errorMessage})
if dbStatus in [TeamsbotSessionStatus.ENDED.value, TeamsbotSessionStatus.ERROR.value]:
if service._audioBuffer:
logger.info(f"[AudioChunk] Flushing remaining buffer on session end ({len(service._audioBuffer)} bytes)")
service._audioBuffer = b""
service._audioBufferStartTime = 0.0
service._audioBufferLastChunkTime = 0.0
if dbStatus == TeamsbotSessionStatus.ENDED.value:
asyncio.create_task(service._generateMeetingSummary(sessionId))
async def _processAudioChunk(
service,
sessionId: str,
audioBase64: str,
sampleRate: int,
captureDiagnostics: Optional[Dict[str, Any]],
interface,
voiceInterface,
websocket: WebSocket,
):
"""Process an audio chunk from WebRTC capture."""
from .serviceConversation import _processTranscript
_MIN_CHUNK_SEC = 1.0
_STALE_TIMEOUT_SEC = 3.0
try:
audioBytes = base64.b64decode(audioBase64)
if len(audioBytes) < 500:
return
if captureDiagnostics:
trackId = captureDiagnostics.get("trackId")
readyState = captureDiagnostics.get("readyState")
rms = captureDiagnostics.get("rms")
nativeSampleRate = captureDiagnostics.get("nativeSampleRate")
logger.debug(
f"[AudioChunk] diagnostics: track={trackId}, readyState={readyState}, "
f"rms={rms}, nativeRate={nativeSampleRate}, bytes={len(audioBytes)}"
)
isSilent = False
if captureDiagnostics and captureDiagnostics.get("rms") is not None:
try:
rmsVal = float(captureDiagnostics.get("rms"))
if rmsVal < 0.0003:
isSilent = True
except Exception:
pass
if not voiceInterface:
logger.warning(f"[AudioChunk] No voice interface available for session {sessionId}")
return
now = time.time()
effectiveRate = sampleRate if sampleRate and sampleRate > 0 else 16000
if not isSilent:
if not service._audioBuffer:
service._audioBufferStartTime = now
service._audioBuffer += audioBytes
service._audioBufferLastChunkTime = now
service._audioBufferSampleRate = effectiveRate
bufferDuration = len(service._audioBuffer) / (effectiveRate * 2) if service._audioBuffer else 0.0
bufferAge = (now - service._audioBufferStartTime) if service._audioBuffer else 0.0
shouldFlush = (
service._audioBuffer
and (
bufferDuration >= _MIN_CHUNK_SEC
or (bufferAge >= _STALE_TIMEOUT_SEC and bufferDuration > 0.3)
)
)
if not shouldFlush:
return
flushBytes = service._audioBuffer
flushRate = service._audioBufferSampleRate
service._audioBuffer = b""
service._audioBufferStartTime = 0.0
service._audioBufferLastChunkTime = 0.0
flushDuration = len(flushBytes) / (flushRate * 2)
logger.info(f"[AudioChunk] Flushing buffer: {len(flushBytes)} bytes, {flushDuration:.1f}s, {flushRate}Hz")
phraseHints = list(service._knownSpeakers)
if service.config.botName:
phraseHints.append(service.config.botName)
sttResult = await voiceInterface.speechToText(
audioContent=flushBytes,
language=service.config.language or "de-DE",
sampleRate=flushRate,
channels=1,
skipFallbacks=True,
phraseHints=phraseHints if phraseHints else None,
audioFormat="linear16",
)
if sttResult and sttResult.get("success") and sttResult.get("text"):
text = sttResult["text"].strip()
if text:
resolvedSpeaker = service._resolveSpeakerForAudioCapture()
fromCaption = resolvedSpeaker.get("speakerResolvedFromHint", False)
logger.info(
f"[AudioChunk] STT result: speaker={resolvedSpeaker.get('speaker', 'Meeting Audio')} "
f"(fromCaption={fromCaption}), text={text[:80]}..."
)
await _processTranscript(
service,
sessionId=sessionId,
speaker=resolvedSpeaker["speaker"],
text=text,
isFinal=True,
interface=interface,
voiceInterface=voiceInterface,
websocket=websocket,
source="audioCapture",
speakerResolvedFromHint=resolvedSpeaker["speakerResolvedFromHint"],
)
except Exception as e:
logger.error(f"[AudioChunk] STT error for session {sessionId}: {type(e).__name__}: {e}")
async def _cancelInFlightSpeech(
service,
sessionId: str,
websocket: Optional[WebSocket],
reason: str,
) -> None:
"""Hard stop everything the bot is currently doing in the meeting."""
from .service import _emitSessionEvent
service._answerGenerationCounter += 1
gen = service._answerGenerationCounter
logger.info(
f"Session {sessionId}: Cancelling in-flight speech "
f"(reason={reason}, gen={gen})"
)
if service._pendingNameTrigger:
logger.info(
f"Session {sessionId}: Dropping pending debounced name "
f"trigger (was queued before stop)"
)
service._pendingNameTrigger = None
for taskAttr in ("_currentEscalationTask", "_currentQuickAckTask"):
task = getattr(service, taskAttr, None)
if task is not None and not task.done():
logger.info(
f"Session {sessionId}: Cancelling background task "
f"{taskAttr}"
)
task.cancel()
if websocket is not None:
try:
await websocket.send_text(json.dumps({
"type": "stopAudio",
"sessionId": sessionId,
"reason": reason,
}))
except Exception as stopErr:
logger.warning(
f"Session {sessionId}: Failed to send stopAudio to "
f"browser bot: {stopErr}"
)
try:
await _emitSessionEvent(sessionId, "speechCancelled", {
"reason": reason,
"generation": gen,
"timestamp": getUtcTimestamp(),
})
except Exception:
pass

371
modules/features/trustee/handlerTrusteeAccounting.py Normal file
View file

@ -0,0 +1,371 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Business logic for Trustee accounting integration endpoints.
Extracted from routeFeatureTrustee.py for maintainability.
"""
import json
import logging
import time
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
_CONFIG_PLACEHOLDER = "***"
class SaveAccountingConfigBody(BaseModel):
"""Request body for saving accounting config."""
connectorType: str = ""
displayLabel: str = ""
config: Dict[str, Any] = Field(default_factory=dict, description="Connector credentials (e.g. clientName, apiKey)")
def getConfigMasked(connectorType: str, plainConfig: Dict[str, Any]) -> Dict[str, str]:
"""Build config with secret values replaced by placeholder for GET response."""
from .accounting.accountingRegistry import getAccountingRegistry
connector = getAccountingRegistry().getConnector(connectorType)
if not connector:
return {k: (v if isinstance(v, str) else str(v)) for k, v in (plainConfig or {}).items()}
secretKeys = {f.key for f in connector.getRequiredConfigFields() if f.secret}
return {
k: _CONFIG_PLACEHOLDER if k in secretKeys else (v if isinstance(v, str) else str(v) if v is not None else "")
for k, v in (plainConfig or {}).items()
}
async def refreshChartSilently(interface, instanceId: str) -> None:
"""Best-effort chart-of-accounts cache refresh. Logs but does not raise on failure."""
try:
from .accounting.accountingBridge import AccountingBridge
bridge = AccountingBridge(interface)
charts = await bridge.refreshChartOfAccounts(instanceId)
logger.info(f"Chart cache refreshed: {len(charts)} entries for instance {instanceId}")
except Exception as e:
logger.warning(f"Chart cache refresh failed (non-critical): {e}")
def readAccountingConfig(interface, instanceId: str) -> Dict[str, Any]:
"""Read and return the masked accounting config for an instance."""
from .datamodelFeatureTrustee import TrusteeAccountingConfig
from modules.shared.configuration import decryptValue
records = interface.db.getRecordset(
TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True}
)
if not records:
return {"configured": False}
record = {k: v for k, v in records[0].items() if not k.startswith("_")}
encryptedConfig = record.pop("encryptedConfig", None)
record["configured"] = True
if encryptedConfig:
try:
plain = json.loads(decryptValue(encryptedConfig, keyName="accountingConfig"))
record["configMasked"] = getConfigMasked(record.get("connectorType", ""), plain)
except Exception:
record["configMasked"] = {}
else:
record["configMasked"] = {}
return record
async def saveAccountingConfig(interface, instanceId: str, mandateId: str, body: "SaveAccountingConfigBody") -> Dict[str, Any]:
"""Save or update accounting config with encrypted credentials and config merging."""
import uuid as _uuid
from .datamodelFeatureTrustee import TrusteeAccountingConfig
from modules.shared.configuration import encryptValue, decryptValue
plainConfig = body.config if isinstance(body.config, dict) else {}
if not plainConfig and body.connectorType:
logger.warning("Accounting config save: config is empty (credentials will not be stored or updated)")
else:
logger.info(
"Accounting config save: instanceId=%s connectorType=%s configKeys=%s",
instanceId, body.connectorType, list(plainConfig.keys())
)
existing = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId})
if existing:
configId = existing[0].get("id")
updatePayload = {
"connectorType": body.connectorType or "",
"displayLabel": body.displayLabel or "",
"isActive": True,
}
if plainConfig:
existingEnc = existing[0].get("encryptedConfig") or ""
merged = {}
if existingEnc:
try:
merged = json.loads(decryptValue(existingEnc, keyName="accountingConfig"))
except Exception:
pass
for k, v in plainConfig.items():
if v is not None and str(v).strip() and str(v).strip() != _CONFIG_PLACEHOLDER:
merged[k] = v
updatePayload["encryptedConfig"] = encryptValue(json.dumps(merged), keyName="accountingConfig")
interface.db.recordModify(TrusteeAccountingConfig, configId, updatePayload)
await refreshChartSilently(interface, instanceId)
return {"message": "Accounting config updated", "id": configId}
if not plainConfig:
return None # Signal to route handler: raise 400
encryptedConfig = encryptValue(json.dumps(plainConfig), keyName="accountingConfig")
configRecord = {
"id": str(_uuid.uuid4()),
"featureInstanceId": instanceId,
"connectorType": body.connectorType or "",
"displayLabel": body.displayLabel or "",
"encryptedConfig": encryptedConfig,
"isActive": True,
"mandateId": mandateId,
}
interface.db.recordCreate(TrusteeAccountingConfig, configRecord)
await refreshChartSilently(interface, instanceId)
return {"message": "Accounting config created", "id": configRecord["id"]}
def getImportStatus(interface, instanceId: str) -> Dict[str, Any]:
"""Get counts of imported TrusteeData* records for this instance."""
from .datamodelFeatureTrustee import (
TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine,
TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig,
)
filt = {"featureInstanceId": instanceId}
counts = {
"accounts": len(interface.db.getRecordset(TrusteeDataAccount, recordFilter=filt) or []),
"journalEntries": len(interface.db.getRecordset(TrusteeDataJournalEntry, recordFilter=filt) or []),
"journalLines": len(interface.db.getRecordset(TrusteeDataJournalLine, recordFilter=filt) or []),
"contacts": len(interface.db.getRecordset(TrusteeDataContact, recordFilter=filt) or []),
"accountBalances": len(interface.db.getRecordset(TrusteeDataAccountBalance, recordFilter=filt) or []),
}
cfgRecords = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True})
if cfgRecords:
cfg = cfgRecords[0]
counts["lastSyncAt"] = cfg.get("lastSyncAt")
counts["lastSyncStatus"] = cfg.get("lastSyncStatus")
counts["lastSyncErrorMessage"] = cfg.get("lastSyncErrorMessage")
counts["lastSyncDateFrom"] = cfg.get("lastSyncDateFrom")
counts["lastSyncDateTo"] = cfg.get("lastSyncDateTo")
counts["lastSyncCounts"] = cfg.get("lastSyncCounts")
return counts
def wipeImportedData(interface, instanceId: str) -> Dict[str, Any]:
"""Delete all TrusteeData* rows imported for this instance and reset sync markers."""
from .datamodelFeatureTrustee import (
TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine,
TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig,
)
from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache
removed: Dict[str, int] = {}
for tableName, model in [
("accounts", TrusteeDataAccount),
("journalEntries", TrusteeDataJournalEntry),
("journalLines", TrusteeDataJournalLine),
("contacts", TrusteeDataContact),
("accountBalances", TrusteeDataAccountBalance),
]:
try:
removed[tableName] = int(interface.db.recordDeleteWhere(model, {"featureInstanceId": instanceId}) or 0)
except Exception as ex:
logger.warning("wipeImportedData: failed for %s: %s", tableName, ex)
removed[tableName] = 0
cfgRecords = interface.db.getRecordset(
TrusteeAccountingConfig,
recordFilter={"featureInstanceId": instanceId, "isActive": True},
)
if cfgRecords:
cfgId = cfgRecords[0].get("id")
if cfgId:
try:
interface.db.recordModify(TrusteeAccountingConfig, cfgId, {
"lastSyncAt": None,
"lastSyncStatus": None,
"lastSyncErrorMessage": None,
"lastSyncDateFrom": None,
"lastSyncDateTo": None,
"lastSyncCounts": None,
})
except Exception as ex:
logger.warning("wipeImportedData: failed to reset lastSync* on cfg %s: %s", cfgId, ex)
cacheCleared = clearFeatureQueryCache(instanceId)
logger.info("wipeImportedData instance=%s removed=%s cacheCleared=%s", instanceId, removed, cacheCleared)
return {
"removed": removed,
"totalRemoved": sum(removed.values()),
"cacheCleared": cacheCleared,
"featureInstanceId": instanceId,
}
def exportAccountingData(interface, instanceId: str, mandateId: str) -> Dict[str, Any]:
"""Build the export payload for all TrusteeData* tables for this instance."""
from .datamodelFeatureTrustee import (
TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine,
TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig,
)
_filter = {"featureInstanceId": instanceId}
tables: Dict[str, Any] = {}
for tableName, model in [
("TrusteeDataAccount", TrusteeDataAccount),
("TrusteeDataJournalEntry", TrusteeDataJournalEntry),
("TrusteeDataJournalLine", TrusteeDataJournalLine),
("TrusteeDataContact", TrusteeDataContact),
("TrusteeDataAccountBalance", TrusteeDataAccountBalance),
]:
records = interface.db.getRecordset(model, recordFilter=_filter) or []
tables[tableName] = records
cfgRecords = interface.db.getRecordset(
TrusteeAccountingConfig,
recordFilter={"featureInstanceId": instanceId, "isActive": True},
)
syncInfo = {}
if cfgRecords:
cfg = cfgRecords[0]
syncInfo = {
"connectorType": cfg.get("connectorType", ""),
"lastSyncAt": cfg.get("lastSyncAt"),
"lastSyncStatus": cfg.get("lastSyncStatus", ""),
}
return {
"exportedAt": time.time(),
"featureInstanceId": instanceId,
"mandateId": mandateId,
"syncInfo": syncInfo,
"tables": tables,
}
# ---------------------------------------------------------------------------
# Background Job Handlers
# ---------------------------------------------------------------------------
TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE = "trusteeAccountingPush"
TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE = "trusteeAccountingSync"
async def accountingPushJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]:
"""BackgroundJob handler: pushes a batch of positions to the external accounting system."""
from modules.security.rootAccess import getRootUser
from .accounting.accountingBridge import AccountingBridge, SyncResult
from .interfaceFeatureTrustee import getInterface
instanceId = job["featureInstanceId"]
mandateId = job["mandateId"]
payload = job.get("payload") or {}
positionIds: List[str] = list(payload.get("positionIds") or [])
if not positionIds:
return {"total": 0, "success": 0, "skipped": 0, "errors": 0, "results": []}
rootUser = getRootUser()
interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId)
bridge = AccountingBridge(interface)
results = []
total = len(positionIds)
progressCb(
2,
messageKey="Sync wird vorbereitet ({total} Position(en))...",
messageParams={"total": total},
)
try:
connector, plainConfig, configRecord = await bridge._resolveConnectorAndConfig(instanceId)
except Exception as resolveErr:
logger.exception("Accounting push: failed to resolve connector/config")
progressCb(100, messageKey="Verbindungsaufbau fehlgeschlagen.")
raise resolveErr
if not connector or not plainConfig:
results = [SyncResult(success=False, errorMessage="No active accounting configuration found") for _ in positionIds]
progressCb(100, messageKey="Keine aktive Buchhaltungs-Konfiguration gefunden.")
return {
"total": len(results),
"success": 0,
"skipped": 0,
"errors": len(results),
"results": [r.model_dump() for r in results],
}
for index, positionId in enumerate(positionIds, start=1):
result = await bridge.pushPositionToAccounting(
instanceId,
positionId,
_resolvedConnector=connector,
_resolvedPlainConfig=plainConfig,
_resolvedConfigRecord=configRecord,
)
results.append(result)
pct = 5 + int(90 * index / total)
progressCb(
pct,
messageKey="Position {index}/{total} verarbeitet",
messageParams={"index": index, "total": total},
)
skipped = [r for r in results if not r.success and r.errorMessage and "already synced" in r.errorMessage]
failed = [r for r in results if not r.success and r not in skipped]
if skipped:
logger.info("Accounting sync: %s position(s) already synced, skipped", len(skipped))
if failed:
logger.warning(
"Accounting sync had %s failure(s): %s",
len(failed),
"; ".join(r.errorMessage or "unknown" for r in failed[:3]),
)
progressCb(100, messageKey="Sync abgeschlossen.")
return {
"total": len(results),
"success": sum(1 for r in results if r.success),
"skipped": len(skipped),
"errors": len(failed),
"results": [r.model_dump() for r in results],
}
async def accountingSyncJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]:
"""BackgroundJob handler: imports accounting data from the external system."""
from modules.security.rootAccess import getRootUser
from .accounting.accountingDataSync import AccountingDataSync
from .interfaceFeatureTrustee import getInterface
instanceId = job["featureInstanceId"]
mandateId = job["mandateId"]
payload = job.get("payload") or {}
rootUser = getRootUser()
progressCb(5, messageKey="Initialisiere Import...")
interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId)
sync = AccountingDataSync(interface)
progressCb(10, messageKey="Verbinde mit Buchhaltungssystem...")
result = await sync.importData(
featureInstanceId=instanceId,
mandateId=mandateId,
dateFrom=payload.get("dateFrom"),
dateTo=payload.get("dateTo"),
progressCb=progressCb,
)
progressCb(100, messageKey="Import abgeschlossen.")
return result
# Register background job handlers
try:
from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler
registerJobHandler(TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE, accountingPushJobHandler)
registerJobHandler(TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, accountingSyncJobHandler)
except Exception as _regErr:
logger.warning("Failed to register accounting job handlers: %s", _regErr)

24
modules/features/trustee/mainTrustee.py
View file

@ -1097,3 +1097,27 @@ def onMandateDelete(mandateId: str, instances: list) -> None:
except Exception as e:
logger.warning(f"Failed to cascade-delete trustee data for mandate {mandateId}: {e}")
def onUserDelete(userId: str, currentUser) -> dict:
"""Delete/anonymize user data from the trustee database (GDPR)."""
from modules.system.gdprDeletion import deleteUserDataFromDatabase
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.shared.configuration import APP_CONFIG
dbName = "poweron_trustee"
try:
db = DatabaseConnector(
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
dbDatabase=dbName,
dbUser=APP_CONFIG.get("DB_USER"),
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"),
dbPort=int(APP_CONFIG.get("DB_PORT", 5432)),
userId=None,
)
stats = deleteUserDataFromDatabase(db, userId, dbName)
db.close()
return stats
except Exception as e:
logger.warning(f"onUserDelete trustee failed: {e}")
return {"database": dbName, "tablesProcessed": 0, "recordsDeleted": 0, "recordsAnonymized": 0, "errors": [str(e)]}

451
modules/features/trustee/routeFeatureTrustee.py
View file

@ -1395,6 +1395,19 @@ def delete_position(
# ===== Accounting Integration Endpoints =====
from .handlerTrusteeAccounting import (
SaveAccountingConfigBody,
readAccountingConfig,
saveAccountingConfig as _saveAccountingConfig,
refreshChartSilently as _refreshChartSilently,
getImportStatus as _getImportStatus,
wipeImportedData as _wipeImportedData,
exportAccountingData as _exportAccountingData,
TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE,
TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE,
)
@router.get("/{instanceId}/accounting/connectors")
@limiter.limit("30/minute")
def get_available_accounting_connectors(
@ -1408,23 +1421,6 @@ def get_available_accounting_connectors(
return getAccountingRegistry().getAvailableConnectors()
# Placeholder returned for secret config fields so frontend can prefill form without sending real secrets.
_CONFIG_PLACEHOLDER = "***"
def _getConfigMasked(connectorType: str, plainConfig: Dict[str, Any]) -> Dict[str, str]:
"""Build config with secret values replaced by placeholder for GET response."""
from .accounting.accountingRegistry import getAccountingRegistry
connector = getAccountingRegistry().getConnector(connectorType)
if not connector:
return {k: (v if isinstance(v, str) else str(v)) for k, v in (plainConfig or {}).items()}
secretKeys = {f.key for f in connector.getRequiredConfigFields() if f.secret}
return {
k: _CONFIG_PLACEHOLDER if k in secretKeys else (v if isinstance(v, str) else str(v) if v is not None else "")
for k, v in (plainConfig or {}).items()
}
@router.get("/{instanceId}/accounting/config")
@limiter.limit("30/minute")
def get_accounting_config(
@ -1432,33 +1428,10 @@ def get_accounting_config(
instanceId: str = Path(..., description="Feature Instance ID"),
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Get the active accounting config for this instance. Credentials are masked (secret fields = ***) for form prefill."""
"""Get the active accounting config for this instance. Credentials are masked."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingConfig
from modules.shared.configuration import decryptValue
records = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True})
if not records:
return {"configured": False}
record = {k: v for k, v in records[0].items() if not k.startswith("_")}
encryptedConfig = record.pop("encryptedConfig", None)
record["configured"] = True
if encryptedConfig:
try:
plain = json.loads(decryptValue(encryptedConfig, keyName="accountingConfig"))
record["configMasked"] = _getConfigMasked(record.get("connectorType", ""), plain)
except Exception:
record["configMasked"] = {}
else:
record["configMasked"] = {}
return record
class SaveAccountingConfigBody(BaseModel):
"""Request body for saving accounting config. Ensures 'config' is present and used."""
connectorType: str = ""
displayLabel: str = ""
config: Dict[str, Any] = Field(default_factory=dict, description="Connector credentials (e.g. clientName, apiKey)")
return readAccountingConfig(interface, instanceId)
@router.post("/{instanceId}/accounting/config", status_code=201)
@ -1469,73 +1442,16 @@ async def save_accounting_config(
body: SaveAccountingConfigBody = Body(...),
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Save or update the accounting config for this instance.
Body: { connectorType, displayLabel, config: { clientName, apiKey, ... } }
The 'config' object is stored encrypted; without it credentials would be empty in DB.
"""
"""Save or update the accounting config for this instance."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingConfig
from modules.shared.configuration import encryptValue
plainConfig = body.config if isinstance(body.config, dict) else {}
# When updating, empty config is normal (frontend never receives credentials from GET).
# Do not overwrite encryptedConfig with empty – keep existing credentials.
if not plainConfig and body.connectorType:
logger.warning("Accounting config save: config is empty (credentials will not be stored or updated)")
else:
logger.info(
"Accounting config save: instanceId=%s connectorType=%s configKeys=%s",
instanceId, body.connectorType, list(plainConfig.keys())
)
existing = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId})
if existing:
configId = existing[0].get("id")
updatePayload = {
"connectorType": body.connectorType or "",
"displayLabel": body.displayLabel or "",
"isActive": True,
}
if plainConfig:
# Merge with existing: placeholder or empty = keep existing value (so form prefill does not overwrite secrets).
from modules.shared.configuration import decryptValue
existingEnc = existing[0].get("encryptedConfig") or ""
merged = {}
if existingEnc:
try:
merged = json.loads(decryptValue(existingEnc, keyName="accountingConfig"))
except Exception:
pass
for k, v in plainConfig.items():
if v is not None and str(v).strip() and str(v).strip() != _CONFIG_PLACEHOLDER:
merged[k] = v
updatePayload["encryptedConfig"] = encryptValue(json.dumps(merged), keyName="accountingConfig")
interface.db.recordModify(TrusteeAccountingConfig, configId, updatePayload)
await _refreshChartSilently(interface, instanceId)
return {"message": "Accounting config updated", "id": configId}
if not plainConfig:
result = await _saveAccountingConfig(interface, instanceId, mandateId, body)
if result is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=routeApiMsg("config is required for new integration (e.g. clientName, apiKey).")
)
encryptedConfig = encryptValue(json.dumps(plainConfig), keyName="accountingConfig")
configRecord = {
"id": str(uuid.uuid4()),
"featureInstanceId": instanceId,
"connectorType": body.connectorType or "",
"displayLabel": body.displayLabel or "",
"encryptedConfig": encryptedConfig,
"isActive": True,
"mandateId": mandateId,
}
interface.db.recordCreate(TrusteeAccountingConfig, configRecord)
await _refreshChartSilently(interface, instanceId)
return {"message": "Accounting config created", "id": configRecord["id"]}
return result
@router.post("/{instanceId}/accounting/test-connection")
@ -1545,7 +1461,7 @@ async def test_accounting_connection(
instanceId: str = Path(..., description="Feature Instance ID"),
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Test the connection to the configured accounting system. On success, refreshes the local chart-of-accounts cache."""
"""Test the connection to the configured accounting system."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .accounting.accountingBridge import AccountingBridge
@ -1581,7 +1497,7 @@ async def get_chart_of_accounts(
accountType: Optional[str] = Query(None, description="Filter by type: expense, asset, liability, revenue"),
context: RequestContext = Depends(getRequestContext)
) -> List[Dict[str, Any]]:
"""Load the chart of accounts from the connected accounting system. Optional filter by accountType."""
"""Load the chart of accounts from the connected accounting system."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .accounting.accountingBridge import AccountingBridge
@ -1590,17 +1506,6 @@ async def get_chart_of_accounts(
return [c.model_dump() for c in charts]
async def _refreshChartSilently(interface, instanceId: str) -> None:
"""Best-effort chart-of-accounts cache refresh. Logs but does not raise on failure."""
try:
from .accounting.accountingBridge import AccountingBridge
bridge = AccountingBridge(interface)
charts = await bridge.refreshChartOfAccounts(instanceId)
logger.info(f"Chart cache refreshed: {len(charts)} entries for instance {instanceId}")
except Exception as e:
logger.warning(f"Chart cache refresh failed (non-critical): {e}")
@router.post("/{instanceId}/accounting/refresh-chart")
@limiter.limit("5/minute")
async def refresh_chart_of_accounts(
@ -1617,108 +1522,6 @@ async def refresh_chart_of_accounts(
return {"message": f"Chart of accounts refreshed: {len(charts)} entries", "count": len(charts)}
TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE = "trusteeAccountingPush"
async def _trusteeAccountingPushJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]:
"""BackgroundJob handler: pushes a batch of positions to the external
accounting system. Runs in the worker without blocking the original HTTP
request, so the user can continue navigating while the sync runs.
Reads inputs from `job["payload"]` (`positionIds`) and reports incremental
progress via `progressCb(percent, message)`. The job result has the same
shape that the legacy synchronous endpoint used to return.
"""
from modules.security.rootAccess import getRootUser
from .accounting.accountingBridge import AccountingBridge
instanceId = job["featureInstanceId"]
mandateId = job["mandateId"]
payload = job.get("payload") or {}
positionIds: List[str] = list(payload.get("positionIds") or [])
if not positionIds:
return {"total": 0, "success": 0, "skipped": 0, "errors": 0, "results": []}
rootUser = getRootUser()
interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId)
bridge = AccountingBridge(interface)
results = []
total = len(positionIds)
progressCb(
2,
messageKey="Sync wird vorbereitet ({total} Position(en))...",
messageParams={"total": total},
)
# Resolve connector + plain config once to avoid decryption rate-limits
# (mirrors the optimisation in pushBatchToAccounting). We push positions
# one-by-one inside the job so we can emit incremental progress and so
# one bad row never aborts the rest.
from .accounting.accountingBridge import SyncResult
try:
connector, plainConfig, configRecord = await bridge._resolveConnectorAndConfig(instanceId)
except Exception as resolveErr:
logger.exception("Accounting push: failed to resolve connector/config")
progressCb(100, messageKey="Verbindungsaufbau fehlgeschlagen.")
raise resolveErr
if not connector or not plainConfig:
results = [SyncResult(success=False, errorMessage="No active accounting configuration found") for _ in positionIds]
progressCb(100, messageKey="Keine aktive Buchhaltungs-Konfiguration gefunden.")
return {
"total": len(results),
"success": 0,
"skipped": 0,
"errors": len(results),
"results": [r.model_dump() for r in results],
}
for index, positionId in enumerate(positionIds, start=1):
result = await bridge.pushPositionToAccounting(
instanceId,
positionId,
_resolvedConnector=connector,
_resolvedPlainConfig=plainConfig,
_resolvedConfigRecord=configRecord,
)
results.append(result)
# Reserve 5..95% for the push loop, keep the tail for summary.
pct = 5 + int(90 * index / total)
progressCb(
pct,
messageKey="Position {index}/{total} verarbeitet",
messageParams={"index": index, "total": total},
)
skipped = [r for r in results if not r.success and r.errorMessage and "already synced" in r.errorMessage]
failed = [r for r in results if not r.success and r not in skipped]
if skipped:
logger.info("Accounting sync: %s position(s) already synced, skipped", len(skipped))
if failed:
logger.warning(
"Accounting sync had %s failure(s): %s",
len(failed),
"; ".join(r.errorMessage or "unknown" for r in failed[:3]),
)
progressCb(100, messageKey="Sync abgeschlossen.")
return {
"total": len(results),
"success": sum(1 for r in results if r.success),
"skipped": len(skipped),
"errors": len(failed),
"results": [r.model_dump() for r in results],
}
try:
from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler as _registerPushJobHandler
_registerPushJobHandler(TRUSTEE_ACCOUNTING_PUSH_JOB_TYPE, _trusteeAccountingPushJobHandler)
except Exception as _pushRegErr:
logger.warning("Failed to register trusteeAccountingPush job handler: %s", _pushRegErr)
@router.post("/{instanceId}/accounting/sync", status_code=status.HTTP_202_ACCEPTED)
@limiter.limit("5/minute")
async def sync_positions_to_accounting(
@ -1727,21 +1530,10 @@ async def sync_positions_to_accounting(
data: Dict[str, Any] = Body(...),
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Submit a background job that pushes positions to the accounting system.
Body: ``{ positionIds: [...] }``
Returns ``{ jobId, status: "pending" }`` immediately so the user is not
blocked while the (potentially long) external accounting calls run.
Clients poll ``GET /api/jobs/{jobId}`` until status is ``SUCCESS`` /
``ERROR`` and then read the same ``{ total, success, skipped, errors,
results }`` payload from ``job.result`` that the legacy synchronous
endpoint returned.
"""
"""Submit a background job that pushes positions to the accounting system."""
from modules.serviceCenter.services.serviceBackgroundJobs import startJob
mandateId = _validateInstanceAccess(instanceId, context)
positionIds = data.get("positionIds", [])
if not positionIds:
raise HTTPException(status_code=400, detail=routeApiMsg("positionIds required"))
@ -1771,11 +1563,7 @@ async def sync_single_position_to_accounting(
bridge = AccountingBridge(interface)
result = await bridge.pushPositionToAccounting(instanceId, positionId)
if not result.success:
logger.warning(
"Accounting sync failed for positionId=%s: %s",
positionId,
result.errorMessage or "unknown",
)
logger.warning("Accounting sync failed for positionId=%s: %s", positionId, result.errorMessage or "unknown")
return result.model_dump()
@ -1791,8 +1579,7 @@ def get_sync_status(
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingSync
records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId})
items = [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]
return {"items": items}
return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]}
@router.get("/{instanceId}/accounting/sync-status/{positionId}")
@ -1808,52 +1595,11 @@ def get_position_sync_status(
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingSync
records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"positionId": positionId, "featureInstanceId": instanceId})
items = [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]
return {"items": items}
return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]}
# ===== Accounting Data Import =====
TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE = "trusteeAccountingSync"
async def _trusteeAccountingSyncJobHandler(job: Dict[str, Any], progressCb) -> Dict[str, Any]:
"""BackgroundJob handler: imports accounting data from the external system.
Reads inputs from `job["payload"]` (dateFrom, dateTo, userId) and runs
`AccountingDataSync.importData(...)` in the worker's event loop without
blocking the original HTTP request that submitted the job.
"""
from modules.security.rootAccess import getRootUser
from .accounting.accountingDataSync import AccountingDataSync
instanceId = job["featureInstanceId"]
mandateId = job["mandateId"]
payload = job.get("payload") or {}
rootUser = getRootUser()
progressCb(5, messageKey="Initialisiere Import...")
interface = getInterface(rootUser, mandateId=mandateId, featureInstanceId=instanceId)
sync = AccountingDataSync(interface)
progressCb(10, messageKey="Verbinde mit Buchhaltungssystem...")
result = await sync.importData(
featureInstanceId=instanceId,
mandateId=mandateId,
dateFrom=payload.get("dateFrom"),
dateTo=payload.get("dateTo"),
progressCb=progressCb,
)
progressCb(100, messageKey="Import abgeschlossen.")
return result
try:
from modules.serviceCenter.services.serviceBackgroundJobs import registerJobHandler
registerJobHandler(TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE, _trusteeAccountingSyncJobHandler)
except Exception as _regErr:
logger.warning("Failed to register trusteeAccountingSync job handler: %s", _regErr)
@router.post("/{instanceId}/accounting/import-data", status_code=status.HTTP_202_ACCEPTED)
@limiter.limit("3/minute")
async def import_accounting_data(
@ -1862,18 +1608,11 @@ async def import_accounting_data(
data: Dict[str, Any] = Body(default={}),
context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
"""Submit a background job to import accounting data.
Returns immediately with `{ jobId }`; clients poll `GET /api/jobs/{jobId}`
until status is SUCCESS / ERROR.
"""
"""Submit a background job to import accounting data."""
from modules.serviceCenter.services.serviceBackgroundJobs import startJob
mandateId = _validateInstanceAccess(instanceId, context)
payload = {
"dateFrom": data.get("dateFrom"),
"dateTo": data.get("dateTo"),
}
payload = {"dateFrom": data.get("dateFrom"), "dateTo": data.get("dateTo")}
jobId = await startJob(
TRUSTEE_ACCOUNTING_SYNC_JOB_TYPE,
payload,
@ -1894,28 +1633,7 @@ def get_import_status(
"""Get counts of imported TrusteeData* records for this instance."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import (
TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine,
TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig,
)
filt = {"featureInstanceId": instanceId}
counts = {
"accounts": len(interface.db.getRecordset(TrusteeDataAccount, recordFilter=filt) or []),
"journalEntries": len(interface.db.getRecordset(TrusteeDataJournalEntry, recordFilter=filt) or []),
"journalLines": len(interface.db.getRecordset(TrusteeDataJournalLine, recordFilter=filt) or []),
"contacts": len(interface.db.getRecordset(TrusteeDataContact, recordFilter=filt) or []),
"accountBalances": len(interface.db.getRecordset(TrusteeDataAccountBalance, recordFilter=filt) or []),
}
cfgRecords = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId, "isActive": True})
if cfgRecords:
cfg = cfgRecords[0]
counts["lastSyncAt"] = cfg.get("lastSyncAt")
counts["lastSyncStatus"] = cfg.get("lastSyncStatus")
counts["lastSyncErrorMessage"] = cfg.get("lastSyncErrorMessage")
counts["lastSyncDateFrom"] = cfg.get("lastSyncDateFrom")
counts["lastSyncDateTo"] = cfg.get("lastSyncDateTo")
counts["lastSyncCounts"] = cfg.get("lastSyncCounts")
return counts
return _getImportStatus(interface, instanceId)
# ===== AI Data Cache =====
@ -1927,12 +1645,7 @@ def clear_ai_data_cache(
instanceId: str = Path(..., description="Feature Instance ID"),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, Any]:
"""Clear ONLY the AI feature-data query result cache (in-memory, ~5 min TTL).
Important: this does NOT touch the synchronised ``TrusteeData*`` tables.
The synced rows (chart of accounts, journal entries/lines, contacts, balances)
stay exactly as imported. To wipe those rows, use POST .../wipe-imported-data.
"""
"""Clear the AI feature-data query result cache (in-memory)."""
_validateInstanceAccess(instanceId, context)
from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache
removed = clearFeatureQueryCache(instanceId)
@ -1946,66 +1659,10 @@ def wipe_imported_accounting_data(
instanceId: str = Path(..., description="Feature Instance ID"),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, Any]:
"""Delete every ``TrusteeData*`` row imported for this feature instance.
Use when the source system was changed, test data needs to be cleared, or
the user suspects stale rows from earlier connector versions. Also resets
the ``lastSync*`` markers on the active config so the UI no longer reports
a stale "letzter Import" status. The connector configuration / credentials
remain untouched -- only synchronised payload data is removed.
"""
"""Delete all TrusteeData* rows imported for this feature instance and reset sync markers."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import (
TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine,
TrusteeDataContact, TrusteeDataAccountBalance, TrusteeAccountingConfig,
)
from modules.serviceCenter.services.serviceAgent.coreTools._featureSubAgentTools import clearFeatureQueryCache
removed: Dict[str, int] = {}
for tableName, model in [
("accounts", TrusteeDataAccount),
("journalEntries", TrusteeDataJournalEntry),
("journalLines", TrusteeDataJournalLine),
("contacts", TrusteeDataContact),
("accountBalances", TrusteeDataAccountBalance),
]:
try:
removed[tableName] = int(interface.db.recordDeleteWhere(model, {"featureInstanceId": instanceId}) or 0)
except Exception as ex:
logger.warning("wipeImportedData: failed for %s: %s", tableName, ex)
removed[tableName] = 0
cfgRecords = interface.db.getRecordset(
TrusteeAccountingConfig,
recordFilter={"featureInstanceId": instanceId, "isActive": True},
)
if cfgRecords:
cfgId = cfgRecords[0].get("id")
if cfgId:
try:
interface.db.recordModify(TrusteeAccountingConfig, cfgId, {
"lastSyncAt": None,
"lastSyncStatus": None,
"lastSyncErrorMessage": None,
"lastSyncDateFrom": None,
"lastSyncDateTo": None,
"lastSyncCounts": None,
})
except Exception as ex:
logger.warning("wipeImportedData: failed to reset lastSync* on cfg %s: %s", cfgId, ex)
cacheCleared = clearFeatureQueryCache(instanceId)
logger.info(
"wipeImportedData instance=%s removed=%s cacheCleared=%s",
instanceId, removed, cacheCleared,
)
return {
"removed": removed,
"totalRemoved": sum(removed.values()),
"cacheCleared": cacheCleared,
"featureInstanceId": instanceId,
}
return _wipeImportedData(interface, instanceId)
# ===== Data Export =====
@ -2017,52 +1674,10 @@ def export_accounting_data(
instanceId: str = Path(..., description="Feature Instance ID"),
context: RequestContext = Depends(getRequestContext),
) -> Response:
"""Export all TrusteeData* tables for this instance as a JSON download (admin only)."""
"""Export all TrusteeData* tables for this instance as a JSON download."""
mandateId = _validateInstanceAccess(instanceId, context)
from .datamodelFeatureTrustee import (
TrusteeDataAccount,
TrusteeDataJournalEntry,
TrusteeDataJournalLine,
TrusteeDataContact,
TrusteeDataAccountBalance,
TrusteeAccountingConfig,
)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
_filter = {"featureInstanceId": instanceId}
tables: Dict[str, Any] = {}
for tableName, model in [
("TrusteeDataAccount", TrusteeDataAccount),
("TrusteeDataJournalEntry", TrusteeDataJournalEntry),
("TrusteeDataJournalLine", TrusteeDataJournalLine),
("TrusteeDataContact", TrusteeDataContact),
("TrusteeDataAccountBalance", TrusteeDataAccountBalance),
]:
records = interface.db.getRecordset(model, recordFilter=_filter) or []
tables[tableName] = records
cfgRecords = interface.db.getRecordset(
TrusteeAccountingConfig,
recordFilter={"featureInstanceId": instanceId, "isActive": True},
)
syncInfo = {}
if cfgRecords:
cfg = cfgRecords[0]
syncInfo = {
"connectorType": cfg.get("connectorType", ""),
"lastSyncAt": cfg.get("lastSyncAt"),
"lastSyncStatus": cfg.get("lastSyncStatus", ""),
}
payload = {
"exportedAt": time.time(),
"featureInstanceId": instanceId,
"mandateId": mandateId,
"syncInfo": syncInfo,
"tables": tables,
}
payload = _exportAccountingData(interface, instanceId, mandateId)
jsonBytes = json.dumps(payload, ensure_ascii=False, default=str).encode("utf-8")
return Response(
content=jsonBytes,

3
modules/features/trustee/workflows/__init__.py Normal file
View file

@ -0,0 +1,3 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""Trustee feature-owned workflow methods."""

0
modules/workflows/methods/methodTrustee/__init__.py → modules/features/trustee/workflows/methodTrustee/__init__.py
View file

2
modules/workflows/methods/methodTrustee/actions/extractFromFiles.py → modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py
View file

@ -19,7 +19,7 @@ from typing import Dict, Any, List, Optional, Tuple
from modules.datamodels.datamodelChat import ActionResult, ActionDocument, ChatDocument, ChatMessage
from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference
from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum
from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
logger = logging.getLogger(__name__)

0
modules/workflows/methods/methodTrustee/actions/processDocuments.py → modules/features/trustee/workflows/methodTrustee/actions/processDocuments.py
View file

0
modules/workflows/methods/methodTrustee/actions/queryData.py → modules/features/trustee/workflows/methodTrustee/actions/queryData.py
View file

0
modules/workflows/methods/methodTrustee/actions/refreshAccountingData.py → modules/features/trustee/workflows/methodTrustee/actions/refreshAccountingData.py
View file

0
modules/workflows/methods/methodTrustee/actions/syncToAccounting.py → modules/features/trustee/workflows/methodTrustee/actions/syncToAccounting.py
View file

0
modules/workflows/methods/methodTrustee/methodTrustee.py → modules/features/trustee/workflows/methodTrustee/methodTrustee.py
View file

10
modules/interfaces/interfaceBootstrap.py
View file

@ -111,7 +111,7 @@ def initBootstrap(db: DatabaseConnector) -> None:
logger.warning(f"Mandate retention purge failed: {e}")
# Let features run their own bootstrap logic via lifecycle hooks
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
for _fCode, _fMod in loadFeatureMainModules().items():
_bootHook = getattr(_fMod, "onBootstrap", None)
if _bootHook:
@ -172,7 +172,7 @@ def initRootMandateFeatures(db: DatabaseConnector, mandateId: str) -> None:
"""
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.interfaces.interfaceFeatures import getFeatureInterface
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
logger.info("Initializing root mandate features")
@ -241,7 +241,7 @@ def initRootMandateFeatures(db: DatabaseConnector, mandateId: str) -> None:
def _cleanupRemovedFeatureInstances(db: DatabaseConnector) -> None:
"""Remove feature instances whose featureCode no longer exists in the codebase."""
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
mainModules = loadFeatureMainModules()
activeCodes = set()
@ -1144,7 +1144,7 @@ def _createUiContextRules(db: DatabaseConnector) -> None:
Args:
db: Database connector instance
"""
from modules.system.mainSystem import NAVIGATION_SECTIONS
from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS
uiRules = []
adminId = _getRoleId(db, "admin")
@ -1200,7 +1200,7 @@ def _ensureUiContextRules(db: DatabaseConnector) -> None:
Args:
db: Database connector instance
"""
from modules.system.mainSystem import NAVIGATION_SECTIONS
from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS
# Template role IDs
adminId = _getRoleId(db, "admin")

4
modules/interfaces/interfaceDbApp.py
View file

@ -1579,7 +1579,7 @@ class AppObjects:
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
from modules.interfaces.interfaceFeatures import getFeatureInterface
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
plan = BUILTIN_PLANS.get(planKey)
if not plan:
raise ValueError(f"Unknown plan: {planKey}")
@ -1871,7 +1871,7 @@ class AppObjects:
instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
# 0-pre. Let features cascade-delete their own data via lifecycle hooks
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
for _fCode, _fMod in loadFeatureMainModules().items():
_hook = getattr(_fMod, "onMandateDelete", None)
if _hook:

2
modules/interfaces/interfaceDbSubscription.py
View file

@ -30,7 +30,7 @@ from modules.datamodels.datamodelSubscription import (
getEffectiveLimits,
)
from modules.shared.serviceExceptions import SubscriptionCapacityException
from modules.datamodels.serviceExceptions import SubscriptionCapacityException
logger = logging.getLogger(__name__)

2
modules/interfaces/interfaceFeatures.py
View file

@ -287,7 +287,7 @@ class FeatureInterface:
RuntimeError: If templates exist but cannot be copied.
Caller decides whether to swallow or re-raise.
"""
from modules.system.registry import loadFeatureMainModules
from modules.shared.featureDiscovery import loadFeatureMainModules
mainModules = loadFeatureMainModules()
featureModule = mainModules.get(featureCode)
if not featureModule:

399
modules/routes/billingWebhookHandler.py Normal file
View file

@ -0,0 +1,399 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Stripe webhook and subscription business logic for billing.
Extracted from routeBilling.py for maintainability.
"""
import logging
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from fastapi import HTTPException
from modules.datamodels.datamodelBilling import (
BillingTransaction,
TransactionTypeEnum,
ReferenceTypeEnum,
)
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeBilling")
logger = logging.getLogger(__name__)
def creditStripeSessionIfNeeded(
billingInterface,
session: Dict[str, Any],
eventId: Optional[str] = None,
CheckoutConfirmResponse=None,
):
"""Credit balance from Stripe Checkout session if not already credited.
Uses Checkout session ID for idempotency across webhook + manual confirmation flows.
"""
from modules.serviceCenter.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF
session_id = session.get("id")
metadata = session.get("metadata") or {}
mandate_id = metadata.get("mandateId")
user_id = metadata.get("userId") or None
amount_chf_str = metadata.get("amountChf", "0")
if not session_id:
raise HTTPException(status_code=400, detail=routeApiMsg("Stripe session id missing"))
if not mandate_id:
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid session metadata: mandateId missing"))
existing_payment_tx = billingInterface.getPaymentTransactionByReferenceId(session_id)
if existing_payment_tx:
if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId):
billingInterface.createStripeWebhookEvent(eventId)
return CheckoutConfirmResponse(
credited=False,
alreadyCredited=True,
sessionId=session_id,
mandateId=mandate_id,
amountChf=float(existing_payment_tx.get("amount", 0.0)),
)
try:
amount_chf = float(amount_chf_str)
except (TypeError, ValueError):
amount_chf = None
if amount_chf is None or amount_chf not in ALLOWED_AMOUNTS_CHF:
amount_total = session.get("amount_total")
if amount_total is not None:
amount_chf = amount_total / 100.0
else:
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid amount in Stripe session"))
settings = billingInterface.getSettings(mandate_id)
if not settings:
raise HTTPException(status_code=404, detail=routeApiMsg("Billing settings not found"))
account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0)
transaction = BillingTransaction(
accountId=account["id"],
transactionType=TransactionTypeEnum.CREDIT,
amount=amount_chf,
description="Stripe-Zahlung",
referenceType=ReferenceTypeEnum.PAYMENT,
referenceId=session_id,
createdByUserId=user_id,
)
billingInterface.createTransaction(transaction)
if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId):
billingInterface.createStripeWebhookEvent(eventId)
logger.info(f"Stripe credit applied: {amount_chf} CHF for session {session_id} on mandate {mandate_id}")
return CheckoutConfirmResponse(
credited=True,
alreadyCredited=False,
sessionId=session_id,
mandateId=mandate_id,
amountChf=amount_chf,
)
def handleSubscriptionCheckoutCompleted(session, eventId: str, getRootInterface) -> None:
"""Handle checkout.session.completed for mode=subscription.
Resolves the local PENDING record by ID from webhook metadata and transitions it."""
from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface
from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import (
getService as getSubscriptionService,
_notifySubscriptionChange,
)
from modules.security.rootAccess import getRootUser
if not isinstance(session, dict):
from modules.shared.stripeClient import stripeToDict
session = stripeToDict(session)
metadata = session.get("metadata") or {}
subscriptionRecordId = metadata.get("subscriptionRecordId")
mandateId = metadata.get("mandateId")
planKey = metadata.get("planKey", "")
platformUrl = metadata.get("platformUrl", "")
if not subscriptionRecordId:
stripeSub = session.get("subscription")
if stripeSub:
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
from modules.shared.stripeClient import stripeToDict
subObj = stripeToDict(stripe.Subscription.retrieve(stripeSub))
metadata = subObj.get("metadata") or {}
subscriptionRecordId = metadata.get("subscriptionRecordId")
mandateId = metadata.get("mandateId")
planKey = metadata.get("planKey", "")
platformUrl = platformUrl or metadata.get("platformUrl", "")
except Exception as e:
logger.error(
"Stripe Subscription.retrieve(%s) failed during checkout "
"metadata recovery: %s", stripeSub, e,
)
raise
stripeSubId = session.get("subscription")
if not mandateId or not subscriptionRecordId:
logger.warning("Subscription checkout missing metadata: %s", metadata)
return
subInterface = getSubRootInterface()
rootUser = getRootUser()
sub = subInterface.getById(subscriptionRecordId)
if not sub:
logger.error("Subscription record %s not found for checkout webhook", subscriptionRecordId)
return
if sub.get("status") != SubscriptionStatusEnum.PENDING.value:
logger.warning("Subscription %s is %s, expected PENDING — skipping", subscriptionRecordId, sub.get("status"))
return
stripeData: Dict[str, Any] = {}
if stripeSubId:
stripeData["stripeSubscriptionId"] = stripeSubId
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
from modules.shared.stripeClient import stripeToDict
stripeSub = stripeToDict(stripe.Subscription.retrieve(stripeSubId, expand=["items"]))
if stripeSub.get("current_period_start"):
stripeData["currentPeriodStart"] = float(stripeSub["current_period_start"])
if stripeSub.get("current_period_end"):
stripeData["currentPeriodEnd"] = float(stripeSub["current_period_end"])
from modules.serviceCenter.services.serviceSubscription.stripeBootstrap import getStripePricesForPlan
priceMapping = getStripePricesForPlan(planKey)
items = stripeSub.get("items") or {}
if not isinstance(items, dict):
items = dict(items)
for item in items.get("data", []):
priceId = (item.get("price") or {}).get("id", "")
if priceMapping and priceId == priceMapping.stripePriceIdUsers:
stripeData["stripeItemIdUsers"] = item["id"]
elif priceMapping and priceId == priceMapping.stripePriceIdInstances:
stripeData["stripeItemIdInstances"] = item["id"]
except Exception as e:
logger.error(
"Error retrieving Stripe subscription %s during checkout "
"completion (will be retried by Stripe): %s",
stripeSubId, e,
)
raise
if stripeData:
subInterface.updateFields(subscriptionRecordId, stripeData)
operative = subInterface.getOperativeForMandate(mandateId)
hasActivePredecessor = operative is not None and operative["id"] != subscriptionRecordId
predecessorIsTrial = (
hasActivePredecessor
and operative.get("status") == SubscriptionStatusEnum.TRIALING.value
)
if hasActivePredecessor and predecessorIsTrial:
try:
subInterface.forceExpire(operative["id"])
logger.info(
"Trial subscription %s expired immediately for mandate %s due to paid upgrade %s",
operative["id"], mandateId, subscriptionRecordId,
)
except Exception as e:
logger.error("Failed to expire trial predecessor %s: %s", operative["id"], e)
toStatus = SubscriptionStatusEnum.ACTIVE
elif hasActivePredecessor:
toStatus = SubscriptionStatusEnum.SCHEDULED
if operative.get("recurring", True):
operativeStripeId = operative.get("stripeSubscriptionId")
if operativeStripeId:
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
stripe.Subscription.modify(operativeStripeId, cancel_at_period_end=True)
except Exception as e:
logger.error("Failed to set cancel_at_period_end on predecessor %s: %s", operativeStripeId, e)
subInterface.updateFields(operative["id"], {"recurring": False})
effectiveFrom = operative.get("currentPeriodEnd")
if effectiveFrom:
subInterface.updateFields(subscriptionRecordId, {"effectiveFrom": effectiveFrom})
else:
toStatus = SubscriptionStatusEnum.ACTIVE
try:
subInterface.transitionStatus(
subscriptionRecordId, SubscriptionStatusEnum.PENDING, toStatus,
{"recurring": True},
)
except Exception as e:
logger.error("Failed to transition subscription %s: %s", subscriptionRecordId, e)
return
subService = getSubscriptionService(rootUser, mandateId)
subService.invalidateCache(mandateId)
if toStatus == SubscriptionStatusEnum.ACTIVE:
plan = getPlan(planKey)
updatedSub = subInterface.getById(subscriptionRecordId)
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=updatedSub, platformUrl=platformUrl)
try:
billingIf = getRootInterface()
billingIf.creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung")
except Exception as ex:
logger.error("creditSubscriptionBudget on activation failed: %s", ex)
logger.info(
"Checkout completed: sub=%s -> %s, mandate=%s, plan=%s",
subscriptionRecordId, toStatus.value, mandateId, planKey,
)
def handleSubscriptionWebhook(event, getRootInterface) -> None:
"""Process Stripe subscription webhook events.
All record resolution is by stripeSubscriptionId."""
from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface
from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import (
getService as getSubscriptionService,
_notifySubscriptionChange,
)
from modules.security.rootAccess import getRootUser
obj = event.data.object
rawSub = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription")
stripeSubId = rawSub.get("id") if isinstance(rawSub, dict) else rawSub
if not stripeSubId:
logger.warning("Subscription webhook %s has no subscription ID", event.type)
return
subInterface = getSubRootInterface()
sub = subInterface.getByStripeSubscriptionId(stripeSubId)
if not sub:
logger.warning("No local record for Stripe subscription %s (event: %s)", stripeSubId, event.type)
return
subId = sub["id"]
mandateId = sub["mandateId"]
currentStatus = SubscriptionStatusEnum(sub["status"])
rootUser = getRootUser()
subService = getSubscriptionService(rootUser, mandateId)
subMetadata = obj.get("metadata") or {}
webhookPlatformUrl = subMetadata.get("platformUrl", "")
if event.type == "customer.subscription.updated":
stripeStatus = obj.get("status", "")
periodData: Dict[str, Any] = {}
if obj.get("current_period_start"):
periodData["currentPeriodStart"] = float(obj["current_period_start"])
if obj.get("current_period_end"):
periodData["currentPeriodEnd"] = float(obj["current_period_end"])
if periodData:
subInterface.updateFields(subId, periodData)
if stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.SCHEDULED:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE)
subService.invalidateCache(mandateId)
planKey = sub.get("planKey", "")
plan = getPlan(planKey)
refreshedSub = subInterface.getById(subId)
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedSub, platformUrl=webhookPlatformUrl)
try:
getRootInterface().creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung")
except Exception as ex:
logger.error("creditSubscriptionBudget SCHEDULED->ACTIVE failed: %s", ex)
logger.info("SCHEDULED -> ACTIVE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.PAST_DUE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.PAST_DUE, SubscriptionStatusEnum.ACTIVE)
subService.invalidateCache(mandateId)
logger.info("PAST_DUE -> ACTIVE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "past_due" and currentStatus == SubscriptionStatusEnum.ACTIVE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE)
subService.invalidateCache(mandateId)
logger.info("ACTIVE -> PAST_DUE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.ACTIVE:
subService.invalidateCache(mandateId)
logger.info("Period renewed for sub %s (mandate %s)", subId, mandateId)
elif event.type == "customer.subscription.deleted":
if currentStatus not in (SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE,
SubscriptionStatusEnum.SCHEDULED):
logger.info("Ignoring deletion for sub %s in status %s", subId, currentStatus.value)
return
subInterface.transitionStatus(subId, currentStatus, SubscriptionStatusEnum.EXPIRED)
subService.invalidateCache(mandateId)
logger.info("Sub %s -> EXPIRED (Stripe deleted, mandate %s)", subId, mandateId)
scheduled = subInterface.getScheduledForMandate(mandateId)
if scheduled:
try:
subInterface.transitionStatus(
scheduled["id"], SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE,
)
subService.invalidateCache(mandateId)
plan = getPlan(scheduled.get("planKey", ""))
refreshedScheduled = subInterface.getById(scheduled["id"])
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedScheduled, platformUrl=webhookPlatformUrl)
logger.info("Promoted SCHEDULED sub %s -> ACTIVE (mandate %s)", scheduled["id"], mandateId)
except Exception as e:
logger.error("Failed to promote SCHEDULED sub %s: %s", scheduled["id"], e)
elif event.type == "invoice.payment_failed":
if currentStatus == SubscriptionStatusEnum.ACTIVE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE)
subService.invalidateCache(mandateId)
plan = getPlan(sub.get("planKey", ""))
_notifySubscriptionChange(mandateId, "payment_failed", plan, subscriptionRecord=sub, platformUrl=webhookPlatformUrl)
logger.info("Payment failed for sub %s (mandate %s)", subId, mandateId)
elif event.type == "customer.subscription.trial_will_end":
logger.info("Trial ending soon for sub %s (mandate %s)", subId, mandateId)
try:
from modules.system.notifyMandateAdmins import notifyMandateAdmins
notifyMandateAdmins(
mandateId,
"[PowerOn] Testphase endet bald",
"Testphase endet bald",
[
"Die kostenlose Testphase für Ihren Mandanten endet in Kürze.",
"Bitte wählen Sie einen Plan unter Billing-Verwaltung › Abonnement.",
],
)
except Exception as e:
logger.error("Failed to notify about trial ending: %s", e)
elif event.type == "invoice.paid":
period_ts = obj.get("period_start")
periodLabel = ""
if period_ts:
period_start_at = datetime.fromtimestamp(int(period_ts), tz=timezone.utc)
periodLabel = period_start_at.strftime("%Y-%m-%d")
try:
billing_if = getRootInterface()
billing_if.resetStorageBillingPeriod(mandateId, period_start_at)
billing_if.reconcileMandateStorageBilling(mandateId)
except Exception as ex:
logger.error("Storage billing on invoice.paid failed: %s", ex)
planKey = sub.get("planKey", "")
try:
billing_if = getRootInterface()
billing_if.creditSubscriptionBudget(mandateId, planKey, periodLabel=periodLabel or "Periodenverlängerung")
except Exception as ex:
logger.error("creditSubscriptionBudget on invoice.paid failed: %s", ex)
logger.info("Invoice paid for sub %s (mandate %s)", subId, mandateId)
return None

389
modules/routes/routeBilling.py
View file

@ -330,80 +330,10 @@ def _getStripeClient():
return getStripeClient()
def _creditStripeSessionIfNeeded(
billingInterface,
session: Dict[str, Any],
eventId: Optional[str] = None,
) -> CheckoutConfirmResponse:
"""
Credit balance from Stripe Checkout session if not already credited.
Uses Checkout session ID for idempotency across webhook + manual confirmation flows.
"""
from modules.serviceCenter.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF
session_id = session.get("id")
metadata = session.get("metadata") or {}
mandate_id = metadata.get("mandateId")
user_id = metadata.get("userId") or None
amount_chf_str = metadata.get("amountChf", "0")
if not session_id:
raise HTTPException(status_code=400, detail=routeApiMsg("Stripe session id missing"))
if not mandate_id:
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid session metadata: mandateId missing"))
existing_payment_tx = billingInterface.getPaymentTransactionByReferenceId(session_id)
if existing_payment_tx:
if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId):
billingInterface.createStripeWebhookEvent(eventId)
return CheckoutConfirmResponse(
credited=False,
alreadyCredited=True,
sessionId=session_id,
mandateId=mandate_id,
amountChf=float(existing_payment_tx.get("amount", 0.0)),
)
try:
amount_chf = float(amount_chf_str)
except (TypeError, ValueError):
amount_chf = None
if amount_chf is None or amount_chf not in ALLOWED_AMOUNTS_CHF:
amount_total = session.get("amount_total")
if amount_total is not None:
amount_chf = amount_total / 100.0
else:
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid amount in Stripe session"))
settings = billingInterface.getSettings(mandate_id)
if not settings:
raise HTTPException(status_code=404, detail=routeApiMsg("Billing settings not found"))
account = billingInterface.getOrCreateMandateAccount(mandate_id, initialBalance=0.0)
transaction = BillingTransaction(
accountId=account["id"],
transactionType=TransactionTypeEnum.CREDIT,
amount=amount_chf,
description="Stripe-Zahlung",
referenceType=ReferenceTypeEnum.PAYMENT,
referenceId=session_id,
createdByUserId=user_id,
)
billingInterface.createTransaction(transaction)
if eventId and not billingInterface.getStripeWebhookEventByEventId(eventId):
billingInterface.createStripeWebhookEvent(eventId)
logger.info(f"Stripe credit applied: {amount_chf} CHF for session {session_id} on mandate {mandate_id}")
return CheckoutConfirmResponse(
credited=True,
alreadyCredited=False,
sessionId=session_id,
mandateId=mandate_id,
amountChf=amount_chf,
)
def _creditStripeSessionIfNeeded(billingInterface, session: Dict[str, Any], eventId: Optional[str] = None) -> CheckoutConfirmResponse:
"""Credit balance from Stripe Checkout session if not already credited."""
from .billingWebhookHandler import creditStripeSessionIfNeeded
return creditStripeSessionIfNeeded(billingInterface, session, eventId, CheckoutConfirmResponse)
# =============================================================================
@ -1148,314 +1078,15 @@ async def stripeWebhook(
def handleSubscriptionCheckoutCompleted(session, eventId: str) -> None:
"""Handle checkout.session.completed for mode=subscription.
Resolves the local PENDING record by ID from webhook metadata and transitions it."""
from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface
from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import (
getService as getSubscriptionService,
_notifySubscriptionChange,
)
from modules.security.rootAccess import getRootUser
if not isinstance(session, dict):
from modules.shared.stripeClient import stripeToDict
session = stripeToDict(session)
metadata = session.get("metadata") or {}
subscriptionRecordId = metadata.get("subscriptionRecordId")
mandateId = metadata.get("mandateId")
planKey = metadata.get("planKey", "")
platformUrl = metadata.get("platformUrl", "")
if not subscriptionRecordId:
stripeSub = session.get("subscription")
if stripeSub:
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
from modules.shared.stripeClient import stripeToDict
subObj = stripeToDict(stripe.Subscription.retrieve(stripeSub))
metadata = subObj.get("metadata") or {}
subscriptionRecordId = metadata.get("subscriptionRecordId")
mandateId = metadata.get("mandateId")
planKey = metadata.get("planKey", "")
platformUrl = platformUrl or metadata.get("platformUrl", "")
except Exception as e:
# Stripe lookup is the only way to recover the metadata at this
# point — if it fails we MUST surface it, otherwise the webhook
# later short-circuits with "missing metadata" and the user
# silently gets stuck in PENDING.
logger.error(
"Stripe Subscription.retrieve(%s) failed during checkout "
"metadata recovery: %s", stripeSub, e,
)
raise
stripeSubId = session.get("subscription")
if not mandateId or not subscriptionRecordId:
logger.warning("Subscription checkout missing metadata: %s", metadata)
return
subInterface = getSubRootInterface()
rootUser = getRootUser()
sub = subInterface.getById(subscriptionRecordId)
if not sub:
logger.error("Subscription record %s not found for checkout webhook", subscriptionRecordId)
return
if sub.get("status") != SubscriptionStatusEnum.PENDING.value:
logger.warning("Subscription %s is %s, expected PENDING — skipping", subscriptionRecordId, sub.get("status"))
return
stripeData: Dict[str, Any] = {}
if stripeSubId:
stripeData["stripeSubscriptionId"] = stripeSubId
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
from modules.shared.stripeClient import stripeToDict
stripeSub = stripeToDict(stripe.Subscription.retrieve(stripeSubId, expand=["items"]))
if stripeSub.get("current_period_start"):
stripeData["currentPeriodStart"] = float(stripeSub["current_period_start"])
if stripeSub.get("current_period_end"):
stripeData["currentPeriodEnd"] = float(stripeSub["current_period_end"])
from modules.serviceCenter.services.serviceSubscription.stripeBootstrap import getStripePricesForPlan
priceMapping = getStripePricesForPlan(planKey)
items = stripeSub.get("items") or {}
if not isinstance(items, dict):
items = dict(items)
for item in items.get("data", []):
priceId = (item.get("price") or {}).get("id", "")
if priceMapping and priceId == priceMapping.stripePriceIdUsers:
stripeData["stripeItemIdUsers"] = item["id"]
elif priceMapping and priceId == priceMapping.stripePriceIdInstances:
stripeData["stripeItemIdInstances"] = item["id"]
except Exception as e:
# Without these enrichment fields the activation completes anyway
# (status flips to ACTIVE/SCHEDULED below), but periods + Stripe
# item-IDs are missing on the local record, which breaks later
# add-on billing and renewal accounting. Re-raise so the webhook
# is retried by Stripe instead of silently shipping a broken row.
logger.error(
"Error retrieving Stripe subscription %s during checkout "
"completion (will be retried by Stripe): %s",
stripeSubId, e,
)
raise
if stripeData:
subInterface.updateFields(subscriptionRecordId, stripeData)
operative = subInterface.getOperativeForMandate(mandateId)
hasActivePredecessor = operative is not None and operative["id"] != subscriptionRecordId
predecessorIsTrial = (
hasActivePredecessor
and operative.get("status") == SubscriptionStatusEnum.TRIALING.value
)
if hasActivePredecessor and predecessorIsTrial:
try:
subInterface.forceExpire(operative["id"])
logger.info(
"Trial subscription %s expired immediately for mandate %s due to paid upgrade %s",
operative["id"], mandateId, subscriptionRecordId,
)
except Exception as e:
logger.error("Failed to expire trial predecessor %s: %s", operative["id"], e)
toStatus = SubscriptionStatusEnum.ACTIVE
elif hasActivePredecessor:
toStatus = SubscriptionStatusEnum.SCHEDULED
if operative.get("recurring", True):
operativeStripeId = operative.get("stripeSubscriptionId")
if operativeStripeId:
try:
from modules.shared.stripeClient import getStripeClient
stripe = getStripeClient()
stripe.Subscription.modify(operativeStripeId, cancel_at_period_end=True)
except Exception as e:
logger.error("Failed to set cancel_at_period_end on predecessor %s: %s", operativeStripeId, e)
subInterface.updateFields(operative["id"], {"recurring": False})
effectiveFrom = operative.get("currentPeriodEnd")
if effectiveFrom:
subInterface.updateFields(subscriptionRecordId, {"effectiveFrom": effectiveFrom})
else:
toStatus = SubscriptionStatusEnum.ACTIVE
try:
subInterface.transitionStatus(
subscriptionRecordId, SubscriptionStatusEnum.PENDING, toStatus,
{"recurring": True},
)
except Exception as e:
logger.error("Failed to transition subscription %s: %s", subscriptionRecordId, e)
return
subService = getSubscriptionService(rootUser, mandateId)
subService.invalidateCache(mandateId)
if toStatus == SubscriptionStatusEnum.ACTIVE:
plan = getPlan(planKey)
updatedSub = subInterface.getById(subscriptionRecordId)
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=updatedSub, platformUrl=platformUrl)
try:
billingIf = getRootInterface()
billingIf.creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung")
except Exception as ex:
logger.error("creditSubscriptionBudget on activation failed: %s", ex)
logger.info(
"Checkout completed: sub=%s -> %s, mandate=%s, plan=%s",
subscriptionRecordId, toStatus.value, mandateId, planKey,
)
"""Handle checkout.session.completed for mode=subscription."""
from .billingWebhookHandler import handleSubscriptionCheckoutCompleted as _handler
_handler(session, eventId, getRootInterface)
def _handleSubscriptionWebhook(event) -> None:
"""Process Stripe subscription webhook events.
All record resolution is by stripeSubscriptionId — no mandate-based guessing."""
from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface
from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum, getPlan
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import (
getService as getSubscriptionService,
_notifySubscriptionChange,
)
from modules.security.rootAccess import getRootUser
obj = event.data.object
rawSub = obj.get("id") if event.type.startswith("customer.subscription") else obj.get("subscription")
stripeSubId = rawSub.get("id") if isinstance(rawSub, dict) else rawSub
if not stripeSubId:
logger.warning("Subscription webhook %s has no subscription ID", event.type)
return
subInterface = getSubRootInterface()
sub = subInterface.getByStripeSubscriptionId(stripeSubId)
if not sub:
logger.warning("No local record for Stripe subscription %s (event: %s)", stripeSubId, event.type)
return
subId = sub["id"]
mandateId = sub["mandateId"]
currentStatus = SubscriptionStatusEnum(sub["status"])
rootUser = getRootUser()
subService = getSubscriptionService(rootUser, mandateId)
subMetadata = obj.get("metadata") or {}
webhookPlatformUrl = subMetadata.get("platformUrl", "")
if event.type == "customer.subscription.updated":
stripeStatus = obj.get("status", "")
periodData: Dict[str, Any] = {}
if obj.get("current_period_start"):
periodData["currentPeriodStart"] = float(obj["current_period_start"])
if obj.get("current_period_end"):
periodData["currentPeriodEnd"] = float(obj["current_period_end"])
if periodData:
subInterface.updateFields(subId, periodData)
if stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.SCHEDULED:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE)
subService.invalidateCache(mandateId)
planKey = sub.get("planKey", "")
plan = getPlan(planKey)
refreshedSub = subInterface.getById(subId)
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedSub, platformUrl=webhookPlatformUrl)
try:
getRootInterface().creditSubscriptionBudget(mandateId, planKey, periodLabel="Erstaktivierung")
except Exception as ex:
logger.error("creditSubscriptionBudget SCHEDULED->ACTIVE failed: %s", ex)
logger.info("SCHEDULED -> ACTIVE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.PAST_DUE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.PAST_DUE, SubscriptionStatusEnum.ACTIVE)
subService.invalidateCache(mandateId)
logger.info("PAST_DUE -> ACTIVE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "past_due" and currentStatus == SubscriptionStatusEnum.ACTIVE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE)
subService.invalidateCache(mandateId)
logger.info("ACTIVE -> PAST_DUE for sub %s (mandate %s)", subId, mandateId)
elif stripeStatus == "active" and currentStatus == SubscriptionStatusEnum.ACTIVE:
subService.invalidateCache(mandateId)
logger.info("Period renewed for sub %s (mandate %s)", subId, mandateId)
elif event.type == "customer.subscription.deleted":
if currentStatus not in (SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE,
SubscriptionStatusEnum.SCHEDULED):
logger.info("Ignoring deletion for sub %s in status %s", subId, currentStatus.value)
return
subInterface.transitionStatus(subId, currentStatus, SubscriptionStatusEnum.EXPIRED)
subService.invalidateCache(mandateId)
logger.info("Sub %s -> EXPIRED (Stripe deleted, mandate %s)", subId, mandateId)
scheduled = subInterface.getScheduledForMandate(mandateId)
if scheduled:
try:
subInterface.transitionStatus(
scheduled["id"], SubscriptionStatusEnum.SCHEDULED, SubscriptionStatusEnum.ACTIVE,
)
subService.invalidateCache(mandateId)
plan = getPlan(scheduled.get("planKey", ""))
refreshedScheduled = subInterface.getById(scheduled["id"])
_notifySubscriptionChange(mandateId, "activated", plan, subscriptionRecord=refreshedScheduled, platformUrl=webhookPlatformUrl)
logger.info("Promoted SCHEDULED sub %s -> ACTIVE (mandate %s)", scheduled["id"], mandateId)
except Exception as e:
logger.error("Failed to promote SCHEDULED sub %s: %s", scheduled["id"], e)
elif event.type == "invoice.payment_failed":
if currentStatus == SubscriptionStatusEnum.ACTIVE:
subInterface.transitionStatus(subId, SubscriptionStatusEnum.ACTIVE, SubscriptionStatusEnum.PAST_DUE)
subService.invalidateCache(mandateId)
plan = getPlan(sub.get("planKey", ""))
_notifySubscriptionChange(mandateId, "payment_failed", plan, subscriptionRecord=sub, platformUrl=webhookPlatformUrl)
logger.info("Payment failed for sub %s (mandate %s)", subId, mandateId)
elif event.type == "customer.subscription.trial_will_end":
logger.info("Trial ending soon for sub %s (mandate %s)", subId, mandateId)
try:
from modules.system.notifyMandateAdmins import notifyMandateAdmins
notifyMandateAdmins(
mandateId,
"[PowerOn] Testphase endet bald",
"Testphase endet bald",
[
"Die kostenlose Testphase für Ihren Mandanten endet in Kürze.",
"Bitte wählen Sie einen Plan unter Billing-Verwaltung › Abonnement.",
],
)
except Exception as e:
logger.error("Failed to notify about trial ending: %s", e)
elif event.type == "invoice.paid":
period_ts = obj.get("period_start")
periodLabel = ""
if period_ts:
period_start_at = datetime.fromtimestamp(int(period_ts), tz=timezone.utc)
periodLabel = period_start_at.strftime("%Y-%m-%d")
try:
billing_if = getRootInterface()
billing_if.resetStorageBillingPeriod(mandateId, period_start_at)
billing_if.reconcileMandateStorageBilling(mandateId)
except Exception as ex:
logger.error("Storage billing on invoice.paid failed: %s", ex)
planKey = sub.get("planKey", "")
try:
billing_if = getRootInterface()
billing_if.creditSubscriptionBudget(mandateId, planKey, periodLabel=periodLabel or "Periodenverlängerung")
except Exception as ex:
logger.error("creditSubscriptionBudget on invoice.paid failed: %s", ex)
logger.info("Invoice paid for sub %s (mandate %s)", subId, mandateId)
return None
"""Process Stripe subscription webhook events."""
from .billingWebhookHandler import handleSubscriptionWebhook as _handler
_handler(event, getRootInterface)
@router.get("/admin/accounts/{targetMandateId}", response_model=List[AccountSummary])

2
modules/routes/routeClickup.py
View file

@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request, sta
from modules.auth import getCurrentUser, limiter
from modules.datamodels.datamodelUam import AuthAuthority, User, UserConnection
from modules.interfaces.interfaceDbApp import getInterface
from modules.serviceHub import getInterface as getServices
from modules.serviceCenter.serviceHub import getInterface as getServices
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeClickup")

2
modules/routes/routeSharepoint.py
View file

@ -12,7 +12,7 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Query, Request, sta
from modules.auth import limiter, getCurrentUser
from modules.datamodels.datamodelUam import User, UserConnection
from modules.interfaces.interfaceDbApp import getInterface
from modules.serviceHub import getInterface as getServices
from modules.serviceCenter.serviceHub import getInterface as getServices
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeSharepoint")

44
modules/security/rootAccess.py
View file

@ -4,7 +4,7 @@
Root access management for system-level operations.
Provides secure access to root user and DbApp database connector.
Bei leerer Datenbank wird automatisch Bootstrap ausgeführt.
Bootstrap is guaranteed by app.py lifespan before any access.
"""
import logging
@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
_rootDbAppConnector = None
_rootUser = None
_bootstrapExecuted = False
def getRootDbAppConnector() -> DatabaseConnector:
"""
@ -39,34 +39,12 @@ def getRootDbAppConnector() -> DatabaseConnector:
return _rootDbAppConnector
def _ensureBootstrap():
"""
Führt Bootstrap aus, falls noch nicht geschehen.
Wird automatisch aufgerufen, wenn getRootUser() keinen User findet.
"""
global _bootstrapExecuted
if _bootstrapExecuted:
return
logger.info("Running bootstrap to initialize database")
# Import here to avoid circular imports
from modules.interfaces.interfaceBootstrap import initBootstrap
dbApp = getRootDbAppConnector()
initBootstrap(dbApp)
_bootstrapExecuted = True
logger.info("Bootstrap completed")
def getRootUser() -> User:
"""
Returns the root user (initial user from database).
Used for system-level operations that require root privileges.
Falls kein User existiert, wird Bootstrap automatisch ausgeführt.
Raises RuntimeError if no user exists (bootstrap incomplete).
"""
global _rootUser
@ -74,19 +52,15 @@ def getRootUser() -> User:
dbApp = getRootDbAppConnector()
initialUserId = dbApp.getInitialId(UserInDB)
# Wenn kein User existiert, Bootstrap ausführen
if not initialUserId:
logger.info("No initial user found, running bootstrap")
_ensureBootstrap()
# Nochmal versuchen nach Bootstrap
initialUserId = dbApp.getInitialId(UserInDB)
if not initialUserId:
raise ValueError("No initial user ID found in database after bootstrap")
raise RuntimeError(
"No root user found - bootstrap incomplete. "
"Ensure app.py lifespan runs initBootstrap before any service access."
)
users = dbApp.getRecordset(UserInDB, recordFilter={"id": initialUserId})
if not users:
raise ValueError("Initial user not found in database")
raise RuntimeError("Initial user not found in database")
user_data = users[0]
_rootUser = User(**user_data)

224
modules/serviceCenter/core/serviceStreaming/eventManager.py
View file

@ -1,222 +1,8 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Event manager for SSE streaming.
Manages event queues for Server-Sent Events (SSE) streaming across features.
"""
"""Re-export shim — canonical source is modules.shared.eventManager."""
import logging
import asyncio
from typing import Dict, Optional, Any
logger = logging.getLogger(__name__)
class EventManager:
"""
Manages event queues for SSE streaming.
Each workflow has its own async queue for events.
"""
def __init__(self):
"""Initialize the event manager."""
self._queues: Dict[str, asyncio.Queue] = {}
self._cleanup_tasks: Dict[str, asyncio.Task] = {}
self._agent_tasks: Dict[str, asyncio.Task] = {}
self._cancelled: Dict[str, bool] = {}
def create_queue(self, workflow_id: str) -> asyncio.Queue:
"""
Create an event queue for a workflow.
Args:
workflow_id: Workflow ID
Returns:
Async queue for events
"""
if workflow_id in self._cleanup_tasks:
self._cleanup_tasks[workflow_id].cancel()
del self._cleanup_tasks[workflow_id]
logger.debug(f"Cancelled pending cleanup for workflow {workflow_id}")
if workflow_id not in self._queues:
self._queues[workflow_id] = asyncio.Queue()
logger.debug(f"Created event queue for workflow {workflow_id}")
else:
old = self._queues[workflow_id]
while not old.empty():
try:
old.get_nowait()
except asyncio.QueueEmpty:
break
logger.debug(f"Reusing event queue for workflow {workflow_id} (drained stale events)")
return self._queues[workflow_id]
def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]:
"""
Get the event queue for a workflow.
Args:
workflow_id: Workflow ID
Returns:
Async queue if exists, None otherwise
"""
return self._queues.get(workflow_id)
def has_queue(self, workflow_id: str) -> bool:
"""
Check if a queue exists for a workflow.
Args:
workflow_id: Workflow ID
Returns:
True if queue exists, False otherwise
"""
return workflow_id in self._queues
def register_agent_task(self, workflow_id: str, task: asyncio.Task) -> None:
"""Register the asyncio Task running the agent for a workflow."""
self._agent_tasks[workflow_id] = task
self._cancelled.pop(workflow_id, None)
def is_cancelled(self, workflow_id: str) -> bool:
"""Check if a workflow has been cancelled."""
return self._cancelled.get(workflow_id, False)
async def cancel_agent(self, workflow_id: str) -> bool:
"""Cancel the running agent task for a workflow. Returns True if cancelled."""
self._cancelled[workflow_id] = True
task = self._agent_tasks.pop(workflow_id, None)
if task and not task.done():
task.cancel()
logger.info(f"Cancelled agent task for workflow {workflow_id}")
return True
logger.debug(f"No running agent task found for workflow {workflow_id}")
return False
def _unregister_agent_task(self, workflow_id: str) -> None:
"""Remove the agent task reference after completion."""
self._agent_tasks.pop(workflow_id, None)
self._cancelled.pop(workflow_id, None)
async def emit_event(
self,
context_id: str,
event_type: str,
data: Dict[str, Any],
event_category: str = "chat",
message: Optional[str] = None,
step: Optional[str] = None
) -> None:
"""
Emit an event to the queue for a workflow.
Args:
context_id: Workflow ID (context)
event_type: Type of event (e.g., "chatdata", "complete", "error")
data: Event data dictionary
event_category: Category of event (e.g., "chat", "workflow")
message: Optional message string
step: Optional step identifier
"""
queue = self._queues.get(context_id)
if not queue:
# DEBUG level: This is normal for background workflows without active SSE listener
return
event = {
"type": event_type,
"data": data,
"category": event_category,
"message": message,
"step": step
}
try:
await queue.put(event)
if event_type not in ("chunk",):
logger.debug(f"Emitted {event_type} event for workflow {context_id}")
except Exception as e:
logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True)
async def cleanup(self, workflow_id: str, delay: float = 60.0) -> None:
"""
Schedule cleanup of a queue after a delay.
Args:
workflow_id: Workflow ID
delay: Delay in seconds before cleanup
"""
# Cancel existing cleanup task if any
if workflow_id in self._cleanup_tasks:
self._cleanup_tasks[workflow_id].cancel()
async def _cleanup():
try:
await asyncio.sleep(delay)
if workflow_id in self._queues:
# Drain remaining events
queue = self._queues[workflow_id]
while not queue.empty():
try:
queue.get_nowait()
except asyncio.QueueEmpty:
break
# Remove queue
del self._queues[workflow_id]
logger.info(f"Cleaned up event queue for workflow {workflow_id}")
except asyncio.CancelledError:
logger.debug(f"Cleanup cancelled for workflow {workflow_id}")
except Exception as e:
logger.error(f"Error during cleanup for workflow {workflow_id}: {e}", exc_info=True)
finally:
if workflow_id in self._cleanup_tasks:
del self._cleanup_tasks[workflow_id]
# Schedule cleanup
task = asyncio.create_task(_cleanup())
self._cleanup_tasks[workflow_id] = task
def shutdown(self) -> None:
"""Cancel all pending cleanup and agent tasks for fast process exit.
Injects ``None`` sentinels into every live queue so that SSE generators
(which block on ``queue.get()``) break out of their loop immediately
instead of waiting up to the keepalive timeout.
"""
for _wfId, q in list(self._queues.items()):
try:
q.put_nowait(None)
except Exception:
pass
for wfId, task in list(self._cleanup_tasks.items()):
if not task.done():
task.cancel()
self._cleanup_tasks.clear()
for wfId, task in list(self._agent_tasks.items()):
if not task.done():
task.cancel()
self._agent_tasks.clear()
self._queues.clear()
logger.info("EventManager shutdown: all tasks cancelled, queues drained")
# Global event manager instance
_event_manager: Optional[EventManager] = None
def get_event_manager() -> EventManager:
"""
Get the global event manager instance.
Returns:
EventManager instance
"""
global _event_manager
if _event_manager is None:
_event_manager = EventManager()
return _event_manager
from modules.shared.eventManager import ( # noqa: F401
EventManager,
get_event_manager,
)

189
modules/serviceCenter/serviceHub.py Normal file
View file

@ -0,0 +1,189 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Service Hub.
Consumer-facing aggregation layer for services, DB interfaces, and runtime state.
Architecture:
- serviceHub delegates service resolution to serviceCenter (DI container)
- serviceHub owns DB interface initialization and runtime state
- serviceCenter knows nothing about serviceHub (one-way dependency)
Import-Regelwerk:
- Zentrale Module (wie dieses) duerfen KEINE Feature-Container importieren
- Feature-spezifische Services werden dynamisch geladen
- Shared Services werden via serviceCenter resolved
"""
import os
import importlib
import glob
from typing import Any, Optional, TYPE_CHECKING
import logging
from modules.datamodels.datamodelUam import User
if TYPE_CHECKING:
from modules.datamodels.datamodelChat import ChatWorkflow
logger = logging.getLogger(__name__)
_FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features")
class PublicService:
"""Lightweight proxy exposing only public callable attributes of a target."""
def __init__(self, target: Any, functionsOnly: bool = True, nameFilter=None):
self._target = target
self._functionsOnly = functionsOnly
self._nameFilter = nameFilter
def __getattr__(self, name: str):
if name.startswith('_'):
raise AttributeError(f"'{type(self._target).__name__}' attribute '{name}' is private")
if self._nameFilter and not self._nameFilter(name):
raise AttributeError(f"'{name}' not exposed by policy")
attr = getattr(self._target, name)
if self._functionsOnly and not callable(attr):
raise AttributeError(f"'{name}' is not a function")
return attr
def __dir__(self):
return sorted([
n for n in dir(self._target)
if not n.startswith('_')
and (not self._functionsOnly or callable(getattr(self._target, n, None)))
and (self._nameFilter(n) if self._nameFilter else True)
])
class ServiceHub:
"""
Consumer-facing aggregation of services, DB interfaces, and runtime state.
Services are lazy-resolved via serviceCenter on first access.
DB interfaces and runtime state are initialized eagerly.
Feature services/interfaces are discovered dynamically from features/.
"""
_SERVICE_CENTER_WRAPPING = {
"ai": {"functionsOnly": False},
}
def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None):
self.user: User = user
self.workflow = workflow
self.mandateId: Optional[str] = mandateId
self.featureInstanceId: Optional[str] = featureInstanceId
self.currentUserPrompt: str = ""
self.rawUserPrompt: str = ""
from modules.serviceCenter.context import ServiceCenterContext
self._serviceCenterContext = ServiceCenterContext(
user=user,
workflow=workflow,
mandate_id=mandateId,
feature_instance_id=featureInstanceId,
)
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
self.interfaceDbApp = getAppInterface(user, mandateId=mandateId)
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId)
self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
self.interfaceDbChat = getChatInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId)
self._loadFeatureInterfaces()
self._loadFeatureServices()
def __getattr__(self, name: str):
"""Lazy-resolve services via serviceCenter on first access."""
if name.startswith('_'):
raise AttributeError(name)
try:
from modules.serviceCenter import getService
service = getService(name, self._serviceCenterContext)
wrapping = self._SERVICE_CENTER_WRAPPING.get(name, {})
functionsOnly = wrapping.get("functionsOnly", True)
wrapped = PublicService(service, functionsOnly=functionsOnly)
setattr(self, name, wrapped)
return wrapped
except KeyError:
raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'")
def _loadFeatureInterfaces(self):
"""Dynamically load interfaces from feature containers by filename pattern."""
pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py")
for filepath in glob.glob(pattern):
try:
featureDir = os.path.basename(os.path.dirname(filepath))
filename = os.path.basename(filepath)[:-3]
modulePath = f"modules.features.{featureDir}.{filename}"
module = importlib.import_module(modulePath)
if hasattr(module, "getInterface"):
interface = module.getInterface(self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId)
attrName = filename.replace("interfaceFeature", "interfaceDb")
setattr(self, attrName, interface)
logger.debug(f"Loaded interface: {attrName} from {modulePath}")
except Exception as e:
logger.debug(f"Could not load interface from {filepath}: {e}")
def _loadFeatureServices(self):
"""Dynamically load services from feature containers by filename pattern."""
pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py")
for filepath in glob.glob(pattern):
try:
serviceDir = os.path.basename(os.path.dirname(filepath))
featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath)))
filename = os.path.basename(filepath)[:-3]
modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}"
module = importlib.import_module(modulePath)
serviceClass = None
for attrName in dir(module):
if attrName.endswith("Service") and not attrName.startswith("_"):
cls = getattr(module, attrName)
if isinstance(cls, type):
serviceClass = cls
break
if serviceClass:
attrName = serviceDir.replace("service", "").lower()
if not attrName:
attrName = serviceDir.lower()
functionsOnly = attrName != "ai"
def _makeServiceResolver(hub):
def _resolver(depKey: str):
return getattr(hub, depKey)
return _resolver
import inspect
sig = inspect.signature(serviceClass.__init__)
paramCount = len([p for p in sig.parameters if p != 'self'])
if paramCount >= 2:
serviceInstance = serviceClass(self, _makeServiceResolver(self))
else:
serviceInstance = serviceClass(self)
setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly))
logger.debug(f"Loaded service: {attrName} from {modulePath}")
except Exception as e:
logger.debug(f"Could not load service from {filepath}: {e}")
# Backward-compatible alias
Services = ServiceHub
def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> ServiceHub:
"""Get ServiceHub instance for the given user, mandate, and feature instance context."""
return ServiceHub(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId)

2
modules/serviceCenter/services/serviceBilling/mainServiceBilling.py
View file

@ -524,7 +524,7 @@ class ProviderNotAllowedException(Exception):
super().__init__(self.message)
from modules.shared.serviceExceptions import BillingContextError
from modules.datamodels.serviceExceptions import BillingContextError
# Expose exception classes on BillingService so consumers can use service.InsufficientBalanceException
# instead of importing from this module

58
modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py
View file

@ -76,63 +76,7 @@ def enhancePlainTextWithMarkdownTables(body: str) -> str:
return "\n\n".join(out_parts)
def parseInlineRuns(text: str) -> list:
"""
Parse inline markdown formatting into a list of InlineRun dicts.
Handles: images, links, bold, italic, inline code, plain text.
Uses a regex-based tokenizer that processes tokens left-to-right.
"""
if not text:
return [{"type": "text", "value": ""}]
# Pattern order matters: images before links, bold before italic
_TOKEN_RE = re.compile(
r'!\[(?P<imgAlt>[^\]]*)\]\((?P<imgSrc>[^)"]+)(?:\s+"(?P<imgWidth>\d+)pt")?\)' # image
r'|\[(?P<linkText>[^\]]+)\]\((?P<linkHref>[^)]+)\)' # link
r'|`(?P<code>[^`]+)`' # inline code
r'|\*\*(?P<bold>.+?)\*\*' # bold
r'|(?<!\w)\*(?P<italic1>.+?)\*(?!\w)' # italic *x*
r'|(?<!\w)_(?P<italic2>.+?)_(?!\w)' # italic _x_
)
runs = []
lastEnd = 0
for m in _TOKEN_RE.finditer(text):
# Plain text before this match
if m.start() > lastEnd:
runs.append({"type": "text", "value": text[lastEnd:m.start()]})
if m.group("imgAlt") is not None or m.group("imgSrc") is not None:
alt = (m.group("imgAlt") or "").strip() or "Image"
src = (m.group("imgSrc") or "").strip()
widthStr = m.group("imgWidth")
run = {"type": "image", "value": alt}
if src.startswith("file:"):
run["fileId"] = src[5:]
else:
run["href"] = src
if widthStr:
run["widthPt"] = int(widthStr)
runs.append(run)
elif m.group("linkText") is not None:
runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")})
elif m.group("code") is not None:
runs.append({"type": "code", "value": m.group("code")})
elif m.group("bold") is not None:
runs.append({"type": "bold", "value": m.group("bold")})
elif m.group("italic1") is not None:
runs.append({"type": "italic", "value": m.group("italic1")})
elif m.group("italic2") is not None:
runs.append({"type": "italic", "value": m.group("italic2")})
lastEnd = m.end()
# Trailing plain text
if lastEnd < len(text):
runs.append({"type": "text", "value": text[lastEnd:]})
return runs if runs else [{"type": "text", "value": text}]
from modules.shared.documentUtils import parseInlineRuns # noqa: F401 — canonical source in shared/
def _imageRefToDict(token: str) -> Optional[Dict[str, Any]]:

2
modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py
View file

@ -1039,7 +1039,7 @@ def _buildCancelSummaryHtml(subRecord: Dict[str, Any], platformUrl: str = "") ->
# Exception Classes (defined in shared, re-exported here for backward compat)
# ============================================================================
from modules.shared.serviceExceptions import (
from modules.datamodels.serviceExceptions import (
SubscriptionInactiveException,
SubscriptionCapacityException,
SUBSCRIPTION_USER_ACTION_UPGRADE,

196
modules/serviceHub/__init__.py
View file

@ -1,189 +1,7 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Service Hub.
Consumer-facing aggregation layer for services, DB interfaces, and runtime state.
Architecture:
- serviceHub delegates service resolution to serviceCenter (DI container)
- serviceHub owns DB interface initialization and runtime state
- serviceCenter knows nothing about serviceHub (one-way dependency)
Import-Regelwerk:
- Zentrale Module (wie dieses) duerfen KEINE Feature-Container importieren
- Feature-spezifische Services werden dynamisch geladen
- Shared Services werden via serviceCenter resolved
"""
import os
import importlib
import glob
from typing import Any, Optional, TYPE_CHECKING
import logging
from modules.datamodels.datamodelUam import User
if TYPE_CHECKING:
from modules.datamodels.datamodelChat import ChatWorkflow
logger = logging.getLogger(__name__)
_FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features")
class PublicService:
"""Lightweight proxy exposing only public callable attributes of a target."""
def __init__(self, target: Any, functionsOnly: bool = True, nameFilter=None):
self._target = target
self._functionsOnly = functionsOnly
self._nameFilter = nameFilter
def __getattr__(self, name: str):
if name.startswith('_'):
raise AttributeError(f"'{type(self._target).__name__}' attribute '{name}' is private")
if self._nameFilter and not self._nameFilter(name):
raise AttributeError(f"'{name}' not exposed by policy")
attr = getattr(self._target, name)
if self._functionsOnly and not callable(attr):
raise AttributeError(f"'{name}' is not a function")
return attr
def __dir__(self):
return sorted([
n for n in dir(self._target)
if not n.startswith('_')
and (not self._functionsOnly or callable(getattr(self._target, n, None)))
and (self._nameFilter(n) if self._nameFilter else True)
])
class ServiceHub:
"""
Consumer-facing aggregation of services, DB interfaces, and runtime state.
Services are lazy-resolved via serviceCenter on first access.
DB interfaces and runtime state are initialized eagerly.
Feature services/interfaces are discovered dynamically from features/.
"""
_SERVICE_CENTER_WRAPPING = {
"ai": {"functionsOnly": False},
}
def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None):
self.user: User = user
self.workflow = workflow
self.mandateId: Optional[str] = mandateId
self.featureInstanceId: Optional[str] = featureInstanceId
self.currentUserPrompt: str = ""
self.rawUserPrompt: str = ""
from modules.serviceCenter.context import ServiceCenterContext
self._serviceCenterContext = ServiceCenterContext(
user=user,
workflow=workflow,
mandate_id=mandateId,
feature_instance_id=featureInstanceId,
)
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
self.interfaceDbApp = getAppInterface(user, mandateId=mandateId)
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId)
self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
self.interfaceDbChat = getChatInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId)
self._loadFeatureInterfaces()
self._loadFeatureServices()
def __getattr__(self, name: str):
"""Lazy-resolve services via serviceCenter on first access."""
if name.startswith('_'):
raise AttributeError(name)
try:
from modules.serviceCenter import getService
service = getService(name, self._serviceCenterContext)
wrapping = self._SERVICE_CENTER_WRAPPING.get(name, {})
functionsOnly = wrapping.get("functionsOnly", True)
wrapped = PublicService(service, functionsOnly=functionsOnly)
setattr(self, name, wrapped)
return wrapped
except KeyError:
raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'")
def _loadFeatureInterfaces(self):
"""Dynamically load interfaces from feature containers by filename pattern."""
pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py")
for filepath in glob.glob(pattern):
try:
featureDir = os.path.basename(os.path.dirname(filepath))
filename = os.path.basename(filepath)[:-3]
modulePath = f"modules.features.{featureDir}.{filename}"
module = importlib.import_module(modulePath)
if hasattr(module, "getInterface"):
interface = module.getInterface(self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId)
attrName = filename.replace("interfaceFeature", "interfaceDb")
setattr(self, attrName, interface)
logger.debug(f"Loaded interface: {attrName} from {modulePath}")
except Exception as e:
logger.debug(f"Could not load interface from {filepath}: {e}")
def _loadFeatureServices(self):
"""Dynamically load services from feature containers by filename pattern."""
pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py")
for filepath in glob.glob(pattern):
try:
serviceDir = os.path.basename(os.path.dirname(filepath))
featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath)))
filename = os.path.basename(filepath)[:-3]
modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}"
module = importlib.import_module(modulePath)
serviceClass = None
for attrName in dir(module):
if attrName.endswith("Service") and not attrName.startswith("_"):
cls = getattr(module, attrName)
if isinstance(cls, type):
serviceClass = cls
break
if serviceClass:
attrName = serviceDir.replace("service", "").lower()
if not attrName:
attrName = serviceDir.lower()
functionsOnly = attrName != "ai"
def _makeServiceResolver(hub):
def _resolver(depKey: str):
return getattr(hub, depKey)
return _resolver
import inspect
sig = inspect.signature(serviceClass.__init__)
paramCount = len([p for p in sig.parameters if p != 'self'])
if paramCount >= 2:
serviceInstance = serviceClass(self, _makeServiceResolver(self))
else:
serviceInstance = serviceClass(self)
setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly))
logger.debug(f"Loaded service: {attrName} from {modulePath}")
except Exception as e:
logger.debug(f"Could not load service from {filepath}: {e}")
# Backward-compatible alias
Services = ServiceHub
def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> ServiceHub:
"""Get ServiceHub instance for the given user, mandate, and feature instance context."""
return ServiceHub(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId)
# Re-export shim — canonical source: modules.serviceCenter.serviceHub
from modules.serviceCenter.serviceHub import ( # noqa: F401
PublicService,
ServiceHub,
Services,
getInterface,
)

64
modules/shared/documentUtils.py Normal file
View file

@ -0,0 +1,64 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Document utility functions (Layer L0 - shared).
Pure text-processing helpers with zero internal dependencies.
"""
import re
def parseInlineRuns(text: str) -> list:
"""
Parse inline markdown formatting into a list of InlineRun dicts.
Handles: images, links, bold, italic, inline code, plain text.
Uses a regex-based tokenizer that processes tokens left-to-right.
"""
if not text:
return [{"type": "text", "value": ""}]
_TOKEN_RE = re.compile(
r'!\[(?P<imgAlt>[^\]]*)\]\((?P<imgSrc>[^)"]+)(?:\s+"(?P<imgWidth>\d+)pt")?\)'
r'|\[(?P<linkText>[^\]]+)\]\((?P<linkHref>[^)]+)\)'
r'|`(?P<code>[^`]+)`'
r'|\*\*(?P<bold>.+?)\*\*'
r'|(?<!\w)\*(?P<italic1>.+?)\*(?!\w)'
r'|(?<!\w)_(?P<italic2>.+?)_(?!\w)'
)
runs = []
lastEnd = 0
for m in _TOKEN_RE.finditer(text):
if m.start() > lastEnd:
runs.append({"type": "text", "value": text[lastEnd:m.start()]})
if m.group("imgAlt") is not None or m.group("imgSrc") is not None:
alt = (m.group("imgAlt") or "").strip() or "Image"
src = (m.group("imgSrc") or "").strip()
widthStr = m.group("imgWidth")
run = {"type": "image", "value": alt}
if src.startswith("file:"):
run["fileId"] = src[5:]
else:
run["href"] = src
if widthStr:
run["widthPt"] = int(widthStr)
runs.append(run)
elif m.group("linkText") is not None:
runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")})
elif m.group("code") is not None:
runs.append({"type": "code", "value": m.group("code")})
elif m.group("bold") is not None:
runs.append({"type": "bold", "value": m.group("bold")})
elif m.group("italic1") is not None:
runs.append({"type": "italic", "value": m.group("italic1")})
elif m.group("italic2") is not None:
runs.append({"type": "italic", "value": m.group("italic2")})
lastEnd = m.end()
if lastEnd < len(text):
runs.append({"type": "text", "value": text[lastEnd:]})
return runs if runs else [{"type": "text", "value": text}]

167
modules/shared/eventManager.py Normal file
View file

@ -0,0 +1,167 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Event manager for SSE streaming (Layer L0 - shared).
Manages event queues for Server-Sent Events (SSE) streaming across features.
Generic pub/sub infrastructure with zero internal dependencies.
"""
import logging
import asyncio
from typing import Dict, Optional, Any
logger = logging.getLogger(__name__)
class EventManager:
"""
Manages event queues for SSE streaming.
Each workflow has its own async queue for events.
"""
def __init__(self):
"""Initialize the event manager."""
self._queues: Dict[str, asyncio.Queue] = {}
self._cleanup_tasks: Dict[str, asyncio.Task] = {}
self._agent_tasks: Dict[str, asyncio.Task] = {}
self._cancelled: Dict[str, bool] = {}
def create_queue(self, workflow_id: str) -> asyncio.Queue:
"""Create an event queue for a workflow."""
if workflow_id in self._cleanup_tasks:
self._cleanup_tasks[workflow_id].cancel()
del self._cleanup_tasks[workflow_id]
logger.debug(f"Cancelled pending cleanup for workflow {workflow_id}")
if workflow_id not in self._queues:
self._queues[workflow_id] = asyncio.Queue()
logger.debug(f"Created event queue for workflow {workflow_id}")
else:
old = self._queues[workflow_id]
while not old.empty():
try:
old.get_nowait()
except asyncio.QueueEmpty:
break
logger.debug(f"Reusing event queue for workflow {workflow_id} (drained stale events)")
return self._queues[workflow_id]
def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]:
"""Get the event queue for a workflow."""
return self._queues.get(workflow_id)
def has_queue(self, workflow_id: str) -> bool:
"""Check if a queue exists for a workflow."""
return workflow_id in self._queues
def register_agent_task(self, workflow_id: str, task: asyncio.Task) -> None:
"""Register the asyncio Task running the agent for a workflow."""
self._agent_tasks[workflow_id] = task
self._cancelled.pop(workflow_id, None)
def is_cancelled(self, workflow_id: str) -> bool:
"""Check if a workflow has been cancelled."""
return self._cancelled.get(workflow_id, False)
async def cancel_agent(self, workflow_id: str) -> bool:
"""Cancel the running agent task for a workflow. Returns True if cancelled."""
self._cancelled[workflow_id] = True
task = self._agent_tasks.pop(workflow_id, None)
if task and not task.done():
task.cancel()
logger.info(f"Cancelled agent task for workflow {workflow_id}")
return True
logger.debug(f"No running agent task found for workflow {workflow_id}")
return False
def _unregister_agent_task(self, workflow_id: str) -> None:
"""Remove the agent task reference after completion."""
self._agent_tasks.pop(workflow_id, None)
self._cancelled.pop(workflow_id, None)
async def emit_event(
self,
context_id: str,
event_type: str,
data: Dict[str, Any],
event_category: str = "chat",
message: Optional[str] = None,
step: Optional[str] = None
) -> None:
"""Emit an event to the queue for a workflow."""
queue = self._queues.get(context_id)
if not queue:
return
event = {
"type": event_type,
"data": data,
"category": event_category,
"message": message,
"step": step
}
try:
await queue.put(event)
if event_type not in ("chunk",):
logger.debug(f"Emitted {event_type} event for workflow {context_id}")
except Exception as e:
logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True)
async def cleanup(self, workflow_id: str, delay: float = 60.0) -> None:
"""Schedule cleanup of a queue after a delay."""
if workflow_id in self._cleanup_tasks:
self._cleanup_tasks[workflow_id].cancel()
async def _cleanup():
try:
await asyncio.sleep(delay)
if workflow_id in self._queues:
queue = self._queues[workflow_id]
while not queue.empty():
try:
queue.get_nowait()
except asyncio.QueueEmpty:
break
del self._queues[workflow_id]
logger.info(f"Cleaned up event queue for workflow {workflow_id}")
except asyncio.CancelledError:
logger.debug(f"Cleanup cancelled for workflow {workflow_id}")
except Exception as e:
logger.error(f"Error during cleanup for workflow {workflow_id}: {e}", exc_info=True)
finally:
if workflow_id in self._cleanup_tasks:
del self._cleanup_tasks[workflow_id]
task = asyncio.create_task(_cleanup())
self._cleanup_tasks[workflow_id] = task
def shutdown(self) -> None:
"""Cancel all pending cleanup and agent tasks for fast process exit."""
for _wfId, q in list(self._queues.items()):
try:
q.put_nowait(None)
except Exception:
pass
for wfId, task in list(self._cleanup_tasks.items()):
if not task.done():
task.cancel()
self._cleanup_tasks.clear()
for wfId, task in list(self._agent_tasks.items()):
if not task.done():
task.cancel()
self._agent_tasks.clear()
self._queues.clear()
logger.info("EventManager shutdown: all tasks cancelled, queues drained")
# Global event manager instance
_event_manager: Optional[EventManager] = None
def get_event_manager() -> EventManager:
"""Get the global event manager instance."""
global _event_manager
if _event_manager is None:
_event_manager = EventManager()
return _event_manager

59
modules/shared/featureDiscovery.py Normal file
View file

@ -0,0 +1,59 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Feature discovery utility (Layer L0 - shared).
Dynamically discovers and loads feature main modules from the features directory.
Zero internal dependencies — only os, glob, importlib, logging.
"""
import os
import glob
import importlib
import logging
from typing import Dict, Any
logger = logging.getLogger(__name__)
FEATURES_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "features"
)
_cachedMainModules = None
def loadFeatureMainModules() -> Dict[str, Any]:
"""
Dynamically load main modules from all discovered feature containers.
Results are cached after the first call.
"""
global _cachedMainModules
if _cachedMainModules is not None:
return _cachedMainModules
mainModules = {}
pattern = os.path.join(FEATURES_DIR, "*", "main*.py")
for filepath in glob.glob(pattern):
filename = os.path.basename(filepath)
if filename == "__init__.py":
continue
featureDir = os.path.basename(os.path.dirname(filepath))
if featureDir.startswith("_"):
continue
if featureDir in mainModules:
continue
mainFile = filename[:-3]
try:
modulePath = f"modules.features.{featureDir}.{mainFile}"
module = importlib.import_module(modulePath)
mainModules[featureDir] = module
logger.debug(f"Loaded main module: {featureDir}")
except Exception as e:
logger.error(f"Failed to load main module from {featureDir}: {e}")
_cachedMainModules = mainModules
return mainModules

46
modules/system/gdprDeletion.py
View file

@ -561,39 +561,27 @@ def _deleteUserDataFromFeatureDatabases(userId: str, currentUser) -> Dict[str, A
logger.info(f"Found {len(featureCodes)} feature types to process: {featureCodes}")
# Process each feature type
# Process each feature type via lifecycle hooks
from modules.shared.featureDiscovery import loadFeatureMainModules
featureModules = loadFeatureMainModules()
for featureCode in featureCodes:
try:
dbName = f"poweron_{featureCode}"
# Try to get feature interface
featureInterface = None
if featureCode == "trustee":
from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface
featureInterface = getTrusteeInterface(currentUser)
elif featureCode == "realestate":
from modules.features.realestate.interfaceFeatureRealEstate import getInterface as getRealEstateInterface
featureInterface = getRealEstateInterface(currentUser)
elif featureCode == "neutralization":
from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutralizerInterface
featureInterface = getNeutralizerInterface(currentUser)
else:
logger.warning(f"No interface found for feature code: {featureCode}")
featureModule = featureModules.get(featureCode)
hook = getattr(featureModule, "onUserDelete", None) if featureModule else None
if hook is None:
logger.warning(f"No onUserDelete hook for feature: {featureCode}")
continue
if featureInterface and hasattr(featureInterface, 'db'):
featureStats = deleteUserDataFromDatabase(
featureInterface.db,
userId,
dbName
)
featureStats = hook(userId, currentUser)
if featureStats:
stats["databases"].append(featureStats)
stats["totalTablesProcessed"] += featureStats["tablesProcessed"]
stats["totalRecordsDeleted"] += featureStats["recordsDeleted"]
stats["totalRecordsAnonymized"] += featureStats["recordsAnonymized"]
stats["errors"].extend(featureStats["errors"])
stats["totalTablesProcessed"] += featureStats.get("tablesProcessed", 0)
stats["totalRecordsDeleted"] += featureStats.get("recordsDeleted", 0)
stats["totalRecordsAnonymized"] += featureStats.get("recordsAnonymized", 0)
stats["errors"].extend(featureStats.get("errors", []))
except Exception as featureErr:
errorMsg = f"Error processing feature {featureCode}: {featureErr}"
logger.warning(errorMsg)

85
modules/system/i18nBootSync.py
View file

@ -202,19 +202,19 @@ def _registerRbacLabels():
logger.info("i18n rbac labels: %d new keys (rbac.* context)", added)
def _registerServiceCenterLabels():
"""Register service-center category labels and bootstrap role descriptions."""
def _registerServiceCenterLabels(serviceLabels: list = None):
"""Register service-center category labels and bootstrap role descriptions.
serviceLabels is injected by app.py (Composition Root) to avoid
system(L4) → serviceCenter(L5) upward import.
"""
added = 0
try:
from modules.serviceCenter.registry import IMPORTABLE_SERVICES
for svc in IMPORTABLE_SERVICES.values():
key = _extractRegistrySourceText(svc.get("label"))
if key and key not in _REGISTRY:
_REGISTRY[key] = _I18nRegistryEntry(context="service", value="")
added += 1
except ImportError:
pass
for label in (serviceLabels or []):
key = _extractRegistrySourceText(label)
if key and key not in _REGISTRY:
_REGISTRY[key] = _I18nRegistryEntry(context="service", value="")
added += 1
_bootstrapRoleDescriptions = [
"Administrator - Benutzer und Ressourcen im Mandanten verwalten",
@ -288,38 +288,28 @@ def _registerNodeLabels():
logger.info("i18n node labels: %d new keys (node.*/port.* context)", added)
def _registerAccountingConnectorLabels():
"""Register all accounting connector configField labels at boot time."""
def _registerAccountingConnectorLabels(accountingLabels: list = None):
"""Register accounting connector configField labels at boot time.
Args:
accountingLabels: List of dicts with keys 'label' and 'connectorType',
injected from app.py to avoid features-import.
"""
if not accountingLabels:
return
added = 0
try:
from modules.features.trustee.accounting.accountingRegistry import getAccountingRegistry
except ImportError:
logger.debug("i18n accounting connectors: registry not importable")
return
try:
registry = getAccountingRegistry()
except Exception as e:
logger.warning("i18n accounting connectors: registry init failed: %s", e)
return
for connectorType, connector in (registry._connectors or {}).items():
try:
for field in connector.getRequiredConfigFields():
key = getattr(field, "label", "") or ""
if not isinstance(key, str) or not key:
continue
if key not in _REGISTRY:
_REGISTRY[key] = _I18nRegistryEntry(
context=f"connector.accounting.{connectorType}",
value="",
)
added += 1
except Exception as e:
logger.warning(
"i18n accounting connector %s: failed to read fields: %s",
connectorType, e,
for entry in accountingLabels:
key = entry.get("label", "")
connectorType = entry.get("connectorType", "unknown")
if not isinstance(key, str) or not key:
continue
if key not in _REGISTRY:
_REGISTRY[key] = _I18nRegistryEntry(
context=f"connector.accounting.{connectorType}",
value="",
)
added += 1
logger.info("i18n accounting connector labels: %d new keys", added)
@ -385,16 +375,21 @@ def _registerDatamodelOptionLabels():
# Public boot API (called by app.py)
# ---------------------------------------------------------------------------
async def syncRegistryToDb():
"""Boot hook: discover all i18n keys and write them into UiLanguageSet(xx)."""
async def syncRegistryToDb(serviceLabels: list = None, accountingLabels: list = None):
"""Boot hook: discover all i18n keys and write them into UiLanguageSet(xx).
Args:
serviceLabels: Service label strings injected from app.py (avoids upward import).
accountingLabels: Accounting connector field labels injected from app.py.
"""
_scanRouteApiMsgKeys()
_registerNavLabels()
_registerFeatureUiLabels()
_registerRbacLabels()
_registerServiceCenterLabels()
_registerServiceCenterLabels(serviceLabels)
_registerNodeLabels()
_registerDatamodelOptionLabels()
_registerAccountingConnectorLabels()
_registerAccountingConnectorLabels(accountingLabels)
if not _REGISTRY:
logger.info("i18n registry: no keys to sync (empty registry)")

346
modules/system/mainSystem.py
View file

@ -20,351 +20,7 @@ FEATURE_CODE = "system"
FEATURE_LABEL = "System"
FEATURE_ICON = "mdi-cog"
# =============================================================================
# Navigation Structure (Single Source of Truth)
# =============================================================================
#
# Block Order (gemäss Navigation-API-Konzept):
# - System: 10
# - <dynamic/features>: 15 (wird in routeSystem.py eingefügt)
# - Basisdaten: 30
# - Administration: 200
#
# NOTE: Workflows and Migrate sections removed - now handled as features
#
# Item Order: Default-Abstand 10 pro Item
# uiComponent: Abgeleitet von objectKey (ui.system.home -> page.system.home)
# icon: Wird intern gehalten aber NICHT in der API Response zurückgegeben
NAVIGATION_SECTIONS = [
# ─── Meine Sicht (with top-level item + subgroups) ───
{
"id": "system",
"title": t("Meine Sicht"),
"order": 10,
"items": [
{
"id": "home",
"objectKey": "ui.system.home",
"label": t("Start"),
"icon": "FaHome",
"path": "/",
"order": 10,
"public": True,
},
],
"subgroups": [
# ── Übersichten ──
{
"id": "system-overviews",
"title": t("Übersichten"),
"order": 15,
"items": [
{
"id": "integrations",
"objectKey": "ui.system.integrations",
"label": t("Integrationen"),
"icon": "FaProjectDiagram",
"path": "/integrations",
"order": 10,
"public": True,
},
{
"id": "compliance-audit",
"objectKey": "ui.system.complianceAudit",
"label": t("Compliance & Audit"),
"icon": "FaShieldAlt",
"path": "/compliance-audit",
"order": 20,
},
],
},
# ── Basisdaten ──
{
"id": "system-basedata",
"title": t("Basisdaten"),
"order": 20,
"items": [
{
"id": "connections",
"objectKey": "ui.system.connections",
"label": t("Verbindungen"),
"icon": "FaLink",
"path": "/basedata/connections",
"order": 10,
"public": True,
},
{
"id": "files",
"objectKey": "ui.system.files",
"label": t("Dateien"),
"icon": "FaRegFileAlt",
"path": "/basedata/files",
"order": 20,
"public": True,
},
{
"id": "prompts",
"objectKey": "ui.system.prompts",
"label": t("Prompts"),
"icon": "FaLightbulb",
"path": "/basedata/prompts",
"order": 30,
"public": True,
},
],
},
# ── Nutzung ──
{
"id": "system-usage",
"title": t("Nutzung"),
"order": 30,
"items": [
{
"id": "billing-admin",
"objectKey": "ui.system.billingAdmin",
"label": t("Abrechnung"),
"icon": "FaMoneyBillAlt",
"path": "/billing/admin",
"order": 10,
},
{
"id": "statistics",
"objectKey": "ui.system.statistics",
"label": t("Statistiken"),
"icon": "FaChartBar",
"path": "/billing/transactions",
"order": 20,
},
{
"id": "automations",
"objectKey": "ui.system.automations",
"label": t("Automations"),
"icon": "FaRobot",
"path": "/automations",
"order": 30,
},
{
"id": "rag-inventory",
"objectKey": "ui.system.ragInventory",
"label": t("RAG-Inventar"),
"icon": "FaDatabase",
"path": "/rag-inventory",
"order": 35,
},
{
"id": "store",
"objectKey": "ui.system.store",
"label": t("Store"),
"icon": "FaStore",
"path": "/store",
"order": 40,
"public": True,
},
{
"id": "settings",
"objectKey": "ui.system.settings",
"label": t("Einstellungen"),
"icon": "FaCog",
"path": "/settings",
"order": 50,
"public": True,
},
],
},
],
},
# ─── Administration (with subgroups) ───
{
"id": "admin",
"title": t("Administration"),
"order": 200,
"subgroups": [
# ── Wizards ──
{
"id": "admin-wizards",
"title": t("Wizards"),
"order": 10,
"items": [
{
"id": "admin-mandate-wizard",
"objectKey": "ui.admin.mandateWizard",
"label": t("Mandanten-Wizard"),
"icon": "FaMagic",
"path": "/admin/mandate-wizard",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-invitation-wizard",
"objectKey": "ui.admin.invitationWizard",
"label": t("Einladungs-Wizard"),
"icon": "FaEnvelopeOpenText",
"path": "/admin/invitation-wizard",
"order": 20,
"adminOnly": True,
},
],
},
# ── Users ──
{
"id": "admin-users-group",
"title": t("Benutzer"),
"order": 20,
"items": [
{
"id": "admin-users",
"objectKey": "ui.admin.users",
"label": t("Benutzer"),
"icon": "FaUsers",
"path": "/admin/users",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-invitations",
"objectKey": "ui.admin.invitations",
"label": t("Benutzer-Einladungen"),
"icon": "FaEnvelopeOpenText",
"path": "/admin/invitations",
"order": 20,
"adminOnly": True,
},
{
"id": "admin-user-access-overview",
"objectKey": "ui.admin.userAccessOverview",
"label": t("Benutzer-Zugriffsübersicht"),
"icon": "FaClipboardList",
"path": "/admin/user-access-overview",
"order": 30,
"adminOnly": True,
},
{
"id": "admin-subscriptions",
"objectKey": "ui.admin.subscriptions",
"label": t("Abonnements"),
"icon": "FaFileContract",
"path": "/admin/subscriptions",
"order": 40,
"adminOnly": True,
},
],
},
# ── System ──
{
"id": "admin-system-group",
"title": t("System"),
"order": 30,
"items": [
{
"id": "admin-roles",
"objectKey": "ui.admin.roles",
"label": t("Rollen"),
"icon": "FaUserTag",
"path": "/admin/mandate-roles",
"order": 10,
"adminOnly": True,
},
{
"id": "admin-mandate-role-permissions",
"objectKey": "ui.admin.mandateRolePermissions",
"label": t("Rollen-Berechtigungen"),
"icon": "FaKey",
"path": "/admin/mandate-role-permissions",
"order": 20,
"adminOnly": True,
},
{
"id": "admin-mandates",
"objectKey": "ui.admin.mandates",
"label": t("Mandanten"),
"icon": "FaBuilding",
"path": "/admin/mandates",
"order": 30,
"adminOnly": True,
},
{
"id": "admin-user-mandates",
"objectKey": "ui.admin.userMandates",
"label": t("Mandanten-Mitglieder"),
"icon": "FaUserFriends",
"path": "/admin/user-mandates",
"order": 40,
"adminOnly": True,
},
{
"id": "admin-access",
"objectKey": "ui.admin.access",
"label": t("Zugriffsverwaltung"),
"icon": "FaBuilding",
"path": "/admin/access",
"order": 50,
"adminOnly": True,
},
{
"id": "admin-feature-instances",
"objectKey": "ui.admin.featureInstances",
"label": t("Feature-Instanzen"),
"icon": "FaCubes",
"path": "/admin/feature-instances",
"order": 60,
"adminOnly": True,
},
{
"id": "admin-feature-roles",
"objectKey": "ui.admin.featureRoles",
"label": t("Features Rollen-Vorlagen"),
"icon": "FaShieldAlt",
"path": "/admin/feature-roles",
"order": 70,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-logs",
"objectKey": "ui.admin.logs",
"label": t("Logs"),
"icon": "FaFileAlt",
"path": "/admin/logs",
"order": 90,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-languages",
"objectKey": "ui.admin.languages",
"label": t("UI-Sprachen"),
"icon": "FaGlobe",
"path": "/admin/languages",
"order": 95,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-database-health",
"objectKey": "ui.admin.databaseHealth",
"label": t("Datenbank-Gesundheit"),
"icon": "FaDatabase",
"path": "/admin/database-health",
"order": 98,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-demo-config",
"objectKey": "ui.admin.demoConfig",
"label": t("Demo Config"),
"icon": "FaCubes",
"path": "/admin/demo-config",
"order": 100,
"adminOnly": True,
"sysAdminOnly": True,
},
],
},
],
},
]
from modules.datamodels.datamodelNavigation import NAVIGATION_SECTIONS # noqa: F401 — canonical source
def objectKeyToUiComponent(objectKey: str) -> str:

52
modules/system/registry.py
View file

@ -89,45 +89,7 @@ def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]:
return results
_cachedMainModules = None
def loadFeatureMainModules() -> Dict[str, Any]:
"""
Dynamically load main modules from all discovered feature containers.
Results are cached after the first call.
"""
global _cachedMainModules
if _cachedMainModules is not None:
return _cachedMainModules
mainModules = {}
pattern = os.path.join(FEATURES_DIR, "*", "main*.py")
for filepath in glob.glob(pattern):
filename = os.path.basename(filepath)
if filename == "__init__.py":
continue
featureDir = os.path.basename(os.path.dirname(filepath))
if featureDir.startswith("_"):
continue
# Skip if this feature already has a main module loaded (avoid duplicates)
if featureDir in mainModules:
continue
mainFile = filename[:-3] # Remove .py
try:
modulePath = f"modules.features.{featureDir}.{mainFile}"
module = importlib.import_module(modulePath)
mainModules[featureDir] = module
logger.debug(f"Loaded main module: {featureDir}")
except Exception as e:
logger.error(f"Failed to load main module from {featureDir}: {e}")
_cachedMainModules = mainModules
return mainModules
from modules.shared.featureDiscovery import loadFeatureMainModules # noqa: F401 — re-export
def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]:
@ -150,16 +112,8 @@ def registerAllFeaturesInCatalog(catalogService) -> Dict[str, bool]:
logger.error(f"Error registering system RBAC objects: {e}")
results["system"] = False
# Register service center RBAC objects (service.web, service.extraction, etc.)
try:
from modules.serviceCenter import registerServiceObjects
success = registerServiceObjects(catalogService)
results["servicecenter"] = success
except ImportError as e:
logger.warning(f"Service center not found, skipping service RBAC registration: {e}")
except Exception as e:
logger.error(f"Error registering service RBAC objects: {e}")
results["servicecenter"] = False
# Service center RBAC objects are registered by app.py (Composition Root)
# to avoid system(L4) → serviceCenter(L5) upward import.
# Register feature modules
mainModules = loadFeatureMainModules()

4
modules/workflows/automation2/executionEngine.py
View file

@ -31,7 +31,7 @@ from modules.workflows.automation2.executors import (
)
from modules.features.graphicalEditor.portTypes import normalizeToSchema, wrapTransit, unwrapTransit
from modules.features.graphicalEditor.nodeDefinitions import STATIC_NODE_TYPES
from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.workflows.automation2.graphicalEditorRunFileLogger import (
GraphicalEditorRunFileLogger,
graphical_editor_run_file_logging_enabled,
@ -252,7 +252,7 @@ def _merge_node_parameters_into_snap(
def _emitStepEvent(runId: str, stepData: Dict[str, Any]) -> None:
"""Emit a step-log SSE event to any listening client for this run."""
try:
from modules.serviceCenter.core.serviceStreaming.eventManager import get_event_manager
from modules.shared.eventManager import get_event_manager
em = get_event_manager()
queueId = f"run-trace-{runId}"
if not em.has_queue(queueId):

2
modules/workflows/automation2/executors/actionNodeExecutor.py
View file

@ -19,7 +19,7 @@ from modules.features.graphicalEditor.portTypes import (
_normalizeError,
normalizeToSchema,
)
from modules.shared.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.workflows.automation2.executors.inputExecutor import PauseForHumanTaskError
from modules.workflows.methods.methodContext.actions.extractContent import (
PRESENTATION_KIND,

2
modules/workflows/methods/methodAi/actions/consolidate.py
View file

@ -7,7 +7,7 @@ from typing import Any, Dict, List
from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, OperationTypeEnum
from modules.datamodels.datamodelChat import ActionResult
from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)

2
modules/workflows/methods/methodAi/actions/generateCode.py
View file

@ -9,7 +9,7 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData
from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)

2
modules/workflows/methods/methodAi/actions/generateDocument.py
View file

@ -9,7 +9,7 @@ from modules.datamodels.datamodelChat import ActionResult, ActionDocument
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
from modules.datamodels.datamodelWorkflow import AiResponse, DocumentData
from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)

2
modules/workflows/methods/methodAi/actions/process.py
View file

@ -10,7 +10,7 @@ from typing import Dict, Any, List, Optional
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum
from modules.datamodels.datamodelExtraction import ContentPart
from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)

5
modules/workflows/methods/methodAi/actions/webResearch.py
View file

@ -7,8 +7,7 @@ import re
import json
from typing import Dict, Any
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
from modules.serviceCenter import ServiceCenterContext, getService, can_access_service
from modules.shared.serviceExceptions import SubscriptionInactiveException, BillingContextError
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)
@ -45,6 +44,8 @@ def _build_research_prompt(parameters: Dict[str, Any]) -> str:
async def webResearch(self, parameters: Dict[str, Any]) -> ActionResult:
from modules.serviceCenter import ServiceCenterContext, getService, can_access_service
operationId = None
try:
prompt = _build_research_prompt(parameters)

4
modules/workflows/methods/methodContext/actions/extractContent.py
View file

@ -1399,7 +1399,7 @@ def _load_image_bytes_by_file_id(services: Any, file_id: str) -> Optional[bytes]
def _inline_runs_from_presentation_lines(lines: List[Any]) -> List[Dict[str, Any]]:
"""Map presentation ``lines`` to inline runs, preserving line order with explicit breaks."""
from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import parseInlineRuns
from modules.shared.documentUtils import parseInlineRuns
runs: List[Dict[str, Any]] = []
first = True
@ -1537,7 +1537,7 @@ def presentation_envelopes_to_document_json(
services: Any = None,
) -> Dict[str, Any]:
"""Map presentation envelope(s) to ``renderReport`` ``extractedContent`` (documents/sections)."""
from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import parseInlineRuns
from modules.shared.documentUtils import parseInlineRuns
envelopes = normalize_presentation_envelopes(raw)
if not envelopes:

103
modules/workflows/processing/shared/methodDiscovery.py
View file

@ -34,57 +34,78 @@ def _collectActionsUnfiltered(methodInstance) -> Dict[str, Dict[str, Any]]:
return result
def _registerMethodClasses(serviceCenter, modulePath: str, uniqueCount: int) -> int:
"""Import a method module and register all MethodBase subclasses found in it."""
try:
module = importlib.import_module(modulePath)
except Exception as e:
logger.error(f"Error importing method module {modulePath}: {e}")
return uniqueCount
for itemName, item in inspect.getmembers(module):
if (inspect.isclass(item) and
issubclass(item, MethodBase) and
item != MethodBase):
if itemName in methods:
continue
shortName = itemName.replace('Method', '').lower()
methodInstance = item(serviceCenter)
actions = _collectActionsUnfiltered(methodInstance)
methodInfo = {
'instance': methodInstance,
'actions': actions,
'description': item.__doc__ or f"Method {itemName}"
}
methods[itemName] = methodInfo
methods[shortName] = methodInfo
uniqueCount += 1
logger.info(f"Discovered method {itemName} (short: {shortName}) with {len(actions)} actions")
return uniqueCount
def discoverMethods(serviceCenter):
"""Dynamically discover all method classes and their actions in modules methods package.
Always creates fresh method instances bound to the given serviceCenter,
preventing stale or cross-workflow service references.
"""Dynamically discover all method classes and their actions.
Scans two locations:
1. modules.workflows.methods (core methods)
2. modules.features.*/workflows/ (feature-owned methods)
"""
global methods
try:
methodsPackage = importlib.import_module('modules.workflows.methods')
# Clear and rebuild to prevent cross-workflow state contamination
methods.clear()
uniqueCount = 0
for _, name, isPkg in pkgutil.iter_modules(methodsPackage.__path__):
if name.startswith('method'):
try:
module = importlib.import_module(f'modules.workflows.methods.{name}')
for itemName, item in inspect.getmembers(module):
if (inspect.isclass(item) and
issubclass(item, MethodBase) and
item != MethodBase):
shortName = itemName.replace('Method', '').lower()
# Skip if already processed (via another module path)
if itemName in methods:
continue
methodInstance = item(serviceCenter)
actions = _collectActionsUnfiltered(methodInstance)
methodInfo = {
'instance': methodInstance,
'actions': actions,
'description': item.__doc__ or f"Method {itemName}"
}
methods[itemName] = methodInfo
methods[shortName] = methodInfo
uniqueCount += 1
logger.info(f"Discovered method {itemName} (short: {shortName}) with {len(actions)} actions")
except Exception as e:
logger.error(f"Error discovering method {name}: {str(e)}")
continue
uniqueCount = _registerMethodClasses(
serviceCenter, f'modules.workflows.methods.{name}', uniqueCount
)
# Feature-owned methods (e.g. features/trustee/workflows/methodTrustee)
import os
import glob as _glob
featuresDir = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
"features"
)
for wfInit in _glob.glob(os.path.join(featuresDir, "*", "workflows", "__init__.py")):
wfDir = os.path.dirname(wfInit)
featureName = os.path.basename(os.path.dirname(wfDir))
for entry in os.listdir(wfDir):
entryPath = os.path.join(wfDir, entry)
if os.path.isdir(entryPath) and entry.startswith("method"):
modulePath = f"modules.features.{featureName}.workflows.{entry}"
uniqueCount = _registerMethodClasses(serviceCenter, modulePath, uniqueCount)
logger.info(f"Discovered {uniqueCount} unique methods ({len(methods)} entries with aliases)")
except Exception as e:
logger.error(f"Error discovering methods: {str(e)}")

55
modules/workflows/scheduler/mainScheduler.py
View file

@ -396,48 +396,29 @@ def _createRunFailedNotification(
logger.warning("Failed to create in-app run.failed notification: %s", e)
_onRunFailedCallback = None
def setOnRunFailedCallback(callback) -> None:
"""Set the callback for run failure notifications (injected by app.py)."""
global _onRunFailedCallback
_onRunFailedCallback = callback
def _triggerRunFailedSubscription(
workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None
) -> None:
"""Trigger the messaging subscription for run failures (email notifications)."""
"""Trigger the messaging subscription for run failures via injected callback."""
if _onRunFailedCallback is None:
return
try:
from modules.serviceCenter import getService
from modules.serviceCenter.context import ServiceCenterContext
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelMessaging import MessagingEventParameters
rootInterface = getRootInterface()
if not rootInterface:
return
eventUser = rootInterface.getUserByUsername("event")
if not eventUser:
return
ctx = ServiceCenterContext(
user=eventUser,
mandate_id=mandateId or "",
feature_instance_id="",
feature_code="graphicalEditor",
_onRunFailedCallback(
workflowId=workflowId,
runId=runId,
error=error,
mandateId=mandateId,
workflowLabel=workflowLabel,
)
messagingService = getService("messaging", ctx)
subscriptionId = "GraphicalEditorRunFailed"
eventParams = MessagingEventParameters(triggerData={
"workflowId": workflowId,
"workflowLabel": workflowLabel or workflowId,
"runId": runId,
"error": error,
"mandateId": mandateId or "",
})
result = messagingService.executeSubscription(subscriptionId, eventParams)
logger.info(
"Triggered run.failed subscription: sent=%d success=%s",
result.messagesSent, result.success,
)
except FileNotFoundError:
logger.debug("Subscription function GraphicalEditorRunFailed not found (not yet registered)")
except ValueError as e:
logger.debug("Subscription GraphicalEditorRunFailed: %s", e)
except Exception as e:
logger.warning("Failed to trigger run.failed subscription: %s", e)