PowerOn/platform-core
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'int'
Some checks failed
Deploy Plattform-Core / test (push) Failing after 50s
Details
Deploy Plattform-Core / deploy (push) Has been skipped
Details

Browse source
This commit is contained in:
Ida 2026-06-12 10:23:04 +02:00
commit eae5819c44
76 changed files with 3011 additions and 995 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
app.py
View file

@ -21,7 +21,7 @@ from datetime import datetime
from modules.shared.configuration import APP_CONFIG
from modules.shared.eventManagement import eventManager
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.system.registry import loadFeatureMainModules
from modules.system.registry import loadFeatureMainModules, registerAllFeaturesInCatalog, syncCatalogFeaturesToDb
class DailyRotatingFileHandler(RotatingFileHandler):
"""
@ -176,6 +176,20 @@ def initLogging():
pass
return True
# Suppress h11 LocalProtocolError ("Can't send data when our state is ERROR")
# from uvicorn when a client disconnects mid-response (browser abort, HMR, navigation).
# The asyncio event-loop handler (below) only catches event-loop-level exceptions;
# uvicorn logs this via the standard logging module before it reaches the event loop.
class ClientDisconnectFilter(logging.Filter):
def filter(self, record):
if record.exc_info:
excType = record.exc_info[0]
if excType and getattr(excType, "__name__", "") == "LocalProtocolError":
return False
if isinstance(record.msg, str) and "LocalProtocolError" in record.msg:
return False
return True
# Add filter to normalize problematic unicode (e.g., arrows) to ASCII for terminals like cp1252
class UnicodeArrowFilter(logging.Filter):
def filter(self, record):
@ -204,6 +218,7 @@ def initLogging():
consoleHandler.addFilter(ChromeDevToolsFilter())
consoleHandler.addFilter(HttpcoreStarFilter())
consoleHandler.addFilter(HTTPDebugFilter())
consoleHandler.addFilter(ClientDisconnectFilter())
consoleHandler.addFilter(EmojiFilter())
consoleHandler.addFilter(UnicodeArrowFilter())
handlers.append(consoleHandler)
@ -227,6 +242,7 @@ def initLogging():
fileHandler.addFilter(ChromeDevToolsFilter())
fileHandler.addFilter(HttpcoreStarFilter())
fileHandler.addFilter(HTTPDebugFilter())
fileHandler.addFilter(ClientDisconnectFilter())
fileHandler.addFilter(EmojiFilter())
fileHandler.addFilter(UnicodeArrowFilter())
handlers.append(fileHandler)
@ -255,6 +271,12 @@ def initLogging():
for loggerName in noisyLoggers:
logging.getLogger(loggerName).setLevel(logging.WARNING)
# Apply ClientDisconnectFilter to uvicorn's own logger so the
# h11 LocalProtocolError is suppressed regardless of handler setup.
_disconnectFilter = ClientDisconnectFilter()
for _uvName in ("uvicorn.error", "uvicorn"):
logging.getLogger(_uvName).addFilter(_disconnectFilter)
# Log the current logging configuration
logger = logging.getLogger(__name__)
logger.info(f"Logging initialized with level {logLevelName}")
@ -347,10 +369,17 @@ async def lifespan(app: FastAPI):
except Exception as e:
logger.warning(f"Bootstrap check failed (may already be initialized): {str(e)}")
# Migrate vector column dimensions (idempotent — safe on every startup)
try:
from modules.interfaces.interfaceDbKnowledge import migrateVectorDimensions
migrateVectorDimensions()
logger.info("Vector dimension migration check completed")
except Exception as e:
logger.warning(f"Vector dimension migration failed (non-critical): {e}")
# Register all feature definitions in RBAC catalog (for /api/features/ endpoint)
try:
from modules.security.rbacCatalog import getCatalogService
from modules.system.registry import registerAllFeaturesInCatalog, syncCatalogFeaturesToDb
catalogService = getCatalogService()
registerAllFeaturesInCatalog(catalogService)
logger.info("Feature catalog registration completed")
@ -464,7 +493,6 @@ async def lifespan(app: FastAPI):
def _onRunFailed(workflowId, runId, error, mandateId=None, workflowLabel=None):
from modules.serviceCenter import getService
from modules.serviceCenter.context import ServiceCenterContext
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelMessaging import MessagingEventParameters
rootInterface = getRootInterface()
@ -525,6 +553,10 @@ async def lifespan(app: FastAPI):
from modules.serviceCenter.services.serviceSubscription.enterpriseRenewalScheduler import registerEnterpriseRenewalScheduler
registerEnterpriseRenewalScheduler()
# Register token and trusted device cleanup scheduler
from modules.auth.trustedDeviceService import registerTokenCleanupScheduler
registerTokenCleanupScheduler()
# Recover background jobs that were RUNNING when the previous worker died
try:
from modules.serviceCenter.services.serviceBackgroundJobs.mainBackgroundJobService import (
@ -871,6 +903,10 @@ app.include_router(demoConfigRouter)
from modules.routes.routeAdminDatabaseHealth import router as adminDatabaseHealthRouter
app.include_router(adminDatabaseHealthRouter)
from modules.routes.routeAdminSessions import router as adminSessionsRouter, trustedDeviceRouter as adminTrustedDeviceRouter
app.include_router(adminSessionsRouter)
app.include_router(adminTrustedDeviceRouter)
from modules.routes.routeGdpr import router as gdprRouter
app.include_router(gdprRouter)

4
env-dev.env
View file

@ -58,7 +58,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = http://localhost:8000/api/clickup/auth/conn
# Stripe Billing (both end with _SECRET for encryption script)
STRIPE_SECRET_KEY_SECRET = DEV_ENC:Z0FBQUFBQnB5dkd5aHNGejgzQmpTdmprdzQxR19KZkh3MlhYUTNseFN3WnlaWjh2SDZyalN6aU9xSktkbUQwUnZrVnlvbGVRQm4yZFdiRU5aSEk5WVJuUnR4VUwtTm9OVk1WWmJQeU5QaDdib0hfVWV5U1BfYTFXRmdoOWdnOWxkb3JFQmF3bm45UjFUVUxmWGtGRkFKUGd6bmhpQlFnaVI3Q2lLdDlsY1VESk1vOEM0ZFBJNW1qcVZ0N2tPYmRLNmVKajZ2M3o3S05lWnRRVG5LdkRseW4wQ3VjNHNQZTZUdz09
STRIPE_WEBHOOK_SECRET = DEV_ENC:Z0FBQUFBQnB5dkd5dDJMSHBrVk8wTzJhU2xzTTZCZWdvWmU2NGI2WklfRXRJZVUzaVYyOU9GLUZsalUwa2lPdEgtUHo0dVVvRDU1cy1saHJyU0Rxa2xQZjBuakExQzk3bmxBcU9WbEIxUEtpR1JoUFMxZG9ISGRZUXFhdFpSMGxvQUV3a0VLQllfUUtCOHZwTGdteV9rYTFOazBfSlN3ekNWblFpakJlZVlCTmNkWWQ4Sm01a1RCWTlnTlFHWVA0MkZYMlprUExrWFN2V0NVU1BTd1NKczFJbVo3VHpLdlc4UT09
STRIPE_WEBHOOK_SECRET = DEV_ENC:Z0FBQUFBQnFLeUFlb2dfSjZPaWIyRjZsNjhiSDFQNFpxdW50YmlLUjFLX1lJMGdCWUtBUEdrRGhvSzVVWnkxNVZEdmtkQmk5X05YS0JVU1NyX3VQZTV2VjVwakd0RGM2WUl6TTlzbms1d1NCOTQtdURiVjhxdXZGVlR1ZVNTbUkwOFh1R04yUUxxay0=
STRIPE_API_VERSION = 2026-01-28.clover
STRIPE_AUTOMATIC_TAX_ENABLED = false
STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQd14OUoIL0Osj7A0ZQlr0
@ -71,7 +71,7 @@ Connector_AiTavily_API_SECRET = DEV_ENC:Z0FBQUFBQnFCdlFlY1R2WGpuazk5M05SeDIyLWd3
Connector_AiPrivateLlm_API_SECRET = DEV_ENC:Z0FBQUFBQnBudkpGRHM5eFdUVmVZU1R1cHBwN1RlMUx4T0NlLTJLUFFVX3J2OElDWFpuZmJHVmp4Z3BNNWMwZUVVZUd2TFhRSjVmVkVlcFlVRWtybXh0ZHloZ01ZcnVvX195YjdlWVdEcjZSWFFTTlNBWUlaTlNoLWhqVFBIb0thVlBiaWhjYjFQOFY=
Connector_AiMistral_API_SECRET = DEV_ENC:Z0FBQUFBQnFCdlFlelh2T2hqNGcxV0hMV1FKbmFDZjVHUWF6T2FXbGlCSnQzSzNXLWJHeXBFWE1nUlh1b1NHY1JRSEVtTVEtc1MtUnZrX2ZCcURqQ2FYNmFWa2xudGJtS3g2eVo4MFZMd09nZTBNMmo1ZHU0bzBJdFRqLVhHSVZNb2Zrc0VkUXI0SVk=
Service_MSFT_TENANT_ID = common
Service_MSFT_TENANT_ID = organizations
# Google Cloud Speech Services configuration
Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQnFIc3YtU0x4LTlHbTY1NUVGY2V2bUdmck85dDh1ZWVKa2ktR0N6NjdlTGFrUHMybVQ2bVRLN01XNFRZR2lyN0ZNSHhzWVVGNnVtZjRjV2hhR0ViTDYwT25lSmxJY0pSTkl3OUEyT0JxMFVYRndfUFJudExMajdTYUNXS01JU2lhQzZmNWFYdXA4aVZ5Zkh4Zko1Z00tcEE5ZFEwQkFVa1oyR296YXozRFI2WUdXN0ZSREFFclFNaTd6OUVlSmFxS1BTSlNJbnlWNHNfbkk4QzVOUGlkMzdfQUZxUlJOVEZzUlN1aWRWY01JZmlRM0JNZE1EZ3BmbW10c3BDdERpa2FMakstQUlqVEVlRC1hUmZoeFVoQ3pYNXRlRFVSTlI3ekJrU0QwSHBSaWxiSGU0akFGMXUtY2Q0RnUzS0tPOEQtcTdVdWhQeHFDM1hRRVVMcUxCeklvWHNWRUN2bjVHZUUwLTVtaGpUbWdPUnJabWlIcHZ5UjNtN0NMTUNRN29ZRGVXU28xQmhJTVg2eEZnaUdrcW9UVklHMHJycm1nT0JkdGJReVVHeV8tYm12UDlOU0lpNHFidXBQbUFSSVVmWUl1M1BVMFFncm0xSldkVzBrb2poRFMyaVUwcUZvMHl0QlZIZ1h1MjZwR3AtZWhqdzN4UVhtT2hUa1lQU3VudzNXdW1FcVY3VnQ3RmpkQnFQemlrQlF3WGhBNWxOZXJ6Zm9KVFlEZExUXzlqODhYaFNNMzVWTzFNMmVTcWdodDZoRmZTUzlhLVlOSU5fYW1vNXctaFpFMC1pUllRZW11d1JQN25sbldHVjI1anc2UC1ycndjTGtxWk55WmpJeU1wOVR0RnlTdFpad1dkRmlUNDE0d240TDlKc3JFUXdOYzd5UTFYSXUzLTQ2Y1ZGcWE3R2RyQ0I1WDMtMHBScEFzZDV4UEkyanh4ckJZUjdTYnJGZjAxQkU3MEJ6OXdybGRaWHNod1hZZEhVOXRpMWRLbVJsRGd0UDRDN3JsRzF4T0RpcnczRU5TM0RKVjVkWTRqNTl6bmhQdmdvaEg1U2kya0QtQ0l4ZHVUcGxkNi1vNVVVOEcyWXhxZWc5N1lKMk4tT0o3ZFVzYjJtT3NVZFJiSTFNUnpaSmFOeDZaLWVpZlc0VUhZRHdXOUMyQ3cwaXBQUDRJN1g1YkwzaTFiRVRxRFY5UTdZU1dSaGR6NUw3aEtac2RENXF3WEpVN0dXVTlQR0F6MFlpWl83MU44NVR1ZUtPVUNlZ205YUIwOFoxUDBvTlI0SU52emVvQ3VZXy1jTlFXRWZXQ0d5RHJ0eV9JeE5wMHl0b3FVSjNoVzg2d21hYVNYY3Q0dkFaVEZwa09tRnFBbEtoOUlGY2xkeVJoZGYzQUxYNFZfb0ZiaU5VRjJPbGhieXYtWTFKckZwenVCUGFva1IwVVFORVQ4SDMxWHVuRWhBRGd0cVlsc3kyQ0RyY2ZIVDlwcGh5ampySV9uOVpsVmlWbGoxMEg3SXh6NzRJbmZXRlhMMWc0RXhzeWtnQlJ0VnZSdENkbEpOdENwUzItUjZhZWFYRFhzbDM1WDBxaGFPX19CSG1KZjRTTU5JemcxZzJRSFY5bkx4TTlIZFNHOW1USWxBYWhEZ1FSNVdSSDJETUZwMi1Hd0RESkF2cVA1TVJGTEtPUl9oN3gzVEIwSzZOVzlOWXhNa2I1Vzc1SV9tdENfRy1rQTNzRlZGSTYwQmJIaGswZUNWSnRDVXFfdWFCckZZcnJOT2Rfb3FrcWI4S1lVRTMyRnZJQTRZV1VsU0xobGRjekhtbG9LamR2d1hfVklsM3JBeW9SRzJnWVdiWDRzN1ltcXdSVGoxRVBvczViVXNjMUxBazZUdS1WbkRQX0h1MzdNd3ltVDUzd2FGdi1XeUMybV9ia1YxQVBPdnUxY1dfT2M5eEpZR2JHMkdZbWdDZTRERXRYOWxodndkTXltVW40c0t0bVA5YWxuRzM3LWlCdmJiYmF5dkNBY3ozbUw1Zm5zRmpBdk5ORmFZRWJKM3Q2UDdKNl9zaUV5eVVGbkF0QmZSZzk5dGo3UjNIQWxwcjRlVTdUT2s1VGFjdndvX2c3d1VmaHRMZU10M1ZKVk9Ma3dZb1kwYVV5Z2NlTjUxdUYtZXRnRTRzQlp1aFp0OUF5TVBwN1gzU21kRmJ6OUlOeUFOOEhEOU5WSENNZndvLXdoVUFJYVFDTWEyakJEcTVSVDhJOWJscU8taThqNUZkdThCOUlXcldndFBTZk9QVnlMaUphUU5sUktpb1plZDZOQnFzNFNMUzRWbWFVQWhUWmJfem96X0cxWXVTcUxCeDhOc3E2OEpFa2lzWHFIV0p3eGdBZmN1aXBhYjExZTZqaUY4S0ZudTNhcUx2WlpuTU9lNUk2ZmNyN0JCODdYMGNEU2JsZkZXYlRFaTJQUTI5RU5SMmtkV1NHQTVTTjEyZGZLYnhTNTg2Nl9aaWJqX2Q1U1NwQ3pRTGRBSUw0N3FNQ0ItMks1QVZmbURYVWdHMWFZTWhGNURVOUg0bGVuMUozanlxTnRwbVlGX2RnN2FBVTZlZjhDaXVzZEtVR1Z5azhzWHRrS1dYSG9rYkowTjQ1N0hyRWdNVWMya1ZmWmZvSnVTdHNiMHFDODNLckpjQ081SFlieGxuM0picGhKMnNQRURwY2hpQzF3dHRnNEFWcUlPYjVxZEhod0JDbWZhU01Ob21UWmRwd0NQRlpjOE5CUFBOT004U2JKNkFSUlFzRklYZGJobUoxQzZzT2wzZ3J1Z05aYThRVVNzcFktMGJDcXFfSkxVS2hhajI3dTdrR2poa21ZM3Z4UzFRblFsOFlOZVVUM0YxaFRuNjFWQ2E4ZlhvZjZpMWFtOGRuaGx0MTZxZE9TY1dsTTMyMHhsNXJ2MkduaGRkZXpYUWJ3cEt1U3YwMC1IRzM5eWRCb0lvaUhTQ2R4XzhEZl9zRk5GeHhCSWx2X3BkUkJ4NFZLVzdVRFZkbnpNNkpjUTFHY1pDV0ZOMFBaNTVpLUlmSnFrX1N5X05MTjRUeTVERUs5MG9kMFJ3di03U3BpMUM4YXNwaG1fangwYURIVjBpSVdCUkt4UW5HbWtGOUh3TUdPZjMxYXpVZDcwTmlDcTR6WldZb3VzbHRpRUgyN2lFTjlpUV85T0M4blJxMWx0cC1iU0FDOHhueDBLYjdLZGhNbjFPbE1RdmhhNlEzX3ZpT2ZsYllwNkU5TE9fZWFabDE4RWRoRWxiMk5aVFZrWmxjaW5MX1VrUGhUN29vbU1tWldESnczYTNBQ1RPd1VTNGNJdjdJU3p3QXZQLVlDNkQ1cTh4Rk1WNnRMUi1DT3VGREFPa28xejc2NUl1dzJSa2hCTlJublBRNGkydlJVRjlFbFotOWtraWFqQkNNTXBpT1hZM0NXNEpObGMxQUNuS29rOExMSnMxT3NLbjNfLTdpQW1BcDMxR1RZdVRvbElGbENWbHJqRlVrTXhYbFdiMmItUzlxR2ZxT2FCWXpMVVJYZXBfSFVwNTczU3JHUVhET3hSWm80Ry1KcE9mV3FYejVHSEVSS0pxOUtCc3V2VHNFVkRqYk5Od20tM0ttdFQ1eGdsc091WGFYNFgybzNVd3ZvbzEwUDJ0T0hvTVd3YnlHNnpNWC0wbkJOQTIwQ3VYdlUzaXY5NFhDNlNOOW9UdGZNUk4zZ0VJakpwS21SZlJtQjVWLUxfejFYZFc1cjRwR3ZUOGdZb2VJaTdJUS1MYlRJb0ZFYW9uYzM3MDd4b09BR1pnTEh3RFpnaGhxZURQamllNUhqTHg0cHJfN08wMkdGSVQwQUlqWDhLVGViY3J5NlVFTzY3RGhGQ0R6aXNsb2w4dnBVYndTd1Jhd3IwS1BxY0h1X05RcGsySzVNbXR5YlBVQi1IOGFUNkh5QjhRZk5BQmZvcGF6ZTNXenZkdy1GRjFGdE1saGdMSnotUkIyX1VqTlZFWnJER1YyNGQtMFZHU3hmRVNPUWFCdXV3QUxzOGVSbF9EdEZGUFNxbTdiYm5oWHdYak5qa3Zoem5WY1ZUdDREVUxGX0VQeS1jckhqS2lRLXQ1Y2tyOFRjYnVhajNUZmZOUE9kbU9PYXdqdk5DYUtEOVFiMW9yZTYxMFNUaDdvUTExUFZ1bklYSkRKTnJ1RURvOTR3ODREcWdWeHpRS2RETjZqeXpvbUpxMW5lWl84RzVocmJFQ3JfZlpMd3RCZEo5RWZ0MzIxNWV6bHlwdWJJWXhoaWxlM2FHSjBhWG14Sk94ZV96cXFvU1JwWDdKZldmZWdvdWVKdXVfaS1jZjdENXQzSzNyb1d3eWhUMU53QzgxemRiTTlkdFRxZU1OdEN5c1kxOEd2MTJMcnBJWEE0eXdJdFpOYVNMQTNLR292UFlGb0Ztdz0=

4
env-int.env
View file

@ -60,7 +60,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = https://api-int.poweron.swiss/api/clickup/a
# Stripe Billing (both end with _SECRET for encryption script)
STRIPE_SECRET_KEY_SECRET = INT_ENC:Z0FBQUFBQnB5dkd5ekdBaGNGVUlOQUpncTlzLWlTV0V5OWZzQkpDczhCUGw4U1JpTHZ0d3pfYlFNWElLRlNiNlNsaDRYTGZUTkg2OUFrTW1GZXpOUjBVbmRQWjN6ekhHd2ZSQ195OHlaeWh1TmxrUm10V2R3YmdncmFLbFMzVjdqcWJMSUJPR2xuSEozclNoZG1rZVBTaWg3OFQ1Qzdxb0wyQ2RKazc2dG1aZXBUTXlvbDZqLS1KOVI5M3BGc3NQZkZRbnFpRjIwWmh2ZHlVNlpxZVo2dWNmMjQ5eW02QmtzUT09
STRIPE_WEBHOOK_SECRET = INT_ENC:Z0FBQUFBQnA4UXZiUUVqTl9lREVRWTh1aHFDcFpwcXRkOUx4MS1ham9Ddkl6T0xzMnJuM1hhUHdGNG5CenY1MUg4RlJBOGFQTWl5cVd5MjJ2REItcHYyRmdLX3ZlT2p5Z3BRVkMtQnRoTVkteXlfaU92MVBtOEI0Ni1kbGlfa0NiRmFRRXNHLVE2NHI=
STRIPE_WEBHOOK_SECRET = INT_ENC:Z0FBQUFBQnFLeUFWUUtMZ25NQ2ZOWE5nRF9CaFNwcXhSU2tKRktLaElLRHJMM295OXNkVEFLekVUMzN0YUpIZHJfWGNqa0xxOFZRVHZEUXVLZ3ItVGZWc2VFQ2thcUlJalY1b0JDSmR6RF96d1A3OGhyd0w1MHZPeFNZRkl0c19kYUJQcHVwR2tsd0s=
STRIPE_API_VERSION = 2026-01-28.clover
STRIPE_AUTOMATIC_TAX_ENABLED = false
STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQd14OUoIL0Osj7A0ZQlr0
@ -73,7 +73,7 @@ Connector_AiTavily_API_SECRET = INT_ENC:Z0FBQUFBQnFCdlFlbmRSZVRjTzVKRklFbFgwdVZJ
Connector_AiPrivateLlm_API_SECRET = INT_ENC:Z0FBQUFBQnBudkpGSjZ1NWh0aWc1R3Z4MHNaeS1HamtUbndhcUZFZDlqUDhjSmg5eHFfdlVkU0RsVkJ2UVRaMWs3aWhraG5jSlc0YkxNWHVmR2JoSW5ENFFCdkJBM0VienlKSnhzNnBKbTJOUTFKczRfWlQ3bWpmUkRTT1I1OGNUSTlQdExacGRpeXg=
Connector_AiMistral_API_SECRET = INT_ENC:Z0FBQUFBQnFCdlFlU2tMLTFnQWhET2Nia2pTcVpBakRaSVFDdUpHRzZ1bkhGVVhMeEVlSnFZU3F3UFRBUkNMMU4tQU92OUdTeDlpM2VZbXJzLURQZ1lPLVB3azgxSDZabkhkSHJ5Y005aWhtcDJzajk3a2JDQUxCZlNKRGw5elJuSzJMUUpTZ2hiSlU=
Service_MSFT_TENANT_ID = common
Service_MSFT_TENANT_ID = organizations
# Google Cloud Speech Services configuration
Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQnFIc3YtSjhlcklrU2JCOW5mdHFHd0dLTUZZZk9PT3o5RWt5RjAxX2s3ekJRLUUzU0dNSnNseTE4bUpNTnZSTWg0QV9mWm5iX19aWjV4YnRXU1JBSm1INVB5dXNRT2JiYk1tLWRSS29pdTRMdS1lMDZxMkx4VTh3bU5aVWh3cEwyOE1QcXVockgtZWh5bzdNVXQyemFuSmZqRzZZYmNGN21JdjNwNWpPRXB6WU1qSU5rZUVSb3JBS0lhcThvakkwbTRUUHhBdjRZdWNsZ1Z1RmFaNGZLcEpaNVNLdFAxYzFXdTJydU9COWJ0bkNyYUF2X2FNc1BfT05teEs1SE9PeGhPd3VJSFY2VFJ5VEl6V3R3bzd6OTVKTEVRcmt5ZzdBMXBFY1A5dUFJRFJONFBlaDlJcjNBQnBraC0wMTBhNW8wYWZaeHNWclVTOVotLTdWSmVuYzJKcUZSUkdrdXB3VEVESzd4UTI0bGd6SzdCajdoazZXVTVCaGRiaWJaOHg5Z2thSWItcS05U25DbUdrT2M1QV81WEg2dlJfMlBtZU9Bc3V5bmtBWHRoRUVLR2lWNHY3M3hHcU1raFRFOWQwSEtUU1RDWDFRNFlkNHVnTkZDbk5zS3RZeGR2Z015RnRGc3NndFVEQjc4bVpNeE81bXc1MnQ2QjNZeHZCbUJJZVJ2TE5xWEd4M3hHT2hJWW5DOWMxQlNmZE9uMVRGVnRwTUlXZjZCRUZBLU9GWVZGWFpZbUE3WVlpZU1DX1Z0bWQ0bjlaRThHOE9WR3VOVzlYWS1JampTNmxkNmFxWG54WDJjallIT3UyT0tGSzJpeG1tX0JoQjZxbEpESHBhMWZFa205bjdvTVFwSVVidnVzdURZVDAzVVpkekJ2SVZTZmhxQVJ2OWpuRGR2WFE3elMtb3B2ZzhpQVNvRmkzbzRrY1BuamVzM0E2eVM0bXBHTHgtYmhsVG5jNlB1Q1JHZU9HUlNfaTJSQkcwS2FSZnZSOW9oZzdXa1RUVTVTZTgwY01GYXQyQ0xWX1Fnb0xaOTRQY3hTclgweVJ5clc5OVpRWWlDb0JQVXoxVDA0bW8zUE55aGowb1ZZNEpBN2UtSTZTY2llRGhISFFkYWFYVlVBQ0IzbGxzVTQ2V2dsUGV1Y2I5bEZLRnlwdXRHMWZVcnBaTXNzNzNkUVFqR2xnSEQ1VlpTdXpwMFVVYjQ0enFlUnk0d3dDQUtSS1dUVnNyYnBKQW9TRjJxN2JNY2NhRWNONWRpWU5RbzNNZVJBS3EzN2ZMZ1E5VXQtMDFTZklLY1JiSDNYRlFuOF9VYUktS0xoY2IyR0xkT19qTEpIV1p6RFExUWNCQTdqN1kyS0Jaa2lyMDluenc1MS1vdmhPVlE5OUphWEY2dXFYNE04Z3lBUG5DNGZjTUVnYzEzYWhzTHpMdVBzT0dzRGJaT2x5b0pVbWJtUzJxdEd2VGtrc01kTlNPNURoVHhwZzU1d3pTZGJiTUZIME5tQ0xqNWJ2QS1QSEJHV2FEOExHWDByV19rVnc2R2pibnNENEo1cTh4bGNMX2ZpSTBMcjRvQWRhbW5xYVBiZkZzWTRERlVESEU2aHpvdzNMTjlCazRYeEJhMmZwdXY5T25IYkFTaUM3SmdIV1FCX2xxRXctWHZQOHgxLXI1c1JkWmcydkFTUmxFSU03cGtnallnTXplOElQbEJRSEE2aW5KREU0YUxwX25wOFhuS2RIbms1dXNIRHBtNjFtb3B3UGVGb0hwOENKM1hMclBwa3NBa2pFYnZYbEtFbUF0Y3pmeFRmMDNMaTZrR1BZWnBrNUQ1WlU1NVZQSWUxN3dwcXhhcjdXNTl4LVVpYVF3Y0wtRmFyNXZRNTE3UUc2cHVaVVNpaVdHbXRqQVJNZWZmNjdQQ2lwTGd6RFFZN2tSY2NEdmxvaXk4MTZMcmg0VGo3MTN2R2V6cmV3YjdQVlNEZTQySUpaY2pkTHZzUzdJLVJ2WnlOQ3Vmem5FZXRaWjBMWjF4ZEF3ZHJ4VF8tMVNsRnljejVsaEpGOU5JbnhydjNVdzNMOENrWUVsbXp0ZEhuVE1Vd0RJcnp2N0RXUGFuNDM2OXBPbV9LRDUwTWk1NHYwaDhlVEhKUmtEa09INURwNjV5ZE1VWmpRSGdjeXJNc3FqcjZDdmx5WXluNWZ2VlpsWmR2TXVXVnBubEFmQlRfaGRwRndCVXVkMjkyLWVhaDQtZDN1cmFZLUoybGRwbGQ5MTExU2NnZ2lueVNfSjFDQ2NkWGtNX2M1T2I4YnVJOUFueGIxbG1EYlZOcFYtQlE3cm90SE40X0ZjalhLdXM5S2l5aW84ZUJPMlR4MU9EVkhZcHdrX1Zqc0NhWEJacDZHMzQwSzdkdi1Rd2s4Y1dfLS1ES0NfYTNxYl84UTN1S0lIM0pVTTNEYlJ0YW55Tk4yVjBONXNTQWtVZTJ2V3B5eHBJcG9IWGRMMklob0hMbVVZZzJKbTFMUExOQm5HSEZzWHU0VGVIWlJMVzFLeFB0NkkyWFkwWk0wdjdHRmxSWFFoSkJ2Vm5NUWNQQlp6YWlIc2NKLUdhOVVycHd5N3NFMDNVWlAxZGQ1NzRGbm9LcWxEb2tKR1RnVEtvRUc1d3l4aU1IOUQ5RldUT3Z0a3lpRHpVSWJ4MjU4RWY5MEpCQ0VFdHNMbnkxOGswcE44QzJwNXFCVGpIa0VGc2VNXy1qdzVNRU9DaXg2MW9VX3FjUk41QVFVLURwVGFLRTkyNWlENy1IcGZjNW9wY0Y5Q3d5eFg5emVUUF9hV3ZTQWNaNEN0VzdJRlFBR0picXJoUERacWNLbDZhTE8wdWlfZ3kxd2QzOXBOZV9uaUNGMkNJbGhNd3k0S2t3dTRGWVVxTTFRRlg3Ui1zLW1FLU1Mai1yaURjb2Fob2c4MDUyRHN5aldUVWMxLTVNbm5VQTdrYy0zLVFyOHRkNzZ3dGdhbXZXN3JHNkdfZ2RuRXFDM3R2TVB1cDNOdWZGTmpFNnNFTmMxTmFuZDdJUld5bERyQkJ0TGZXRk54NEdqN09hSmVMYV91NXUwNXFvMl9KV0hBNlB4bklNQ2U5WGZLUTdlX2dJenVGcDYwWHBsdTNpbE5mWGhWeXFuUkFPV0puR2h0RkhrR2MwTzJGUmp4bUR6UFlUWTlNbTJLa19hTUZZR0dscVpBbFBReTBRMDNseXo4SXNnZWt4VFdpOERqLV9ZczRkR0QwRFJQM0pqdHluWktDUlp6WU9XSjVNZi1tYnNzcVlGTDRFMzNlSmRTazFfTkNxSjAwM0wxNk9Sd2h1SWpfOW5MVWMtVXYyYlVZR0VuaHRpN1pnNnpHME5raVBMd2h2dDRyMV8yZGFJNnlkcmhtSWdmNlpLN19NcjNkc002dXFxQzhTaDZzRlgzNUJ1SzVpVnp6NVU1Y2luUlM4UEJoajNTOUJadnE1MlhzV0kxSzBObXkteVhNM3RKYW9heDVWWFJ1NGlDM0l0elRPbThwUU9oYkVkbC1PZFNLSHY3WHJiZWpEamNIVC00MlNNWV9qcHdjNDRjRlVhZXlrLTlicVBNaDlDeXdRb0Fwc3RmUGFvbURQZ29yckliaS1VUDNxcXVlYTJJRUhXNUVobk1KUDhHZE16UzBLeDViYVRwZWY3d2w0d253eEZYcExKRGpsaGlBUElaTzB3eUVadnROX1dabENGb3R4ZF9aS05KY0dHTVZaYzRFc1Z4TlZGbFd2NjdYRzJMTzVwU2NaN1Y3MzQ2Z2pzV2RSMzJBbjg0MEhaZmhoREloY0oxOFdjNDZNdVZfYlRKU1Q1M2hYdHgwUjVsTV9USjZCZXlQTTdNRWc3bUxOcXRDVkpTdnJxR0hkWWpaRUdrOEFyNHk4MENwVzdob0hUSkJvam4zZW1kcGxZUjg0RXFRNnBxSUg1MDVHdHRwVlFkWWhHM0ZyZVFvMF96R2V5YjBuMnVZTU5CQ3pVci16SGJlQTQtbnFLa1E2eHFncUg3UmYyYlZvOF82a3d2ZE4tbmxIUlNYYjlrck9QYk5CcV9faXludS1yem1JNjFBdVYyb21RQWFMMFkxX0s1TjQ4czZ2WXI3X0FzRWdNTlZndHl4bnVOTHl2YlZfaURQV053dHl4N1czRFdzaVFnRHB0MWRDV2ZuU2lzX1NZZkRQYzhsT3ItZWw0dVJlVmtFWUM5cEppOGxuYVdpQkN5dV9hQ2dodTJvV3REVkw2dVVDaGtvc0Zqd0V2dldLZEVNRVRRNVRUVmw5aHZmZEpHdk1wS0xwRFc5Vmx4dTdfdGZDRUtCU29qdEVIOW5VdjBmeGpFMFZHSUthamtVN1E2bDZqaEFackVSQnZMN0tyaUhIcUs1ZHMzMzl2TnhadGIwZW5QNS1BM3pSODY3WVFsLU1jeUpCMG1PWmhPVT0=

4
env-prod.env
View file

@ -58,7 +58,7 @@ Service_CLICKUP_OAUTH_REDIRECT_URI = https://api.poweron.swiss/api/clickup/auth/
# Stripe Billing (both end with _SECRET for encryption script)
STRIPE_SECRET_KEY_SECRET = PROD_ENC:Z0FBQUFBQnB5dkd6aVA3R3VRS3VHMUgzUEVjYkR4eUZKWFhPUzFTTVlHNnBvT3FienNQaUlBWVpPLXJyVGpGMWk4LXktMXphX0J6ZTVESkJxdjNNa3ZJbF9wX2ppYzdjYlF0cmdVamlEWWJDSmJYYkJseHctTlh4dnNoQWs4SG5haVl2TTNDdXpuaFpqeDBtNkFCbUxMa0RaWG14dmxyOEdILTNrZ2licmNpbXVkN2lFSWoxZW1BODNpV0ZTQ0VaeXRmR1d4RjExMlVFS3MtQU9zZXZlZE1mTmY3OWctUXJHdz09
STRIPE_WEBHOOK_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGNUpTWldsakYydFhFelBrR1lSaWxYT3kyMENOMUljZTJUZHBWcEhhdWVCMzYxZXQ5b3VlTFVRalFiTVdsbGxrdUx0RDFwSEpsOC1sTDJRTEJNQlA3S3ZaQzBtV1h6bWp5VnlMZUgwUlF3cXYxcnljZVE5SWdzLVg3V0syOWRYS08=
STRIPE_WEBHOOK_SECRET = PROD_ENC:Z0FBQUFBQnFLeUFNQ1FhVE94ZzM3V3NCVGVVWnltUndsOG1Ra0hQTmJ3QWY5aXVWeTJsX3A4a3VBSnFQd3drWFRZNFVDdWxCeFgyQ0RpNGQ0SlJOcm9tVE5KZmVqQU1WUjFjeDRJeGE5THdmR0g1V2dQUk5SSjcySnAzR245NW5NUFVDT3lJUWpjWFo=
STRIPE_API_VERSION = 2026-01-28.clover
STRIPE_AUTOMATIC_TAX_ENABLED = false
STRIPE_TAX_RATE_ID_CH_VAT = txr_1TOQZG8WqlVsabrfFEu49pah
@ -72,7 +72,7 @@ Connector_AiTavily_API_SECRET = PROD_ENC:Z0FBQUFBQnFGdnVLcTlLSFJ5b0gwRmJLMFB5MzA
Connector_AiPrivateLlm_API_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGanZ6U3pzZWkwXzVPWGtIQ040XzFrTXc5QWRnazdEeEktaUJ0akJmNnEzbWUzNHczLTJfc2dIdzBDY0FTaXZYcDhxNFdNbTNtbEJTb2VRZ0ZYd05hdlNLR1h6SUFzVml2Z1FLY1BjTl90UWozUGxtak1URnhhZmNDRWFTb0dKVUo=
Connector_AiMistral_API_SECRET = PROD_ENC:Z0FBQUFBQnFGdnVLRHplbzNheDhIdndsU0xUeGlBYVVXWDRzOF9Tek41WjEtSmNqbnVHRXFaZ0dramlfZWlQelpJWVh5T0F2azBaQWU3ajU0TWljaGpMeTlra0g0LVhKeTRKNGxKY0ZqSkxwdTJLdWM5cWdMVC1TVkpLb2lPdHhyeWtieFJFOHdkVy0=
Service_MSFT_TENANT_ID = common
Service_MSFT_TENANT_ID = organizations
# Google Cloud Speech Services configuration
Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQnFIc3YtNDZzenJuZEZiQnVMOWRmZjl3R29QOWZRaGlPdk56WG1DR0FSZU5DM3dENWdoMmRpaks1U1VDNDJkZ3d3UXhSbXlkZ2h3SGZfdk54WXVidF82VkdJQXZiRTk0UlhZaUY1b2kwNzNPSm52VFdsdkwtaHJBb2dpRDBVLXRwd19Bb0dUZDkyV1VWZDJ1TG5mZ0ktYXpuS3U1U0JkZUk5TXpMdnhOaUtMN3BIb0pEZ1N0SlpFN3NNby15VTRfWWtxaF9DYjlJcnVKb0ZualVMTUx2aVNGY0JJdE1oZy1xSVBUZDF1aDM0TGVlTzVrNkFHcjlhcEk0SmRIMTFGdDFTMVUxX1dERk9NTXZMb0tVTFRoc20xME1uRkdVV0Z5N200ZTQzSjVsVExoa2VRZmFBU21ZczF0Vm9Ib3BZM2ZneDkwak12UmFyWWd0eng3ZVVFTUFLVzNOazcxeUhLVWUxcEFIZWtNRi1mT29kM1pqNGJJUUh3UVBlNGY3SlotOWZFUk5aQXFXcUFVdnUzc0Z5bERXYUNPbG14VnBNenFvb2tiQ3lZeHNHUVBlQTdTdVdXOEkxaGxCX016WWktWmN2WFcwM0VmVHdvMHVnY212VFE2cjJwUjdENkFCZF9GcUktWWpmWlNXNWVTMHBPdzVxRi15d3FSRDFra2k0NEFmTmpUeVh3SHRuZWE3WGJ4eUNIcE5tdnRqX2NCZnJoMEI2emU4U0ZYN1Nmdlhva1NacFo3UFh3WnpSdGw5ZmNpSGhicFo0ZThReXl3LW9vUzZaMkFHX2lJalFEMWtjZVdqbVpIZGk0cEdEU01TMl9xQkdSNDllTS1GV3lXS0xROTJvSlhaTjlXenJhQ3lOd2p0VjR5ZjEyektUZGJ3UThJOVJuMzhsTTVBVW9BcDFtcjk5Y0pVeW0zX3R0Nk81R3VDRWEzZnRqSXhFUW5ONHFTSWlwQU4yazlDb01KYlFQRjBFVTljdEJIY29WdF9hUkRJOThVTVFfWlJQUXI0Z3RzWFlzR1ZxUWFBd2I1SW1EMWlKdVprT3dKYTlaREp6TkZEZmVsZGEyalZGc3dHaUkyamdmQWtUT2czNzBCZEg0Vk1HSHFpRnhRYzBRNnN3TFkyaE9uMTVXN1VJTmJwbTNUMTdZbVRyc2d6Yl9aaVBXNmFvanROQVhfbWpXTDRlR1RfbklnYnJUQTZPX2JfNnlrWDVDUWJ4Z3YwNXVsTkJFQlRhTG5DVHpwejdsMGl1bzRfRXRTU2dmb3BVMUo4VkQwa0hsTmFBZnVjVzRrQmNzS2R0ZHNGV24yQnktWENtMUp6eG1MQW1ENE1vWFpFUF9PMEpWZVlxX05hSW1QUGlVT1l3MFp4bDBDZVVldHlEUlVCY1VvVlBNTlBhWFlmcVRobDNqRHo0QjZvNDBqVUVKN3JOb2dtYXQxSWw5NERSeEVRdHNUWndzUkY5RjdBOG1FZFRiVTNVSzl5bDNwdTl2SVd5aW5Ub2Q1YlBDRnpBUDkteU44YnV5X05ONmNndm9teUpqaFZVcVlHdGVRcXRpZkJLVnRuMTJSUFhGWndibExqRW03YUJTWXZXUXJ5WXlvd01ISDFuUFpaMFJzNFVQbWRUb2h1Zi1rcXJXMkRQSUFPeWFJN3lzOFc1d3BjWG1kbWlQWGUwelNiSnJXbUpnajdlQTlQR19XNTF0Q3JYcUMzaGp3eU0yZGhKa3FtX0tleHBfekZaWlRJRlZlSzNDVU56cml0TnFJeUc3b09uYVlwbGxFVFR6WFJVMzRmak5yWjBhcjl5ZmJpQ3hpajRXV1dwbDF5N25tNnI2bWtFem1TS08yV3JybUF0enYxRXpkUVdTNVp4WVB0aldJUUN3TnhHcHdMczh5MTFETzNWLXZFSktsdU1vM1JSNXhraDlJRDl0MEhvR1NOQWRaQW1NdzhpZnFVa1hvdXNwY2FvaThHQjVMOXdySnNIcWJlWERfLXVOcHhpN2ZZOW4yVzB3VTI2a3hvVmFkc29aX2ZUZkY5bi04WEV4MTlxNXQ4cTcwaHE4X3hDWkQxelRwSUl2amZOQ0JXRlJjRFhJNVhjNjRmaXp5eG15LTN1MFRvN3BHTFRZQ1ZFVFYyNUxleFpKTHlIVzRnVHk1Y3ZUbV9RUDdqN1Z2M2ZqVG8wa2RoVHJPeENFRDNHV0wwdi1DbEdOVDFJZnRiZGEydlZyM2tQVExOVlo3LXhIUnhZUnB6a2UzZXNtTjR0S2NzUmFNOWNiSHhHTnJDWHowWk1tbVFKUC14M25aQ1hyYjhJM2pxOEtZY0J1WTZrU3l6cDJOdk5iSXpBUk41MFFVellVZFU4UWVDZXFkQnJFbGxQX2J0S3pReU8zZUdsZUgtTnJuSlpfTjdxR3UxWTBEV0JaRV93eE9qa2dNa2tVTHRxMWNyeUh2VWNrYkdKM3BZOURkUlBxUDA3R2M4NnlMTVR2dmNMZi1lZlhzalRJWlFocGRleVRJYXBBY2hCXzFGZEU4ZVFxbHNic3RDV2FYN1dNaWpkaGdwYTEzRkZYRlEtRXR1cERHdnJKX1Zzb1Q0MnVYZkVhb0VYU1JPdFhoV29TMlhTaEppR1lTTURLYmZnNS1pSzl4T1k5MXJ0YV9qX0ZyQ1R6RFFzRndrTW9IUVlxcG5jcTEyYVU3dkpIR0tZZTZiOXNIRFpIalRtUDFBLVNyd1NfNUMtLW52NVpFZGpQenJCOGw0UlJZNlZVT1ZXTm92R3k4c3hTQXFoNFE3TUFHcjRWc01zT082anJZT0laakl5VUk1WDdDaWlubjIwS3RNcjBjTTdpbUNxSmxNR05JaWtEQURlS1h6N2h0NE9CcW5rQ3NXWkwyNXVBUU5mLTU5MG8xX29xZ0t6Z2pKWmhMNG1BNXBhYWkzY0loSmluUXNKdURwQWRIV2laM2dHQTFxV19lbkZXWmdfWEdiWEZsMGVIWDdoMnJ5dzM0ZGtBM3BSRVp2QzFNbFJSWXBManN5WmFVMlp6aUpWMF9jMTRPbWptM1lsTE41NG1kUW4tT0ZqTzNaZnZ5ZzBLZzNNc1N1X2FMMVJ0N3o4a25LMkxKVUE0dTNhU3hZX3RFMUtKcEgtX1B0cTdEMmYyMzdPaEhoeWhaUGRITC11NzRWYTJnZldiUkFvdG95a1RwWnNKaERkT0kxN1RJMzZQZzFiSjl1SlJieTJjaHBMYmZDUlhTT2hvQnRPaTNhS3NzaVc1Tms0X0FyUHRsSXdCLW1OUWk1RkRKc3pqSjVQTFFROEN5M3pxUGVjZHI4SVM3Qmx1S1A2bEEzNWlVWkFndGpUSm4wcV9jRjQ5T0l1c3ZqN0w3Z1dMV2ZtbU9MbTVSOXphX3VLMko2ZEs3U0NIaFFIMVFIcnN0OGIxSjdxNGlHUHRnOEJDaGwzcXJYNFBnOGdFSVFuSGUyOWJ3WmtlVGhGQWk0THdZd1hUbGRydk83SWVzWUJrb21tSlNvVkJjdWYtcWo0aEc1Ri1XNTZoSENaRWJISmp3UlJNMU9vSnNzZ0VudXpxMDA3aGdfSDBNZlA0Y1gybkF4dGl6SzFOc1VMN0dzVkQxVllkSDhyby12SWNxTFRYdThJUm13S3p3cGFYc05TbVc2YVNtZEdCOFBCUXhadkIzNmdkbXpnc1pLYUhzOEtsY2kxVmNYZm9wOS1LOERLRHJhY2VhanNjaThUZW1rS01wUW05SFJxOGd1VF9STlJZWDRiTV92dXlQTkdxN3BYYTN1SUhRSjRNTy1PZWpGd0xhUlVES0hiWE5LUkM5dHNvenR3TVMySC1ueUZXUkxFY2VyRmhISGc2U2ZxeXY2VkJULV9pOTU1QkI5VUNndnVQcVItTW96VTBqRTdzem1IQ1UxVWtWdjhvTERFeGJ6M3dJNERUV1BTeUlRcG1fbUVjQ0lNREF5QkpLeHJHRkFxQS1kZEE4bXJ2aVVSckVoTkZwNGtoRElIcUktQjA1bkNRclM4dWlqUVRXXzdlQ0VjQWZGSTZlR01NQmU5bHQ3bGNtZWU1eHVvRVdQRVU4Rmx0OFRTaWF3cGgyeFJoM25sRk1GNXJtdEpfcEJmYVFrZXd4eXl0c0ZKVjQ3MkFNRjh5bDBTbFZNd256dmxpQlo5Z1FRM1ZmVTJSb3VrZTk3cXVQYmZ6SnNUWGhlSUhrUjVWUHFwemNmbW1scWVxTkcxT1p5dVlvUjhCSVJaSnBjU0dpc3YzVkt1WUtrd2xoQlVNQXh1eDhmTXNISWMyUnBUMmIwamxlS0tjMVRiWDlBcE03b1BHR1FmdmlsX2ZlMTNCaFNvNG1TeTNiQXRNZ2Y1eE1IaFAxTUZGZ1YyZjEzTG9PaGRCdHJzVlB5Mm12T1NiX2RyT2d2RERCRWFHT0dadW5DZjNtdXE4cHhEQlpub2l3bz0=

3
modules/aicore/aicorePluginMistral.py
View file

@ -343,7 +343,8 @@ class AiMistral(BaseConnectorAi):
content="", success=False, error="No embeddingInput provided"
)
payload = {"model": model.name, "input": texts}
from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS
payload = {"model": model.name, "input": texts, "output_dimension": KNOWLEDGE_EMBEDDING_DIMENSIONS}
response = await self.httpClient.post(model.apiUrl, json=payload)
if response.status_code != 200:

24
modules/aicore/aicorePluginOpenai.py
View file

@ -297,27 +297,6 @@ class AiOpenai(BaseConnectorAi):
version="text-embedding-3-small",
calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00002
),
AiModel(
name="text-embedding-3-large",
displayName="OpenAI Embedding Large",
connectorType="openai",
apiUrl="https://api.openai.com/v1/embeddings",
temperature=0.0,
maxTokens=0,
contextLength=8191,
costPer1kTokensInput=0.00013, # $0.13/M tokens
costPer1kTokensOutput=0.0,
speedRating=9,
qualityRating=10,
functionCall=self.callEmbedding,
priority=PriorityEnum.QUALITY,
processingMode=ProcessingModeEnum.ADVANCED,
operationTypes=createOperationTypeRatings(
(OperationTypeEnum.EMBEDDING, 10)
),
version="text-embedding-3-large",
calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00013
),
AiModel(
name="gpt-image-1",
displayName="OpenAI GPT Image",
@ -547,7 +526,8 @@ class AiOpenai(BaseConnectorAi):
content="", success=False, error="No embeddingInput provided"
)
payload = {"model": model.name, "input": texts}
from modules.datamodels.datamodelKnowledge import KNOWLEDGE_EMBEDDING_DIMENSIONS
payload = {"model": model.name, "input": texts, "dimensions": KNOWLEDGE_EMBEDDING_DIMENSIONS}
response = await self.httpClient.post(model.apiUrl, json=payload)
if response.status_code != 200:

52
modules/aicore/aicorePluginPrivateLlm.py
View file

@ -14,7 +14,7 @@ Models (current — L4 24 GB):
Models (next-gen — RTX PRO 6000 96 GB, auto-activated when pulled in Ollama):
- poweron-text-reasoning: Reasoning (deepseek-r1:70b); complex logic, math, planning
- poweron-vision-general: Vision (llama4:scout); multimodal, long-context documents
- poweron-embed: Embedding (nomic-embed-text); local RAG embedding
- poweron-embed: Embedding (mxbai-embed-large); local RAG embedding (1024 dim)
Pricing: byte-based (~per-token via bytes/4), configured via the PRICE_* constants below.
"""
@ -377,7 +377,7 @@ class AiPrivateLlm(BaseConnectorAi):
),
"ollamaModel": "llama4:scout"
},
# Local Embedding (nomic-embed-text — replaces OpenAI text-embedding-3-small)
# Local Embedding (mxbai-embed-large — nativ 1024 dim, MTEB 64.68)
{
"model": AiModel(
name="poweron-embed",
@ -386,21 +386,21 @@ class AiPrivateLlm(BaseConnectorAi):
apiUrl=f"{self.baseUrl}/v1/embeddings",
temperature=0.0,
maxTokens=0,
contextLength=8192,
contextLength=512,
costPer1kTokensInput=PRICE_EMBED_PER_1K,
costPer1kTokensOutput=0.0,
speedRating=10,
qualityRating=8,
functionCall=self.callAiText,
functionCall=self.callEmbedding,
priority=PriorityEnum.COST,
processingMode=ProcessingModeEnum.BASIC,
operationTypes=createOperationTypeRatings(
(OperationTypeEnum.EMBEDDING, 9),
),
version="nomic-embed-text",
version="mxbai-embed-large",
calculatepriceCHF=_calcPrivateEmbedPriceCHF
),
"ollamaModel": "nomic-embed-text"
"ollamaModel": "mxbai-embed-large"
},
]
@ -505,6 +505,46 @@ class AiPrivateLlm(BaseConnectorAi):
logger.error(f"Error calling Private-LLM text API: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error calling Private-LLM API: {str(e)}")
async def callEmbedding(self, modelCall: AiModelCall) -> AiModelResponse:
"""Generate embeddings via the Private-LLM Embedding endpoint (OpenAI-compatible)."""
try:
model = modelCall.model
texts = modelCall.embeddingInput or []
if not texts:
return AiModelResponse(
content="", success=False, error="No embeddingInput provided"
)
payload = {"model": model.version, "input": texts}
response = await self.httpClient.post(model.apiUrl, json=payload)
if response.status_code != 200:
errorMessage = f"Private-LLM Embedding API error: {response.status_code} - {response.text}"
if response.status_code == 429:
raise RateLimitExceededException(errorMessage)
raise HTTPException(status_code=500, detail=errorMessage)
responseJson = response.json()
embeddings = [item["embedding"] for item in responseJson["data"]]
usage = responseJson.get("usage", {})
return AiModelResponse(
content="",
success=True,
modelId=model.name,
tokensUsed={
"input": usage.get("prompt_tokens", 0),
"output": 0,
"total": usage.get("total_tokens", 0),
},
metadata={"embeddings": embeddings},
)
except (RateLimitExceededException, HTTPException):
raise
except Exception as e:
logger.error(f"Error calling Private-LLM Embedding API: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error calling Private-LLM Embedding API: {str(e)}")
async def callAiVision(self, modelCall: AiModelCall) -> AiModelResponse:
"""
Call the Private-LLM API for vision-based analysis.

55
modules/auth/homeMandateService.py Normal file
View file

@ -0,0 +1,55 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""Ensure new users receive a Home mandate on first login."""
import logging
logger = logging.getLogger(__name__)
def ensureHomeMandate(rootInterface, user) -> None:
"""Ensure user has a Home mandate, but only if they have no mandate memberships
AND no pending invitations.
Invited users should NOT get a Home mandate — they join existing mandates via
invitation acceptance and can create their own later via onboarding.
"""
userId = str(user.id)
userMandates = rootInterface.getUserMandates(userId)
if userMandates:
for um in userMandates:
mandate = rootInterface.getMandate(um.mandateId)
if mandate and (mandate.name or "").startswith("Home ") and not mandate.isSystem:
return
logger.debug(
f"User {user.username} has {len(userMandates)} mandate(s) but no Home — skipping auto-creation"
)
return
try:
normalizedEmail = (user.email or "").strip().lower() if user.email else None
pendingByUsername = rootInterface.getInvitationsByTargetUsername(user.username)
pendingByEmail = (
rootInterface.getInvitationsByEmail(normalizedEmail) if normalizedEmail else []
)
seenIds = set()
for inv in pendingByUsername + pendingByEmail:
if inv.id in seenIds:
continue
seenIds.add(inv.id)
if not inv.revokedAt and (inv.currentUses or 0) < (inv.maxUses or 1):
logger.info(
f"User {user.username} has pending invitation(s) — skipping Home mandate creation"
)
return
except Exception as e:
logger.warning(f"Could not check pending invitations for {user.username}: {e}")
homeMandateLabel = f"Home {user.username}"
rootInterface._provisionMandateForUser(
userId=userId,
mandateLabel=homeMandateLabel,
planKey="TRIAL_14D",
)
logger.info(f"Created Home mandate '{homeMandateLabel}' for user {user.username}")

15
modules/auth/oauthProviderConfig.py
View file

@ -46,6 +46,21 @@ def msftDataScopesForRefresh() -> str:
return " ".join(msftDataScopes)
# Microsoft — Resource ".default": pulls exactly the permissions already
# admin-consented for the app in the user's tenant. Triggers NO interactive /
# admin consent (errors AADSTS65001 only if consent is truly missing), which is
# what we want for tenants that have disabled user consent but granted tenant-wide
# admin consent. msftAuthScopes / msftDataScopes stay as documentation of the
# expected permission set.
MSFT_GRAPH_RESOURCE = "https://graph.microsoft.com"
def msftGraphDefaultScopes() -> list:
"""Single resource ``.default`` scope for Microsoft Graph (must not be mixed
with individual scopes or reserved scopes — MSAL adds openid/profile/offline_access)."""
return [f"{MSFT_GRAPH_RESOURCE}/.default"]
# Infomaniak intentionally has no OAuth scope set: the kDrive + Mail data APIs
# are only reachable with manually issued Personal Access Tokens (see
# wiki/d-guides/infomaniak-token-setup.md). The OAuth /authorize endpoint at

17
modules/auth/tokenRefreshService.py
View file

@ -21,7 +21,9 @@ class TokenRefreshService:
def __init__(self):
self.rate_limit_map = {} # Track refresh attempts per connection
self.max_attempts_per_hour = 3
# Allow enough proactive refreshes per hour so the wider pre-expiry window
# (see proactive_refresh) is never blocked for an actively used connection.
self.max_attempts_per_hour = 6
self.refresh_window_minutes = 60
def _is_rate_limited(self, connection_id: str) -> bool:
@ -215,7 +217,12 @@ class TokenRefreshService:
async def proactive_refresh(self, user_id: str) -> Dict[str, Any]:
"""
Proactively refresh tokens that expire within 5 minutes
Proactively refresh tokens that expire within the refresh window (30 min).
A wide window means any request during the last 30 minutes of a token's
lifetime renews it via the refresh token, so an actively used connection
effectively never lapses (the stored expiresAt always reflects the real
Microsoft/Google token lifetime — it is never faked).
Args:
user_id: User ID to check tokens for
@ -241,7 +248,7 @@ class TokenRefreshService:
failed_count = 0
rate_limited_count = 0
current_time = getUtcTimestamp()
five_minutes = 5 * 60 # 5 minutes in seconds
refresh_window = 30 * 60 # 30 minutes in seconds (matches TokenManager.getFreshToken)
# Process each connection
for connection in connections:
@ -250,9 +257,9 @@ class TokenRefreshService:
connection.tokenExpiresAt and
connection.authority in [AuthAuthority.GOOGLE, AuthAuthority.MSFT]):
# Check if token expires within 5 minutes
# Check if token expires within the refresh window
time_until_expiry = connection.tokenExpiresAt - current_time
if 0 < time_until_expiry <= five_minutes:
if 0 < time_until_expiry <= refresh_window:
# Check rate limiting
if self._is_rate_limited(connection.id):

219
modules/auth/trustedDeviceService.py Normal file
View file

@ -0,0 +1,219 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""
Trusted Device Service.
After successful MFA verification a device can be marked as trusted for a
configurable duration (default 60 days). On subsequent logins from the same
device the MFA step is skipped.
Cookie: ``mfa_trusted`` (httpOnly, Secure, SameSite policy from jwtService).
DB: ``TrustedDevice`` table in poweron_app.
Regulatory basis:
- NIST SP 800-63B Section 5.2.8: Verifier MAY re-authenticate only after a
configurable period when a device is bound to the subscriber.
- Microsoft, Google, AWS implement identical patterns.
"""
import logging
import secrets
from typing import Optional
from fastapi import Request, Response
from modules.shared.configuration import APP_CONFIG
from modules.shared.timeUtils import getUtcNow, getUtcTimestamp
from modules.datamodels.datamodelSecurity import TrustedDevice, Token, TokenPurpose
logger = logging.getLogger(__name__)
_COOKIE_NAME = "mfa_trusted"
_DEFAULT_TRUST_DAYS = 60
_TOKEN_BYTES = 32
def _getTrustDurationDays() -> int:
raw = (APP_CONFIG.get("MFA_TRUST_DURATION_DAYS") or "").strip()
if raw.isdigit() and int(raw) > 0:
return int(raw)
return _DEFAULT_TRUST_DAYS
def createTrustedDevice(userId: str, request: Request, response: Response, db) -> str:
"""Create a TrustedDevice entry and set the cookie on the response.
Returns the device token (cookie value).
"""
from modules.auth.jwtService import _cookiePolicy
trustDays = _getTrustDurationDays()
deviceToken = secrets.token_urlsafe(_TOKEN_BYTES)
now = getUtcTimestamp()
trustedUntil = now + (trustDays * 86400)
device = TrustedDevice(
id=deviceToken,
userId=userId,
trustedUntil=trustedUntil,
userAgent=(request.headers.get("user-agent") or "")[:512],
ipAddress=_getClientIp(request),
createdAt=now,
)
try:
db.recordCreate(TrustedDevice, device.model_dump())
except Exception as e:
logger.error(f"Failed to persist TrustedDevice for userId={userId}: {e}")
return ""
useSecure, samesite, _ = _cookiePolicy()
response.set_cookie(
key=_COOKIE_NAME,
value=deviceToken,
httponly=True,
secure=useSecure,
samesite=samesite,
path="/",
max_age=trustDays * 86400,
)
logger.info(f"Trusted device created for userId={userId}, valid {trustDays}d")
return deviceToken
def isTrustedDevice(request: Request, userId: str, db) -> bool:
"""Check if the current request comes from a trusted device for the given user."""
deviceToken = request.cookies.get(_COOKIE_NAME)
if not deviceToken:
return False
try:
records = db.getRecordset(
TrustedDevice,
recordFilter={"id": deviceToken, "userId": userId},
)
if not records:
return False
device = records[0]
trustedUntil = device.get("trustedUntil", 0)
if isinstance(trustedUntil, (int, float)) and trustedUntil > getUtcTimestamp():
return True
return False
except Exception as e:
logger.warning(f"Error checking trusted device for userId={userId}: {e}")
return False
def revokeTrustedDevices(userId: str, db) -> int:
"""Revoke all trusted devices for a user. Returns count of deleted entries."""
try:
records = db.getRecordset(TrustedDevice, recordFilter={"userId": userId})
count = 0
for rec in records:
db.recordDelete(TrustedDevice, rec["id"])
count += 1
if count:
logger.info(f"Revoked {count} trusted device(s) for userId={userId}")
return count
except Exception as e:
logger.error(f"Failed to revoke trusted devices for userId={userId}: {e}")
return 0
def clearTrustedDeviceCookie(response: Response) -> None:
"""Clear the mfa_trusted cookie."""
from modules.auth.jwtService import _cookiePolicy
useSecure, samesite, samesiteHeader = _cookiePolicy()
secure_flag = "; Secure" if useSecure else ""
response.headers.append(
"Set-Cookie",
f"{_COOKIE_NAME}=deleted; Path=/; Max-Age=0; Expires=Thu, 01 Jan 1970 00:00:00 GMT; HttpOnly{secure_flag}; SameSite={samesiteHeader}"
)
response.delete_cookie(
key=_COOKIE_NAME,
path="/",
secure=useSecure,
httponly=True,
samesite=samesite,
)
def cleanupExpiredDevices(db) -> int:
"""Remove TrustedDevice entries past their trustedUntil. Returns deleted count."""
try:
records = db.getRecordset(TrustedDevice, recordFilter={})
now = getUtcTimestamp()
count = 0
for rec in records:
if rec.get("trustedUntil", 0) < now:
db.recordDelete(TrustedDevice, rec["id"])
count += 1
if count:
logger.info(f"Cleaned up {count} expired trusted device(s)")
return count
except Exception as e:
logger.error(f"Error cleaning up expired trusted devices: {e}")
return 0
def _getClientIp(request: Request) -> Optional[str]:
"""Extract client IP from request (respects X-Forwarded-For)."""
forwarded = request.headers.get("x-forwarded-for")
if forwarded:
return forwarded.split(",")[0].strip()
if request.client:
return request.client.host
return None
# --- Scheduler Integration ---
async def _runTokenAndDeviceCleanup() -> None:
"""Scheduled task: remove expired tokens and trusted devices."""
try:
from modules.connectors.connectorDbPostgre import ConnectorPostgre
db = ConnectorPostgre("poweron_app")
now = getUtcTimestamp()
# Expired auth-session tokens
tokens = db.getRecordset(
Token,
recordFilter={"tokenPurpose": TokenPurpose.AUTH_SESSION.value},
)
expiredCount = 0
for t in tokens:
if t.get("expiresAt", 0) < now:
db.recordDelete(Token, t["id"])
expiredCount += 1
# Expired trusted devices
deviceCount = cleanupExpiredDevices(db)
if expiredCount or deviceCount:
logger.info(
f"Token cleanup: {expiredCount} expired token(s), "
f"{deviceCount} expired trusted device(s) removed"
)
except Exception as e:
logger.error(f"Token/device cleanup failed: {e}")
def registerTokenCleanupScheduler() -> None:
"""Register daily token cleanup job. Call during app startup."""
try:
from modules.shared.eventManagement import eventManager
eventManager.registerCron(
jobId="token_device_cleanup",
func=_runTokenAndDeviceCleanup,
cronKwargs={"hour": "4", "minute": "0"},
)
logger.info("Token/device cleanup scheduler registered (daily 04:00)")
except Exception as e:
logger.warning(f"Failed to register token cleanup scheduler: {e}")

1
modules/connectors/connectorDbPostgre.py
View file

@ -871,6 +871,7 @@ class DatabaseConnector:
("jsonb", "TEXT"): "TEXT USING \"{col}\"::text",
("text", "DOUBLE PRECISION"): _TEXT_TO_DOUBLE,
("text", "INTEGER"): "INTEGER USING NULLIF(\"{col}\", '')::integer",
("text", "BOOLEAN"): "BOOLEAN USING CASE WHEN \"{col}\" IN ('true', '1', 't', 'yes') THEN TRUE ELSE FALSE END",
("timestamp without time zone", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}" AT TIME ZONE \'UTC\')',
("timestamp with time zone", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}")',
("date", "DOUBLE PRECISION"): 'DOUBLE PRECISION USING EXTRACT(EPOCH FROM "{col}"::timestamp AT TIME ZONE \'UTC\')',

17
modules/connectors/connectorVoiceGoogle.py
View file

@ -19,6 +19,21 @@ from modules.shared.voiceCatalog import getDefaultVoice
logger = logging.getLogger(__name__)
_STT_LANGUAGE_MAP = {
"de-CH": "de-DE",
"de-AT": "de-DE",
"en-GB": "en-US",
"en-AU": "en-US",
"fr-CH": "fr-FR",
"it-CH": "it-IT",
"pt-BR": "pt-PT",
}
def _normalizeSttLanguage(language: str) -> str:
"""Map regional language variants to codes supported by Google STT models."""
return _STT_LANGUAGE_MAP.get(language, language)
def _buildPrimarySttRecognitionFields(
*,
@ -116,6 +131,7 @@ class ConnectorGoogleSpeech:
Returns:
Dict containing transcribed text, confidence, and metadata
"""
language = _normalizeSttLanguage(language)
try:
# Treat sampleRate=0 as unknown (invalid value from client)
if sampleRate is not None and sampleRate <= 0:
@ -480,6 +496,7 @@ class ConnectorGoogleSpeech:
Dicts with keys: isFinal, transcript, confidence, stabilityScore, audioDurationSec;
optionally endOfSingleUtterance, reconnectRequired
"""
language = _normalizeSttLanguage(language)
STREAM_LIMIT_SEC = 290
streamStartTs = time.time()
totalAudioBytes = 0

10
modules/datamodels/datamodelKnowledge.py
View file

@ -9,7 +9,7 @@ These models support the 3-tier RAG architecture:
- Global Layer: scope=global (sysAdmin only)
- Workflow Layer: workflowId-scoped (WorkflowMemory)
Vector fields use json_schema_extra={"db_type": "vector(1536)"} for pgvector.
Vector fields use json_schema_extra with db_type=vector(KNOWLEDGE_EMBEDDING_DIMENSIONS) for pgvector.
"""
from typing import Dict, Any, List, Optional
@ -19,6 +19,8 @@ from modules.shared.i18nRegistry import i18nModel
from modules.shared.timeUtils import getUtcTimestamp
import uuid
KNOWLEDGE_EMBEDDING_DIMENSIONS = 1024
@i18nModel("Datei-Inhaltsindex")
class FileContentIndex(PowerOnModel):
@ -163,7 +165,7 @@ class ContentChunk(PowerOnModel):
embedding: Optional[List[float]] = Field(
default=None,
description="pgvector embedding (NOT NULL for text chunks)",
json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"},
json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"},
)
@ -210,7 +212,7 @@ class RoundMemory(PowerOnModel):
embedding: Optional[List[float]] = Field(
default=None,
description="Embedding of summary for semantic retrieval",
json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"},
json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"},
)
@ -251,5 +253,5 @@ class WorkflowMemory(PowerOnModel):
embedding: Optional[List[float]] = Field(
default=None,
description="Optional embedding for semantic lookup",
json_schema_extra={"label": "Embedding", "db_type": "vector(1536)"},
json_schema_extra={"label": "Embedding", "db_type": f"vector({KNOWLEDGE_EMBEDDING_DIMENSIONS})"},
)

10
modules/datamodels/datamodelNavigation.py
View file

@ -320,6 +320,16 @@ NAVIGATION_SECTIONS = [
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-sessions",
"objectKey": "ui.admin.sessions",
"label": t("Sessions & Geräte"),
"icon": "FaDesktop",
"path": "/admin/sessions",
"order": 92,
"adminOnly": True,
"sysAdminOnly": True,
},
{
"id": "admin-languages",
"objectKey": "ui.admin.languages",

2
modules/datamodels/datamodelRbac.py
View file

@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
from modules.datamodels.datamodelBase import PowerOnModel
from modules.shared.i18nRegistry import i18nModel
from modules.datamodels.datamodelUtils import TextMultilingual
from modules.datamodels.datamodelUam import AccessLevel
from modules.datamodels.datamodelUam import AccessLevel, User
class AccessRuleContext(str, Enum):

37
modules/datamodels/datamodelSecurity.py
View file

@ -124,6 +124,43 @@ class Token(PowerOnModel):
return data
@i18nModel("Vertrauenswuerdiges Geraet")
class TrustedDevice(PowerOnModel):
"""A device trusted after successful MFA verification (skips MFA for configured duration)."""
id: str = Field(
default_factory=lambda: str(uuid.uuid4()),
description="Random token stored as httpOnly cookie value",
json_schema_extra={"label": "ID"},
)
userId: str = Field(
...,
description="User this trusted device belongs to",
json_schema_extra={"label": "Benutzer-ID", "fk_target": {"db": "poweron_app", "table": "UserInDB", "labelField": "username"}},
)
trustedUntil: float = Field(
...,
description="UTC timestamp until which the device is trusted",
json_schema_extra={"label": "Vertrauenswuerdig bis", "frontend_type": "timestamp"},
)
userAgent: Optional[str] = Field(
default=None,
description="Browser user agent at time of trust grant",
json_schema_extra={"label": "User-Agent"},
)
ipAddress: Optional[str] = Field(
default=None,
description="IP address at time of trust grant",
json_schema_extra={"label": "IP-Adresse"},
)
createdAt: float = Field(
default_factory=getUtcTimestamp,
description="When the device was trusted",
json_schema_extra={"label": "Erstellt am", "frontend_type": "timestamp"},
)
model_config = ConfigDict(use_enum_values=True)
@i18nModel("Authentifizierungsereignis")
class AuthEvent(PowerOnModel):
"""Authentication event for audit logging."""

14
modules/features/commcoach/interfaceFeatureCommcoach.py
View file

@ -23,6 +23,9 @@ from .datamodelCommcoach import (
CoachingTask, CoachingTaskStatus,
CoachingScore,
CoachingUserProfile,
CoachingPersona,
ModulePersonaMapping,
CoachingBadge,
)
logger = logging.getLogger(__name__)
@ -261,35 +264,29 @@ class CommcoachObjects:
# =========================================================================
def getPersonas(self, userId: str, instanceId: str) -> List[Dict[str, Any]]:
from .datamodelCommcoach import CoachingPersona
builtins = self.db.getRecordset(CoachingPersona, recordFilter={"userId": "system"})
custom = self.db.getRecordset(CoachingPersona, recordFilter={"userId": userId, "instanceId": instanceId})
all = builtins + custom
return [p for p in all if p.get("isActive", True)]
def getPersona(self, personaId: str) -> Optional[Dict[str, Any]]:
from .datamodelCommcoach import CoachingPersona
records = self.db.getRecordset(CoachingPersona, recordFilter={"id": personaId})
return records[0] if records else None
def createPersona(self, data: Dict[str, Any]) -> Dict[str, Any]:
from .datamodelCommcoach import CoachingPersona
data["createdAt"] = getIsoTimestamp()
data["updatedAt"] = getIsoTimestamp()
return self.db.recordCreate(CoachingPersona, data)
def updatePersona(self, personaId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]:
from .datamodelCommcoach import CoachingPersona
updates["updatedAt"] = getIsoTimestamp()
return self.db.recordModify(CoachingPersona, personaId, updates)
def deletePersona(self, personaId: str) -> bool:
from .datamodelCommcoach import CoachingPersona
return self.db.recordDelete(CoachingPersona, personaId)
def getAllPersonas(self, instanceId: str) -> List[Dict[str, Any]]:
"""All personas (builtin + custom for this instance), including inactive."""
from .datamodelCommcoach import CoachingPersona
builtins = self.db.getRecordset(CoachingPersona, recordFilter={"userId": "system"})
custom = self.db.getRecordset(CoachingPersona, recordFilter={"instanceId": instanceId})
custom = [p for p in custom if p.get("userId") != "system"]
@ -300,11 +297,9 @@ class CommcoachObjects:
# =========================================================================
def getModulePersonas(self, moduleId: str) -> List[Dict[str, Any]]:
from .datamodelCommcoach import ModulePersonaMapping
return self.db.getRecordset(ModulePersonaMapping, recordFilter={"moduleId": moduleId})
def setModulePersonas(self, moduleId: str, personaIds: List[str], instanceId: str) -> List[Dict[str, Any]]:
from .datamodelCommcoach import ModulePersonaMapping
existing = self.db.getRecordset(ModulePersonaMapping, recordFilter={"moduleId": moduleId})
for rec in existing:
self.db.recordDelete(ModulePersonaMapping, rec["id"])
@ -325,18 +320,15 @@ class CommcoachObjects:
# =========================================================================
def getBadges(self, userId: str, instanceId: str) -> List[Dict[str, Any]]:
from .datamodelCommcoach import CoachingBadge
records = self.db.getRecordset(CoachingBadge, recordFilter={"userId": userId, "instanceId": instanceId})
records.sort(key=lambda r: r.get("awardedAt") or 0, reverse=True)
return records
def hasBadge(self, userId: str, instanceId: str, badgeKey: str) -> bool:
from .datamodelCommcoach import CoachingBadge
records = self.db.getRecordset(CoachingBadge, recordFilter={"userId": userId, "instanceId": instanceId, "badgeKey": badgeKey})
return len(records) > 0
def awardBadge(self, data: Dict[str, Any]) -> Dict[str, Any]:
from .datamodelCommcoach import CoachingBadge
data["awardedAt"] = getUtcTimestamp()
data["createdAt"] = getIsoTimestamp()
return self.db.recordCreate(CoachingBadge, data)

12
modules/features/commcoach/routeFeatureCommcoach.py
View file

@ -18,7 +18,7 @@ from fastapi import APIRouter, HTTPException, Depends, Request, Query
from fastapi.responses import StreamingResponse, Response
from modules.auth import limiter, getRequestContext, RequestContext
from modules.shared.timeUtils import getIsoTimestamp
from modules.shared.timeUtils import getIsoTimestamp, getUtcTimestamp
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.interfaces.interfaceFeatures import getFeatureInterface
@ -33,7 +33,10 @@ from .datamodelCommcoach import (
UpdateProfileRequest,
CreatePersonaRequest, UpdatePersonaRequest, SetModulePersonasRequest,
)
from .serviceCommcoach import CommcoachService, emitSessionEvent, getSessionEventQueue
from .serviceCommcoach import (
CommcoachService, emitSessionEvent, getSessionEventQueue,
getUserVoicePrefs, stripMarkdownForTts, buildTtsConfigErrorMessage,
)
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeFeatureCommcoach")
logger = logging.getLogger(__name__)
@ -333,7 +336,6 @@ async def startSession(
try:
from modules.interfaces.interfaceVoiceObjects import getVoiceInterface
voiceInterface = getVoiceInterface(context.user, mandateId)
from .serviceCommcoach import getUserVoicePrefs, stripMarkdownForTts, buildTtsConfigErrorMessage
language, voiceName = getUserVoicePrefs(userId, mandateId)
ttsResult = await voiceInterface.textToSpeech(
text=stripMarkdownForTts(greetingText),
@ -378,7 +380,6 @@ async def startSession(
asyncio.create_task(service.processSessionOpening(sessionId, moduleId, interface))
async def _newSessionEventGenerator():
from modules.shared.timeUtils import getIsoTimestamp
timeoutCount = 0
try:
while True:
@ -468,7 +469,6 @@ async def cancelSession(
raise HTTPException(status_code=404, detail=routeApiMsg("Session not found"))
_validateOwnership(session, context)
from modules.shared.timeUtils import getUtcTimestamp
interface.updateSession(sessionId, {
"status": CoachingSessionStatus.CANCELLED.value,
"endedAt": getUtcTimestamp(),
@ -581,7 +581,6 @@ async def sendAudioStream(
if not audioBody:
raise HTTPException(status_code=400, detail=routeApiMsg("No audio data received"))
from .serviceCommcoach import getUserVoicePrefs
language, _ = getUserVoicePrefs(str(context.user.id), mandateId)
moduleId = session.get("moduleId")
@ -765,7 +764,6 @@ async def updateTaskStatus(
updates = {"status": body.status.value}
if body.status == CoachingTaskStatus.DONE:
from modules.shared.timeUtils import getUtcTimestamp
updates["completedAt"] = getUtcTimestamp()
updated = interface.updateTask(taskId, updates)

6
modules/features/commcoach/serviceCommcoach.py
View file

@ -15,7 +15,7 @@ import asyncio
from datetime import datetime, timezone
from typing import Optional, Dict, Any, List
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelUam import User, UserVoicePreferences
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum
from modules.shared.timeUtils import getIsoTimestamp, getUtcTimestamp
@ -98,7 +98,6 @@ def getUserVoicePrefs(userId: str, mandateId: Optional[str] = None) -> tuple:
"""Load voice language and voiceName from central UserVoicePreferences.
Returns (language, voiceName) tuple."""
try:
from modules.datamodels.datamodelUam import UserVoicePreferences
from modules.interfaces.interfaceDbApp import getRootInterface
rootIf = getRootInterface()
prefs = rootIf.db.getRecordset(
@ -430,7 +429,6 @@ async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, A
"""Pre-AI-call: identify which documents the user references and what action is needed."""
if not docs:
return {"read": [], "update": [], "create": [], "noDocumentAction": True}
from . import serviceCommcoachAi as aiPrompts
docCatalog = [{"id": d.get("id", ""), "title": d.get("summary") or d.get("fileName", ""), "summary": (d.get("summary") or "")[:100]} for d in docs]
prompt = aiPrompts.buildDocumentIntentPrompt(combinedUserPrompt, docCatalog)
try:
@ -744,7 +742,6 @@ class CommcoachService:
4. Map agent events to CommCoach SSE events
5. Post-processing: store message, TTS, tasks, scores
"""
from . import interfaceFeatureCommcoach as interfaceDb
# Store user message
userMsg = CoachingMessage(
@ -907,7 +904,6 @@ class CommcoachService:
)
agentService = getService("agent", serviceContext)
from modules.datamodels.datamodelAi import PriorityEnum, OperationTypeEnum
config = AgentConfig(
toolSet="commcoach" if useTools else "none",
maxRounds=3 if useTools else 1,

4
modules/features/neutralization/serviceNeutralization/subProcessList.py
View file

@ -13,7 +13,7 @@ from typing import Dict, List, Any, Union
from dataclasses import dataclass
from io import StringIO
from .subParseString import StringParser
from .subPatterns import getPatternForHeader, HeaderPatterns
from .subPatterns import getPatternForHeader, HeaderPatterns, findPatternsInText, DataPatterns
@dataclass
class NeutralizationTableData:
@ -157,7 +157,6 @@ class ListProcessor:
processedAttrs[attrName] = self.string_parser.mapping[attrValue]
else:
# Check if attribute value matches any data patterns
from .subPatterns import findPatternsInText, DataPatterns
matches = findPatternsInText(attrValue, DataPatterns.patterns)
if matches:
patternName = matches[0][0]
@ -191,7 +190,6 @@ class ListProcessor:
# Skip if already a placeholder
if not self.string_parser._isPlaceholder(text):
# Check if text matches any patterns
from .subPatterns import findPatternsInText, DataPatterns
patternMatches = findPatternsInText(text, DataPatterns.patterns)
if patternMatches:

3
modules/features/realEstate/interfaceFeatureRealEstate.py
View file

@ -29,7 +29,7 @@ from modules.dbHelpers.dbRegistry import registerDatabase
from modules.security.rbac import RbacClass
from modules.datamodels.datamodelRbac import AccessRuleContext
from modules.datamodels.datamodelUam import AccessLevel
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC, buildDataObjectKey
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
logger = logging.getLogger(__name__)
@ -796,7 +796,6 @@ class RealEstateObjects:
return False
tableName = modelClass.__name__
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None)
permissions = self.rbac.getUserPermissions(
self.currentUser,

17
modules/features/teamsbot/routeFeatureTeamsbot.py
View file

@ -49,7 +49,13 @@ from .datamodelTeamsbot import (
)
# Import service
from .service import TeamsbotService
from .service import (
TeamsbotService,
getActiveService,
getActiveService as _getActiveService,
createAiService,
sessionEvents,
)
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeFeatureTeamsbot")
@ -328,7 +334,6 @@ async def startSession(
if context.isSysAdmin and joinMode == TeamsbotJoinMode.SYSTEM_BOT:
systemBot = interface.getActiveSystemBot(mandateId)
if not systemBot:
from .datamodelTeamsbot import TeamsbotSystemBot
allBots = interface.db.getRecordset(TeamsbotSystemBot, recordFilter={"isActive": True})
if allBots:
systemBot = allBots[0]
@ -537,7 +542,6 @@ async def streamSession(
async def _eventGenerator():
"""Generate SSE events from the session event queue."""
from .service import sessionEvents
# Send initial session state with stats
stats = interface.getSessionStats(sessionId)
@ -545,7 +549,6 @@ async def streamSession(
# Send current bot WebSocket connection state so the operator UI can
# render the live indicator without waiting for the next connect/disconnect.
from .service import getActiveService as _getActiveService
yield f"data: {json.dumps({'type': 'botConnectionState', 'data': {'connected': _getActiveService(sessionId) is not None}})}\n\n"
# Stream events
@ -1040,7 +1043,6 @@ async def submitDirectorPrompt(
detail=routeApiMsg(f"Too many files ({len(fileIds)}); max {DIRECTOR_PROMPT_FILE_LIMIT}"),
)
from .service import getActiveService
service = getActiveService(sessionId)
if not service:
raise HTTPException(
@ -1108,7 +1110,6 @@ async def deleteDirectorPrompt(
if not context.isPlatformAdmin and prompt.get("operatorUserId") != str(context.user.id):
raise HTTPException(status_code=404, detail=f"Prompt '{promptId}' not found")
from .service import getActiveService
service = getActiveService(sessionId)
if service:
await service.removePersistentPrompt(promptId)
@ -1134,7 +1135,6 @@ async def testVoice(
):
"""Test TTS voice with AI-generated sample text in the correct language."""
from modules.interfaces.interfaceVoiceObjects import getVoiceInterface
from .service import createAiService
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum
mandateId = _validateInstanceAccess(instanceId, context)
@ -1547,7 +1547,6 @@ async def postTranscript(
originalUser = rootUser
# Process transcript through the service pipeline
from .service import TeamsbotService
from modules.interfaces.interfaceVoiceObjects import getVoiceInterface
service = TeamsbotService(originalUser, mandateId, instanceId, config)
@ -1600,7 +1599,6 @@ async def postBotStatus(
if not originalUser:
originalUser = rootUser
from .service import TeamsbotService
service = TeamsbotService(originalUser, mandateId, instanceId, config)
interface = interfaceDb.getInterface(originalUser, mandateId=mandateId, featureInstanceId=instanceId)
@ -1640,7 +1638,6 @@ async def botWebsocket(
# Load the original user who started the session (has RBAC roles in mandate)
# Bot callbacks have no HTTP auth, so we reconstruct the user context from the session record.
from modules.interfaces.interfaceDbApp import getRootInterface
rootInterface = getRootInterface()
rootUser = rootInterface.currentUser

11
modules/features/trustee/accounting/accountingBridge.py
View file

@ -19,7 +19,12 @@ from .accountingConnectorBase import (
SyncResult,
)
from .accountingRegistry import getAccountingRegistry
from modules.features.trustee.datamodelFeatureTrustee import TrusteeDocument
from modules.features.trustee.datamodelFeatureTrustee import (
TrusteeDocument,
TrusteeAccountingConfig,
TrusteePosition,
TrusteeAccountingSync,
)
logger = logging.getLogger(__name__)
@ -33,7 +38,6 @@ class AccountingBridge:
async def getActiveConfig(self, featureInstanceId: str) -> Optional[Dict[str, Any]]:
"""Load the active TrusteeAccountingConfig for a feature instance."""
from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig
records = self._trusteeInterface.db.getRecordset(
TrusteeAccountingConfig,
recordFilter={"featureInstanceId": featureInstanceId, "isActive": True},
@ -128,7 +132,6 @@ class AccountingBridge:
Optional _resolved* params allow pushBatchToAccounting to pass a pre-resolved
connector/config so we don't decrypt per position (avoids rate-limit).
"""
from modules.features.trustee.datamodelFeatureTrustee import TrusteePosition, TrusteeAccountingSync
connector = _resolvedConnector
plainConfig = _resolvedPlainConfig
@ -306,7 +309,6 @@ class AccountingBridge:
# Update last sync on config record
if configRecord:
from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig
updatePayload = {
"lastSyncAt": time.time(),
"lastSyncStatus": "success" if result.success else "error",
@ -335,7 +337,6 @@ class AccountingBridge:
async def refreshChartOfAccounts(self, featureInstanceId: str) -> List[AccountingChart]:
"""Fetch the full chart of accounts from the external system and cache it locally on TrusteeAccountingConfig."""
from modules.features.trustee.datamodelFeatureTrustee import TrusteeAccountingConfig
connector, plainConfig, configRecord = await self._resolveConnectorAndConfig(featureInstanceId)
if not connector or not plainConfig or not configRecord:

4
modules/features/trustee/interfaceFeatureTrustee.py
View file

@ -16,7 +16,7 @@ from pydantic import ValidationError
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.shared.configuration import APP_CONFIG
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC, buildDataObjectKey
from modules.security.rbac import RbacClass
from modules.datamodels.datamodelUam import User, AccessLevel
from modules.datamodels.datamodelRbac import AccessRuleContext
@ -309,7 +309,6 @@ class TrusteeObjects:
return False
tableName = modelClass.__name__
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None)
permissions = self.rbac.getUserPermissions(
self.currentUser,
@ -338,7 +337,6 @@ class TrusteeObjects:
return AccessLevel.NONE
tableName = modelClass.__name__
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None)
permissions = self.rbac.getUserPermissions(
self.currentUser,

9
modules/features/trustee/routeFeatureTrustee.py
View file

@ -49,7 +49,7 @@ from modules.datamodels.datamodelPagination import (
normalize_pagination_dict,
)
from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext
from modules.shared.i18nRegistry import apiRouteContext
from modules.shared.i18nRegistry import apiRouteContext, resolveText
routeApiMsg = apiRouteContext("routeFeatureTrustee")
@ -170,7 +170,6 @@ def getQuickActions(
if role and role.roleLabel:
userRoleLabels.add(role.roleLabel)
from modules.shared.i18nRegistry import resolveText
lang = (language or "de").strip() or "de"
@ -1201,7 +1200,6 @@ def _buildSyncStatusByPosition(interface, instanceId: str) -> Dict[str, Dict[str
``error``, so a successful retry hides an old failure. Any other status
(`pending`, `cancelled`, ...) is kept verbatim.
"""
from .datamodelFeatureTrustee import TrusteeAccountingSync
syncRecords = interface.db.getRecordset(
TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId}
@ -1290,7 +1288,6 @@ def _handlePositionMode(instanceId, mandateId, mode, column, pagination, context
"""Handle mode=filterValues and mode=ids for trustee positions."""
from modules.dbHelpers.paginationHelpers import handleIdsInMemory, handleFilterValuesInMemory
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
from .datamodelFeatureTrustee import TrusteePositionView
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
if mode == "filterValues":
if not column:
@ -1507,7 +1504,6 @@ def delete_accounting_config(
"""Remove the accounting integration for this instance."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingConfig
records = interface.db.getRecordset(TrusteeAccountingConfig, recordFilter={"featureInstanceId": instanceId})
for r in records:
interface.db.recordDelete(TrusteeAccountingConfig, r.get("id"))
@ -1602,7 +1598,6 @@ def get_sync_status(
"""Get sync status of all positions for this instance."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingSync
records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"featureInstanceId": instanceId})
return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]}
@ -1618,7 +1613,6 @@ def get_position_sync_status(
"""Get sync status for a specific position."""
mandateId = _validateInstanceAccess(instanceId, context)
interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId)
from .datamodelFeatureTrustee import TrusteeAccountingSync
records = interface.db.getRecordset(TrusteeAccountingSync, recordFilter={"positionId": positionId, "featureInstanceId": instanceId})
return {"items": [{k: v for k, v in r.items() if not k.startswith("_")} for r in records]}
@ -1776,7 +1770,6 @@ def _serializeRoleForApi(role) -> Dict[str, Any]:
here (same pattern as ``getQuickActions``). Without this the React tree
crashes with "Objects are not valid as a React child".
"""
from modules.shared.i18nRegistry import resolveText
payload = role.model_dump()
payload["description"] = resolveText(payload.get("description"))
return payload

1
modules/features/trustee/workflows/methodTrustee/actions/extractFromFiles.py
View file

@ -269,7 +269,6 @@ async def _extractWithAi(
) -> Dict[str, Any]:
"""3-step extraction: (1a) OCR/text via Vision AI, (1b) classify text, (2) structure by type."""
await self.services.ai.ensureAiObjectsInitialized()
from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference
docList = DocumentReferenceList(
references=[DocumentItemReference(documentId=chatDocumentId, fileName=fileName)]

228
modules/features/workspace/routeFeatureWorkspace.py
View file

@ -28,7 +28,7 @@ from modules.features.workspace import interfaceFeatureWorkspace
from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface
from modules.interfaces.interfaceAiObjects import AiObjects
from modules.shared.eventManager import get_event_manager
from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum, PendingFileEdit
from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum, PendingFileEdit, AgentConfig
from modules.shared.timeUtils import parseTimestamp
from modules.shared.i18nRegistry import apiRouteContext, resolveText
routeApiMsg = apiRouteContext("routeFeatureWorkspace")
@ -489,6 +489,158 @@ def _collectPriorFileIds(chatInterface, workflowId: str) -> List[str]:
return result
# Default context budget for prior files (metadata injection per file costs tokens
# in EVERY agent round). Override per workspace instance via instanceConfig
# key "maxPriorContextFiles". Files outside the budget remain fully accessible
# to the agent via conversation history and readFile - nothing is lost.
DEFAULT_MAX_PRIOR_CONTEXT_FILES = 10
async def _selectPriorFilesForContext(
priorFileIds: List[str],
prompt: str,
aiObjects,
mandateId: str,
featureInstanceId: str,
maxFiles: int,
) -> List[str]:
"""Select which prior files get their metadata injected into the agent context.
Selection only happens when more candidates exist than the budget allows -
otherwise all files pass through untouched. Selection order:
1. Files whose indexed chunks are semantically relevant to the current prompt
2. Remaining budget filled with the most recent files (so files without an
index entry are never unfairly dropped)
"""
if len(priorFileIds) <= maxFiles:
return priorFileIds
rankedFileIds: List[str] = []
try:
embeddingResponse = await aiObjects.callEmbedding([prompt])
embeddings = (embeddingResponse.metadata or {}).get("embeddings", [])
if embeddings:
knowledgeIf = getKnowledgeInterface()
results = knowledgeIf.semanticSearch(
queryVector=embeddings[0],
featureInstanceId=featureInstanceId,
mandateId=mandateId,
limit=maxFiles * 3,
)
priorSet = set(priorFileIds)
for chunk in results:
fid = chunk.get("fileId") if isinstance(chunk, dict) else getattr(chunk, "fileId", None)
if fid and fid in priorSet and fid not in rankedFileIds:
rankedFileIds.append(fid)
if len(rankedFileIds) >= maxFiles:
break
except Exception as e:
logger.warning(f"Relevance ranking for prior files failed: {e}")
for fid in reversed(priorFileIds):
if len(rankedFileIds) >= maxFiles:
break
if fid not in rankedFileIds:
rankedFileIds.append(fid)
logger.info(
f"Prior-file selection: {len(priorFileIds)} candidates -> {len(rankedFileIds)} in context "
f"(budget={maxFiles}, relevance-ranked, recency fill)"
)
return rankedFileIds
async def _ensureFilesIndexed(
fileIds: List[str],
user,
mandateId: str,
featureInstanceId: str,
) -> int:
"""Ensure all attached files have embeddings in the knowledge store.
Checks FileContentIndex for each file. Files not yet indexed are extracted
and embedded inline so that subsequent RAG queries can find their content.
Indexing is idempotent (content-hash check in requestIngestion), so each
file is only ever processed once - re-attaching an indexed file is a no-op.
Returns the number of files that were newly indexed.
"""
if not fileIds:
return 0
knowledgeIf = getKnowledgeInterface(user)
unindexedIds = []
for fid in fileIds:
existing = knowledgeIf.getFileContentIndex(fid)
status = (existing.get("status") if isinstance(existing, dict) else getattr(existing, "status", "")) if existing else ""
if status not in ("indexed", "embedding"):
unindexedIds.append(fid)
if not unindexedIds:
return 0
logger.info(f"Ensure-embed: {len(unindexedIds)}/{len(fileIds)} files need indexing")
from modules.serviceCenter import getService
from modules.serviceCenter.context import ServiceCenterContext
indexCtx = ServiceCenterContext(
user=user, mandateId=mandateId, featureInstanceId=featureInstanceId,
)
try:
knowledgeService = getService("knowledge", indexCtx)
except Exception as e:
logger.warning(f"Ensure-embed: knowledge service unavailable: {e}")
return 0
chatInterface = interfaceDbChat.getInterface(user)
indexed = 0
for fid in unindexedIds:
try:
fileInfo = chatInterface.getFileInfo(fid) if chatInterface else None
if not fileInfo:
continue
fileName = fileInfo.get("fileName", "")
mimeType = fileInfo.get("mimeType", "")
rawBytes = chatInterface.getFileData(fid)
if not rawBytes:
continue
extractionService = getService("extraction", indexCtx)
extracted = extractionService.extractContentFromBytes(
rawBytes, fileName, mimeType, documentId=fid,
)
contentObjects = [
{
"contentType": getattr(p, "contentType", "text"),
"data": getattr(p, "data", "") or "",
"contentObjectId": getattr(p, "contentObjectId", "") or str(uuid.uuid4()),
"contextRef": getattr(p, "contextRef", {}) or {},
}
for p in (extracted.parts or [])
if getattr(p, "data", None)
]
if not contentObjects:
continue
await knowledgeService.indexFile(
fileId=fid,
fileName=fileName,
mimeType=mimeType,
userId=user.id if user else "",
featureInstanceId=featureInstanceId,
mandateId=mandateId,
contentObjects=contentObjects,
structure=getattr(extracted, "structure", None),
)
indexed += 1
except Exception as e:
logger.debug(f"Ensure-embed: skipping file {fid}: {e}")
if indexed:
logger.info(f"Ensure-embed: indexed {indexed} file(s) before agent start")
return indexed
async def _deriveWorkflowName(prompt: str, aiService) -> str:
"""Use AI to generate a concise workflow title from the user prompt."""
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum
@ -740,23 +892,31 @@ async def _runWorkspaceAgent(
priorFileIds = _collectPriorFileIds(chatInterface, workflowId)
currentFileIdSet = set(fileIds or [])
mergedFileIds = list(fileIds or [])
for pf in priorFileIds:
if pf not in currentFileIdSet:
mergedFileIds.append(pf)
if len(mergedFileIds) > len(fileIds or []):
candidatePriorIds = [pf for pf in priorFileIds if pf not in currentFileIdSet]
# Embed-first rule: newly attached files are indexed into the knowledge
# store BEFORE the agent starts, so RAG retrieval works from round 1.
await _ensureFilesIndexed(fileIds or [], user, mandateId, instanceId)
_cfg = instanceConfig or {}
if candidatePriorIds:
maxPriorFiles = int(_cfg.get("maxPriorContextFiles", DEFAULT_MAX_PRIOR_CONTEXT_FILES))
candidatePriorIds = await _selectPriorFilesForContext(
candidatePriorIds, prompt, aiObjects, mandateId, instanceId, maxPriorFiles,
)
mergedFileIds = list(fileIds or []) + candidatePriorIds
if candidatePriorIds:
logger.info(
f"Merged {len(mergedFileIds) - len(fileIds or [])} prior file(s) into agent context "
f"Merged {len(candidatePriorIds)} prior file(s) into agent context "
f"(total: {len(mergedFileIds)}) for workflow {workflowId}"
)
accumulatedText = ""
messagePersisted = False
_cfg = instanceConfig or {}
_toolSet = _cfg.get("toolSet", "core")
_agentCfg = _cfg.get("agentConfig")
from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentConfig
agentCfgDict = dict(_agentCfg) if isinstance(_agentCfg, dict) else {}
try:
@ -1146,6 +1306,10 @@ async def getWorkspaceMessages(
except Exception as e:
logger.debug(f"getWorkspaceMessages: cannot read attachments for {workflowId}: {e}")
# Resolve labels server-side as the single source of truth: every returned
# ID carries a label, and IDs whose record no longer exists are dropped
# here. The client renders what it gets without any timing-dependent
# reconciliation against its own (asynchronously loaded) source lists.
attachedDsLabels: Dict[str, str] = {}
attachedFdsLabels: Dict[str, str] = {}
if attachedDsIds or attachedFdsIds:
@ -1153,27 +1317,39 @@ async def getWorkspaceMessages(
rootIf = getRootInterface()
if attachedDsIds:
from modules.datamodels.datamodelDataSource import DataSource
resolvedDsIds: List[str] = []
for dsId in attachedDsIds:
try:
records = rootIf.db.getRecordset(DataSource, recordFilter={"id": dsId})
if records:
lbl = records[0].get("label") or records[0].get("path") or ""
if lbl:
attachedDsLabels[dsId] = str(lbl)
except Exception:
pass
except Exception as e:
# Transient DB error: keep the id; the client falls back to
# its own dataSources list for the label.
logger.warning(f"getWorkspaceMessages: label lookup failed for DataSource {dsId}: {e}")
resolvedDsIds.append(dsId)
continue
if not records:
continue # source was deleted -- drop the stale attachment
resolvedDsIds.append(dsId)
lbl = records[0].get("label") or records[0].get("path") or ""
attachedDsLabels[dsId] = str(lbl) if lbl else dsId
attachedDsIds = resolvedDsIds
if attachedFdsIds:
from modules.datamodels.datamodelFeatures import FeatureDataSource
resolvedFdsIds: List[str] = []
for fdsId in attachedFdsIds:
try:
records = rootIf.db.getRecordset(FeatureDataSource, recordFilter={"id": fdsId})
if records:
tbl = records[0].get("tableName") or ""
lbl = records[0].get("label") or tbl
if lbl:
attachedFdsLabels[fdsId] = str(lbl)
except Exception:
pass
except Exception as e:
logger.warning(f"getWorkspaceMessages: label lookup failed for FeatureDataSource {fdsId}: {e}")
resolvedFdsIds.append(fdsId)
continue
if not records:
continue # source was deleted -- drop the stale attachment
resolvedFdsIds.append(fdsId)
tbl = records[0].get("tableName") or ""
lbl = records[0].get("label") or tbl
attachedFdsLabels[fdsId] = str(lbl) if lbl else fdsId
attachedFdsIds = resolvedFdsIds
return JSONResponse({
"messages": items,
@ -1467,9 +1643,9 @@ async def listFeatureDataSources(
from modules.serviceCenter.core.flagResolution import buildEffectiveByWorkspaceFds
rootIf = getRootInterface()
recordFilter: dict = {}
if wsMandateId:
recordFilter["mandateId"] = wsMandateId
if not wsMandateId:
return JSONResponse({"featureDataSources": []})
recordFilter: dict = {"mandateId": wsMandateId}
records = rootIf.db.getRecordset(FeatureDataSource, recordFilter=recordFilter) or []
if not records:
return JSONResponse({"featureDataSources": []})

80
modules/interfaces/interfaceAiObjects.py
View file

@ -4,7 +4,9 @@ import logging
import asyncio
import uuid
import base64
import hashlib
import json
from collections import OrderedDict
from typing import Dict, Any, List, Union, Tuple, Optional, Callable, AsyncGenerator
from dataclasses import dataclass, field
import time
@ -13,7 +15,7 @@ logger = logging.getLogger(__name__)
from modules.aicore.aicoreModelRegistry import modelRegistry
from modules.aicore.aicoreModelSelector import modelSelector
from modules.aicore.aicoreBase import RateLimitExceededException
from modules.aicore.aicoreBase import RateLimitExceededException, ContextLengthExceededException
from modules.datamodels.datamodelAi import (
AiModel,
AiCallOptions,
@ -463,7 +465,6 @@ class AiObjects:
toolChoice: Any = None,
) -> AsyncGenerator[Union[str, AiCallResponse], None]:
"""Stream a model call. Yields str deltas, then final AiCallResponse with billing."""
from modules.datamodels.datamodelAi import AiModelCall, AiModelResponse
inputBytes = sum(len(str(m.get("content", "")).encode("utf-8")) for m in messages)
startTime = time.time()
@ -535,14 +536,33 @@ class AiObjects:
Returns:
AiCallResponse with metadata["embeddings"] containing the vectors.
"""
from modules.aicore.aicoreBase import ContextLengthExceededException
if options is None:
options = AiCallOptions(operationType=OperationTypeEnum.EMBEDDING)
else:
options.operationType = OperationTypeEnum.EMBEDDING
combinedText = " ".join(texts[:3])[:500]
# Serve known vectors from cache; only unknown texts go to the API.
resolvedVectors: Dict[int, List[float]] = {}
pendingTexts: List[str] = []
pendingPositions: List[int] = []
for i, t in enumerate(texts):
cachedVector = _embeddingCacheGet(t)
if cachedVector is not None:
resolvedVectors[i] = cachedVector
else:
pendingTexts.append(t)
pendingPositions.append(i)
if not pendingTexts:
logger.debug(f"Embedding cache hit for all {len(texts)} text(s)")
return AiCallResponse(
content="", modelName="embedding-cache", priceCHF=0.0,
processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=0,
metadata={"embeddings": [resolvedVectors[i] for i in range(len(texts))]},
)
combinedText = " ".join(pendingTexts[:3])[:500]
availableModels = modelRegistry.getAvailableModels()
allowedProviders = getattr(options, 'allowedProviders', None) if options else None
@ -575,13 +595,13 @@ class AiObjects:
for attempt, model in enumerate(failoverModelList):
try:
logger.info(f"Embedding call with {model.name} (attempt {attempt + 1}/{len(failoverModelList)})")
inputBytes = sum(len(t.encode("utf-8")) for t in texts)
inputBytes = sum(len(t.encode("utf-8")) for t in pendingTexts)
startTime = time.time()
batches = _buildEmbeddingBatches(texts, model.contextLength)
batches = _buildEmbeddingBatches(pendingTexts, model.contextLength)
logger.info(
f"Embedding: {len(texts)} texts -> {len(batches)} batch(es), "
f"model contextLength={model.contextLength}"
f"Embedding: {len(pendingTexts)} texts ({len(resolvedVectors)} cached) -> "
f"{len(batches)} batch(es), model contextLength={model.contextLength}"
)
allEmbeddings: List[List[float]] = []
@ -606,11 +626,17 @@ class AiObjects:
if totalPriceCHF == 0.0:
totalPriceCHF = model.calculatepriceCHF(processingTime, inputBytes, 0)
for j, position in enumerate(pendingPositions):
if j < len(allEmbeddings):
resolvedVectors[position] = allEmbeddings[j]
_embeddingCachePut(pendingTexts[j], allEmbeddings[j])
mergedEmbeddings = [resolvedVectors.get(i, []) for i in range(len(texts))]
response = AiCallResponse(
content="", modelName=model.name, provider=model.connectorType,
priceCHF=totalPriceCHF, processingTime=processingTime,
bytesSent=inputBytes, bytesReceived=0, errorCount=0,
metadata={"embeddings": allEmbeddings}
metadata={"embeddings": mergedEmbeddings}
)
if self.billingCallback:
@ -681,6 +707,28 @@ class AiObjects:
_CHARS_PER_TOKEN = 4
_SAFETY_MARGIN = 0.90
# In-process cache for embedding vectors. Identical texts (e.g. the same user
# prompt embedded once for prior-file selection and once for RAG context
# building) hit the cache instead of paying for a second API call.
_EMBEDDING_CACHE_MAX_ENTRIES = 256
_embeddingCache: OrderedDict = OrderedDict()
def _embeddingCacheGet(text: str) -> Optional[List[float]]:
key = hashlib.sha256(text.encode("utf-8")).hexdigest()
vector = _embeddingCache.get(key)
if vector is not None:
_embeddingCache.move_to_end(key)
return vector
def _embeddingCachePut(text: str, vector: List[float]) -> None:
key = hashlib.sha256(text.encode("utf-8")).hexdigest()
_embeddingCache[key] = vector
_embeddingCache.move_to_end(key)
while len(_embeddingCache) > _EMBEDDING_CACHE_MAX_ENTRIES:
_embeddingCache.popitem(last=False)
def _estimateTokens(text: str) -> int:
"""Rough token estimate: 1 token ~ 4 characters."""
@ -691,9 +739,7 @@ def _buildEmbeddingBatches(texts: List[str], contextLength: int) -> List[List[st
"""Split a list of texts into batches whose total estimated token count
stays within the model's contextLength (with safety margin).
Each individual text is assumed to already be within limits (enforced by
the chunking layer). If a single text exceeds the budget, it is placed
in its own batch as a last resort.
Texts that individually exceed the per-input limit are truncated to fit.
"""
if not texts:
return []
@ -701,11 +747,21 @@ def _buildEmbeddingBatches(texts: List[str], contextLength: int) -> List[List[st
return [texts]
maxTokensPerBatch = int(contextLength * _SAFETY_MARGIN)
maxCharsPerInput = maxTokensPerBatch * _CHARS_PER_TOKEN
batches: List[List[str]] = []
currentBatch: List[str] = []
currentTokens = 0
for text in texts:
if len(text) > maxCharsPerInput:
# API hard limit per input. File content never hits this (the
# chunking layer splits at ~400 tokens); only oversized search
# queries can, where truncation is semantically acceptable.
logger.warning(
f"Embedding input truncated from {len(text)} to {maxCharsPerInput} chars "
f"(model input limit {contextLength} tokens)"
)
text = text[:maxCharsPerInput]
textTokens = _estimateTokens(text)
if currentBatch and (currentTokens + textTokens) > maxTokensPerBatch:
batches.append(currentBatch)

32
modules/interfaces/interfaceDbApp.py
View file

@ -22,7 +22,7 @@ from modules.shared.configuration import APP_CONFIG
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp
from modules.shared.i18nRegistry import resolveText
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, buildDataObjectKey, copySystemRolesToMandate
from modules.security.rbac import RbacClass
from modules.datamodels.datamodelUam import (
User,
@ -39,14 +39,14 @@ from modules.datamodels.datamodelRbac import (
)
from modules.datamodels.datamodelUam import AccessLevel
from modules.datamodels.datamodelSecurity import Token, AuthEvent, TokenStatus, TokenPurpose
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult, TableListView
from modules.datamodels.datamodelMembership import (
UserMandate,
UserMandateRole,
FeatureAccess,
FeatureAccessRole,
)
from modules.datamodels.datamodelFeatures import Feature, FeatureInstance
from modules.datamodels.datamodelFeatures import Feature, FeatureInstance, FeatureDataSource, DataNeutralizerAttributes
from modules.datamodels.datamodelInvitation import Invitation
from modules.datamodels.datamodelNotification import UserNotification
@ -220,7 +220,6 @@ class AppObjects:
tableName = modelClass.__name__
# Use buildDataObjectKey for semantic namespace lookup
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName)
permissions = self.rbac.getUserPermissions(
self.currentUser,
@ -1122,8 +1121,6 @@ class AppObjects:
def _deleteUserReferencedData(self, userId: str) -> None:
"""Deletes all data associated with a user (full cascade)."""
try:
from modules.datamodels.datamodelNotification import UserNotification
from modules.datamodels.datamodelInvitation import Invitation
# 1. FeatureAccess + FeatureAccessRole
accesses = self.db.getRecordset(FeatureAccess, recordFilter={"userId": userId})
@ -1560,7 +1557,6 @@ class AppObjects:
# Copy system template roles to new mandate (admin, user, viewer + AccessRules)
try:
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
copiedCount = copySystemRolesToMandate(self.db, mandateId)
logger.info(f"Copied {copiedCount} system roles to new mandate {mandateId}")
except Exception as e:
@ -1576,8 +1572,6 @@ class AppObjects:
``mandateLabel`` is the display name (Voller Name); a unique slug ``name`` (Kurzzeichen) is derived.
"""
from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS
from modules.datamodels.datamodelFeatures import FeatureInstance
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
from modules.interfaces.interfaceFeatures import getFeatureInterface
from modules.shared.featureDiscovery import loadFeatureMainModules
plan = BUILTIN_PLANS.get(planKey)
@ -1847,7 +1841,6 @@ class AppObjects:
raise PermissionError(f"No permission to delete mandate {mandateId}")
if not force:
from modules.shared.timeUtils import getUtcTimestamp
self.db.recordModify(Mandate, mandateId, {"enabled": False, "deletedAt": getUtcTimestamp()})
logger.info(f"Soft-deleted mandate {mandateId} (30-day retention)")
return True
@ -1858,8 +1851,6 @@ class AppObjects:
from modules.datamodels.datamodelFiles import FileItem
from modules.datamodels.datamodelDataSource import DataSource
from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk
from modules.datamodels.datamodelFeatures import FeatureDataSource
from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes
instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
@ -1983,7 +1974,6 @@ class AppObjects:
# 3b. Billing data cascade handled by onMandateDelete lifecycle hook (interfaceDbBilling)
# 3c. Delete Invitations for this mandate
from modules.datamodels.datamodelInvitation import Invitation
invitations = self.db.getRecordset(Invitation, recordFilter={"mandateId": mandateId})
for inv in invitations:
self.db.recordDelete(Invitation, inv.get("id"))
@ -1991,7 +1981,6 @@ class AppObjects:
logger.info(f"Cascade: deleted {len(invitations)} Invitations for mandate {mandateId}")
# 4. Delete mandate-level Roles
from modules.datamodels.datamodelRbac import Role, AccessRule
roles = self.db.getRecordset(Role, recordFilter={"mandateId": mandateId})
for role in roles:
rules = self.db.getRecordset(AccessRule, recordFilter={"roleId": role.get("id")})
@ -3112,8 +3101,12 @@ class AppObjects:
# Token methods
def saveAccessToken(self, token: Token, replace_existing: bool = True) -> None:
"""Save an access token for the current user (must NOT have connectionId)"""
def saveAccessToken(self, token: Token, replace_existing: bool = False) -> None:
"""Save an access token for the current user (must NOT have connectionId).
Multi-session: replace_existing=False (default) keeps existing sessions alive.
Only set replace_existing=True for explicit single-session scenarios.
"""
try:
# Validate that this is NOT a connection token
if token.connectionId:
@ -3957,7 +3950,6 @@ class AppObjects:
def getTableListViews(self, contextKey: str) -> list:
"""Return all saved views for the current user and contextKey."""
from modules.datamodels.datamodelPagination import TableListView
try:
rows = self.db.getRecordset(
TableListView,
@ -3976,7 +3968,6 @@ class AppObjects:
def getTableListView(self, contextKey: str, viewKey: str):
"""Return one view by viewKey or None if not found."""
from modules.datamodels.datamodelPagination import TableListView
try:
rows = self.db.getRecordset(
TableListView,
@ -3992,8 +3983,6 @@ class AppObjects:
def createTableListView(self, contextKey: str, viewKey: str, displayName: str, config: dict):
"""Create a new view. Raises ValueError if viewKey already exists for this context."""
from modules.datamodels.datamodelPagination import TableListView
from modules.shared.timeUtils import getUtcTimestamp
if self.getTableListView(contextKey=contextKey, viewKey=viewKey) is not None:
raise ValueError(f"View '{viewKey}' already exists for context '{contextKey}'")
data = {
@ -4014,8 +4003,6 @@ class AppObjects:
def updateTableListView(self, viewId: str, updates: dict):
"""Update an existing view by its primary key id."""
from modules.datamodels.datamodelPagination import TableListView
from modules.shared.timeUtils import getUtcTimestamp
try:
updates = {**updates, "updatedAt": getUtcTimestamp()}
self.db.recordModify(TableListView, viewId, updates)
@ -4030,7 +4017,6 @@ class AppObjects:
def deleteTableListView(self, viewId: str) -> bool:
"""Delete a view by primary key id. Returns True on success."""
from modules.datamodels.datamodelPagination import TableListView
try:
self.db.recordDelete(TableListView, viewId)
return True

6
modules/interfaces/interfaceDbBilling.py
View file

@ -14,11 +14,11 @@ from typing import Dict, Any, List, Optional, Union
from datetime import date, datetime, timedelta, timezone
import uuid
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.connectors.connectorDbPostgre import DatabaseConnector, getModelFields, parseRecordFields
from modules.shared.configuration import APP_CONFIG
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.shared.timeUtils import getUtcTimestamp
from modules.datamodels.datamodelUam import User, Mandate
from modules.datamodels.datamodelUam import User, Mandate, UserInDB
from modules.datamodels.datamodelMembership import UserMandate
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
from modules.datamodels.datamodelBilling import (
@ -1654,8 +1654,6 @@ class BillingObjects:
`amount` column. Resolves matching mandate/user IDs via the app DB
first, then builds a single SQL query with OR-combined conditions.
"""
from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields
from modules.datamodels.datamodelUam import UserInDB
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
table = BillingTransaction.__name__

6
modules/interfaces/interfaceDbChat.py
View file

@ -27,14 +27,14 @@ from modules.datamodels.datamodelChat import (
UserInputRequest
)
import json
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelUam import User, Mandate
# DYNAMIC PART: Connectors to the Interface
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, buildDataObjectKey
# Basic Configurations
from modules.shared.configuration import APP_CONFIG
@ -393,7 +393,6 @@ class ChatObjects:
tableName = modelClass.__name__
# Use buildDataObjectKey for semantic namespace lookup
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName)
permissions = self.rbac.getUserPermissions(
self.currentUser,
@ -826,7 +825,6 @@ class ChatObjects:
if not effectiveMandateId:
# Fall back to Root mandate (first mandate in system)
try:
from modules.datamodels.datamodelUam import Mandate
from modules.security.rootAccess import getRootDbAppConnector
dbAppConn = getRootDbAppConnector()
allMandates = dbAppConn.getRecordset(Mandate)

60
modules/interfaces/interfaceDbKnowledge.py
View file

@ -13,7 +13,7 @@ from typing import Dict, Any, List, Optional
from modules.connectors.connectorDbPostgre import getCachedConnector
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk, RoundMemory, WorkflowMemory
from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk, RoundMemory, WorkflowMemory, KNOWLEDGE_EMBEDDING_DIMENSIONS
from modules.datamodels.datamodelUam import User
from modules.shared.configuration import APP_CONFIG
from modules.shared.timeUtils import getUtcTimestamp
@ -732,3 +732,61 @@ def getInterface(currentUser: Optional[User] = None) -> KnowledgeObjects:
interface.setUserContext(currentUser)
return interface
def migrateVectorDimensions():
"""Idempotent boot migration: ensures all vector columns match KNOWLEDGE_EMBEDDING_DIMENSIONS.
Checks the actual pgvector dimension via pg_attribute.atttypmod.
If it differs from the target, nulls existing embeddings and alters the column type.
Safe to call on every startup — skips when dimensions already match or table doesn't exist.
"""
targetDim = KNOWLEDGE_EMBEDDING_DIMENSIONS
interface = getInterface()
db = interface.db
vectorTables = [
("ContentChunk", "embedding"),
("RoundMemory", "embedding"),
("WorkflowMemory", "embedding"),
]
for table, col in vectorTables:
try:
with db.borrowConn() as conn:
with conn.cursor() as cursor:
cursor.execute(
"SELECT COUNT(*) FROM information_schema.tables "
"WHERE LOWER(table_name) = LOWER(%s) AND table_schema = 'public'",
(table,),
)
if cursor.fetchone()["count"] == 0:
continue
cursor.execute(
"SELECT a.atttypmod FROM pg_attribute a "
"JOIN pg_class c ON a.attrelid = c.oid "
"JOIN pg_namespace n ON c.relnamespace = n.oid "
"WHERE c.relname = %s AND a.attname = %s AND n.nspname = 'public'",
(table, col),
)
row = cursor.fetchone()
if not row:
continue
currentDim = row["atttypmod"]
if currentDim == targetDim:
continue
logger.info(
"Migrating %s.%s from vector(%s) to vector(%s) — clearing existing embeddings",
table, col, currentDim, targetDim,
)
cursor.execute(f'UPDATE "{table}" SET "{col}" = NULL WHERE "{col}" IS NOT NULL')
cursor.execute(
f'ALTER TABLE "{table}" ALTER COLUMN "{col}" TYPE vector({targetDim})'
)
logger.info("Migration of %s.%s completed", table, col)
except Exception as e:
logger.error("Vector dimension migration failed for %s.%s: %s", table, col, e)

7
modules/interfaces/interfaceDbManagement.py
View file

@ -16,7 +16,7 @@ from typing import Dict, Any, List, Optional, Union
from modules.connectors.connectorDbPostgre import DatabaseConnector, getCachedConnector
from modules.dbHelpers.dbRegistry import registerDatabase
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC
from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, buildDataObjectKey
from modules.security.rbac import RbacClass
from modules.datamodels.datamodelRbac import AccessRuleContext
from modules.datamodels.datamodelUam import AccessLevel
@ -317,7 +317,6 @@ class ComponentObjects:
return False
tableName = modelClass.__name__
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey(tableName)
permissions = self.rbac.getUserPermissions(
self.currentUser,
@ -1066,7 +1065,6 @@ class ComponentObjects:
Owners always can. Non-owners need RBAC ALL level."""
if self._isFolderOwner(folder):
return
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey("FileFolder")
permissions = self.rbac.getUserPermissions(
self.currentUser, AccessRuleContext.DATA, objectKey,
@ -1207,7 +1205,6 @@ class ComponentObjects:
self._requireFolderWriteAccess(folder, folderId, "update")
if scope == "global":
from modules.interfaces.interfaceRbac import buildDataObjectKey
objectKey = buildDataObjectKey("FileFolder")
permissions = self.rbac.getUserPermissions(
self.currentUser, AccessRuleContext.DATA, objectKey,
@ -1387,8 +1384,6 @@ class ComponentObjects:
Owners always can. Non-owners need RBAC ALL level."""
if self._isFileOwner(file):
return
from modules.interfaces.interfaceRbac import buildDataObjectKey
from modules.datamodels.datamodelRbac import AccessRuleContext
objectKey = buildDataObjectKey("FileItem")
permissions = self.rbac.getUserPermissions(
self.currentUser, AccessRuleContext.DATA, objectKey,

10
modules/interfaces/interfaceRbac.py
View file

@ -32,10 +32,10 @@ from datetime import datetime, timezone
from typing import List, Dict, Any, Optional, Type, Union
from pydantic import BaseModel
from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role
from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel, Mandate
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
from modules.datamodels.datamodelUtils import coerce_text_multilingual
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.connectors.connectorDbPostgre import DatabaseConnector, getModelFields, parseRecordFields
from modules.security.rbac import RbacClass
from modules.security.rootAccess import getRootDbAppConnector
@ -379,7 +379,6 @@ def getRecordsetWithRBAC(
# Handle JSONB fields and ensure numeric types are correct
# Import the helper function from connector module
from modules.connectors.connectorDbPostgre import getModelFields
fields = getModelFields(modelClass)
for record in records:
for fieldName, fieldType in fields.items():
@ -511,7 +510,6 @@ def getRecordsetPaginatedWithRBAC(
whereValues.append(value)
if pagination and pagination.filters:
from modules.connectors.connectorDbPostgre import getModelFields
fields = getModelFields(modelClass)
validColumns = set(fields.keys())
for key, val in pagination.filters.items():
@ -545,7 +543,6 @@ def getRecordsetPaginatedWithRBAC(
orderParts: List[str] = []
if pagination and pagination.sort:
from modules.connectors.connectorDbPostgre import getModelFields
validColumns = set(getModelFields(modelClass).keys())
for sf in pagination.sort:
if sf.field in validColumns:
@ -569,7 +566,6 @@ def getRecordsetPaginatedWithRBAC(
cursor.execute(dataSql, whereValues)
records = [dict(row) for row in cursor.fetchall()]
from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields
fields = getModelFields(modelClass)
for record in records:
parseRecordFields(record, fields, f"table {table}")
@ -625,7 +621,6 @@ def getDistinctColumnValuesWithRBAC(
if not connector._ensureTableExists(modelClass):
return []
from modules.connectors.connectorDbPostgre import getModelFields
fields = getModelFields(modelClass)
if column not in fields:
return []
@ -949,7 +944,6 @@ def buildRbacWhereClause(
# Fall back to Root mandate (first mandate in system) for GROUP access
# This allows system-level tables to be accessed without explicit mandate context
try:
from modules.datamodels.datamodelUam import Mandate
dbApp = getRootDbAppConnector()
allMandates = dbApp.getRecordset(Mandate)
if allMandates:

4
modules/interfaces/interfaceTableHelpers.py
View file

@ -12,7 +12,7 @@ from collections import defaultdict
from functools import cmp_to_key
from typing import Any, Dict, List, Optional
from modules.datamodels.datamodelPagination import PaginationParams
from modules.datamodels.datamodelPagination import PaginationParams, SortField, GroupBand, GroupLayout
logger = logging.getLogger(__name__)
@ -85,7 +85,6 @@ def applyViewToParams(params: Optional["PaginationParams"], viewConfig: Optional
Returns the (mutated) params, or a new minimal PaginationParams when
params is None (so callers always get a valid object).
"""
from modules.datamodels.datamodelPagination import SortField
if not viewConfig:
return params
@ -264,7 +263,6 @@ def buildGroupLayout(
-------
(page_items, GroupLayout | None)
"""
from modules.datamodels.datamodelPagination import GroupBand, GroupLayout
if not groupByLevels:
offset = (page - 1) * pageSize

8
modules/interfaces/interfaceWorkflowAutomation.py
View file

@ -375,11 +375,11 @@ class WorkflowAutomationObjects:
return []
records = self.db.getRecordset(
AutoRun,
recordFilter={},
recordFilter={"workflowId": wf_ids},
)
if not records:
return []
runs = [dict(r) for r in records if r.get("workflowId") in wf_ids]
runs = [dict(r) for r in records]
wf_by_id = {w["id"]: w for w in workflows}
for r in runs:
wf = wf_by_id.get(r.get("workflowId"), {})
@ -652,7 +652,7 @@ class WorkflowAutomationObjects:
def copyTemplateToUser(self, templateId: str) -> Optional[Dict[str, Any]]:
"""Copy a template to a new user-owned workflow with templateScope='user'."""
template = self.getWorkflow(templateId)
template = self.db.getRecord(AutoWorkflow, templateId)
if not template or not template.get("isTemplate"):
return None
data = {
@ -672,7 +672,7 @@ class WorkflowAutomationObjects:
def shareTemplate(self, templateId: str, scope: str) -> Optional[Dict[str, Any]]:
"""Change a template's scope. Sets sharedReadOnly=True for shared scopes, False for user scope."""
template = self.getWorkflow(templateId)
template = self.db.getRecord(AutoWorkflow, templateId)
if not template or not template.get("isTemplate"):
return None
updated = self.db.recordModify(AutoWorkflow, templateId, {

1
modules/routes/routeAdminFeatures.py
View file

@ -473,7 +473,6 @@ def list_feature_instances(
items = [inst.model_dump() for inst in instances]
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
from modules.datamodels.datamodelFeatures import FeatureInstance
enrichRowsWithFkLabels(items, FeatureInstance, db=rootInterface.db)
if mode == "filterValues":

5
modules/routes/routeAdminRbacRules.py
View file

@ -20,7 +20,7 @@ import math
from modules.auth import limiter, getRequestContext, requirePlatformAdmin, RequestContext
from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
from modules.datamodels.datamodelRbac import AccessRuleContext, AccessRule, Role
from modules.datamodels.datamodelMembership import UserMandate
from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
from modules.interfaces.interfaceDbApp import getInterface, getRootInterface
from modules.shared.i18nRegistry import apiRouteContext, t, resolveText
@ -40,7 +40,6 @@ def _getAdminMandateIds(context: RequestContext) -> List[str]:
"""Get mandate IDs where the user has an admin role."""
mandateIds = []
try:
from modules.interfaces.interfaceDbApp import getRootInterface
rootInterface = getRootInterface()
userMandates = rootInterface.getUserMandates(str(context.user.id))
for um in userMandates:
@ -64,7 +63,6 @@ def _getAdminMandateIds(context: RequestContext) -> List[str]:
def _isRoleInAdminMandates(roleId: str, adminMandateIds: List[str]) -> bool:
"""Check if a role belongs to one of the admin's mandates."""
try:
from modules.interfaces.interfaceDbApp import getRootInterface
rootInterface = getRootInterface()
role = rootInterface.getRole(roleId)
if not role:
@ -1405,7 +1403,6 @@ def cleanup_duplicate_access_rules(
# Phase 2: Fix template role assignments
# UserMandateRole should reference mandate-instance roles, not templates
# =====================================================================
from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole
allUserMandateRoles = rootInterface.db.getRecordset(UserMandateRole)
templateFixDetails = []

203
modules/routes/routeAdminSessions.py Normal file
View file

@ -0,0 +1,203 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""
Admin endpoints for session and trusted device management.
Allows mandate-admins and platform-admins to view and revoke active sessions
and trusted devices for users under their jurisdiction.
"""
from fastapi import APIRouter, HTTPException, status, Depends, Request, Query
from typing import Dict, Any, List
import logging
from modules.auth import limiter, getCurrentUser
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelSecurity import Token, TokenPurpose, TokenStatus, TrustedDevice
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.shared.timeUtils import getUtcTimestamp
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeAdminSessions")
logger = logging.getLogger(__name__)
router = APIRouter(
prefix="/api/admin/sessions",
tags=["Admin Sessions"],
responses={404: {"description": "Not found"}},
)
def _requireAdmin(currentUser: User) -> None:
"""Ensure the caller is a platform admin or sysAdmin."""
if not (getattr(currentUser, "isPlatformAdmin", False) or getattr(currentUser, "isSysAdmin", False)):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=routeApiMsg("Only platform admins can manage sessions"),
)
@router.get("")
@limiter.limit("30/minute")
def listSessions(
request: Request,
userId: str = Query(..., description="User ID whose sessions to list"),
currentUser: User = Depends(getCurrentUser),
) -> List[Dict[str, Any]]:
"""List active auth sessions for a user."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
tokens = rootInterface.db.getRecordset(
Token,
recordFilter={
"userId": userId,
"tokenPurpose": TokenPurpose.AUTH_SESSION.value,
"status": TokenStatus.ACTIVE.value,
},
)
now = getUtcTimestamp()
result = []
for t in tokens:
expiresAt = t.get("expiresAt", 0)
if expiresAt < now:
continue
result.append({
"sessionId": t.get("sessionId"),
"tokenId": t.get("id"),
"authority": t.get("authority"),
"createdAt": t.get("sysCreatedAt"),
"expiresAt": expiresAt,
})
return result
@router.delete("/{sessionId}")
@limiter.limit("30/minute")
def revokeSession(
request: Request,
sessionId: str,
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Revoke a single session by sessionId (sets status=REVOKED, not delete)."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
adminId = str(currentUser.id)
tokens = rootInterface.db.getRecordset(
Token,
recordFilter={
"sessionId": sessionId,
"tokenPurpose": TokenPurpose.AUTH_SESSION.value,
"status": TokenStatus.ACTIVE.value,
},
)
count = 0
for t in tokens:
rootInterface.revokeTokenById(t["id"], revokedBy=adminId, reason="admin session revoke")
count += 1
if count == 0:
raise HTTPException(status_code=404, detail=routeApiMsg("Session not found"))
logger.info("Admin %s revoked session %s (%d token(s))", currentUser.username, sessionId, count)
return {"revoked": count, "sessionId": sessionId}
@router.delete("")
@limiter.limit("10/minute")
def revokeAllSessions(
request: Request,
userId: str = Query(..., description="User ID whose sessions to revoke"),
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Revoke ALL active sessions for a user (force logout everywhere)."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
adminId = str(currentUser.id)
count = rootInterface.revokeTokensByUser(
userId, revokedBy=adminId, reason="admin revoke all sessions",
)
logger.info("Admin %s revoked all sessions for userId=%s (%d token(s))", currentUser.username, userId, count)
return {"revoked": count, "userId": userId}
# --- Trusted Devices ---
trustedDeviceRouter = APIRouter(
prefix="/api/admin/trusted-devices",
tags=["Admin Sessions"],
responses={404: {"description": "Not found"}},
)
@trustedDeviceRouter.get("")
@limiter.limit("30/minute")
def listTrustedDevices(
request: Request,
userId: str = Query(..., description="User ID whose trusted devices to list"),
currentUser: User = Depends(getCurrentUser),
) -> List[Dict[str, Any]]:
"""List trusted devices for a user."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
devices = rootInterface.db.getRecordset(
TrustedDevice, recordFilter={"userId": userId}
)
now = getUtcTimestamp()
result = []
for d in devices:
result.append({
"id": d.get("id", ""),
"trustedUntil": d.get("trustedUntil"),
"isExpired": d.get("trustedUntil", 0) < now,
"userAgent": d.get("userAgent"),
"ipAddress": d.get("ipAddress"),
"createdAt": d.get("createdAt"),
})
return result
@trustedDeviceRouter.delete("/{deviceId}")
@limiter.limit("30/minute")
def revokeTrustedDevice(
request: Request,
deviceId: str,
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Revoke a single trusted device by ID."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
existing = rootInterface.db.getRecord(TrustedDevice, deviceId)
if not existing:
raise HTTPException(status_code=404, detail=routeApiMsg("Trusted device not found"))
rootInterface.db.recordDelete(TrustedDevice, deviceId)
logger.info("Admin %s revoked trusted device %s", currentUser.username, deviceId)
return {"revoked": 1, "deviceId": deviceId}
@trustedDeviceRouter.delete("")
@limiter.limit("10/minute")
def revokeAllTrustedDevices(
request: Request,
userId: str = Query(..., description="User ID whose trusted devices to revoke"),
currentUser: User = Depends(getCurrentUser),
) -> Dict[str, Any]:
"""Revoke ALL trusted devices for a user (force MFA on next login)."""
_requireAdmin(currentUser)
rootInterface = getRootInterface()
from modules.auth.trustedDeviceService import revokeTrustedDevices
count = revokeTrustedDevices(userId, rootInterface.db)
logger.info("Admin %s revoked all trusted devices for userId=%s (%d)", currentUser.username, userId, count)
return {"revoked": count, "userId": userId}

248
modules/routes/routeBilling.py
View file

@ -290,19 +290,6 @@ class MandateBalanceResponse(BaseModel):
warningThresholdPercent: float
class UserBalanceResponse(BaseModel):
"""User-level balance summary."""
accountId: str
mandateId: str
mandateName: str
userId: str
userName: str
balance: float
warningThreshold: float
isWarning: bool
enabled: bool
class UserTransactionResponse(BaseModel):
"""User-level transaction with user context."""
id: str
@ -429,10 +416,6 @@ def _normalize_billing_tx_dict(t: Dict[str, Any]) -> Dict[str, Any]:
return r
def _load_billing_user_transactions_normalized(billingService) -> List[Dict[str, Any]]:
raw = billingService.getTransactionHistory(limit=5000)
return [_normalize_billing_tx_dict(t) for t in raw]
def _view_user_transactions_filtered_list(
billing_interface,
@ -464,147 +447,6 @@ def _view_user_transactions_filtered_list(
return all_items
@router.get("/transactions")
@limiter.limit("30/minute")
def getTransactions(
request: Request,
limit: int = Query(default=50, ge=1, le=500),
offset: int = Query(default=0, ge=0),
pagination: Optional[str] = Query(
None,
description="JSON PaginationParams for table UI (filters, sort, viewKey, groupByLevels).",
),
mode: Optional[str] = Query(None, description="'filterValues' | 'ids' with pagination"),
column: Optional[str] = Query(None, description="Column for mode=filterValues"),
ctx: RequestContext = Depends(getRequestContext),
):
"""
Get transaction history across all mandates the user belongs to.
Without ``pagination`` query: legacy behaviour — returns a JSON array of
transactions (`limit`/`offset` window).
With ``pagination`` JSON: returns ``{ items, pagination, groupLayout?, appliedView? }``.
Table list views use contextKey ``billing/transactions``.
"""
try:
billingService = getBillingService(
ctx.user,
ctx.mandateId,
featureCode="billing",
)
if pagination:
from modules.interfaces.interfaceTableHelpers import (
applyViewToParams,
buildGroupLayout,
effective_group_by_levels,
resolveView,
)
from modules.dbHelpers.paginationHelpers import (
handleFilterValuesInMemory,
handleIdsInMemory,
)
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
from modules.interfaces.interfaceDbManagement import ComponentObjects
CONTEXT_KEY = "billing/transactions"
try:
paginationDict = json.loads(pagination)
if not paginationDict:
raise ValueError("empty pagination")
paginationDict = normalize_pagination_dict(paginationDict)
paginationParams = PaginationParams(**paginationDict)
except (json.JSONDecodeError, ValueError, TypeError) as e:
raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}")
appInterface = getAppInterface(ctx.user)
viewKey = paginationParams.viewKey
viewConfig, viewDisplayName = resolveView(appInterface, CONTEXT_KEY, viewKey)
viewMeta = AppliedViewMeta(viewKey=viewKey, displayName=viewDisplayName) if viewKey else None
paginationParams = applyViewToParams(paginationParams, viewConfig)
groupByLevels = effective_group_by_levels(paginationParams, viewConfig)
all_items = _load_billing_user_transactions_normalized(billingService)
if mode == "filterValues":
if not column:
raise HTTPException(status_code=400, detail="column parameter required for mode=filterValues")
return handleFilterValuesInMemory(all_items, column, pagination)
if mode == "ids":
return handleIdsInMemory(all_items, pagination)
comp = ComponentObjects()
comp.setUserContext(ctx.user)
if paginationParams.filters:
all_items = comp._applyFilters(all_items, paginationParams.filters)
if paginationParams.sort:
all_items = comp._applySorting(all_items, paginationParams.sort)
totalItems = len(all_items)
totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0
if not groupByLevels:
pstart = (paginationParams.page - 1) * paginationParams.pageSize
page_items = all_items[pstart : pstart + paginationParams.pageSize]
group_layout = None
else:
page_items, group_layout = buildGroupLayout(
all_items,
groupByLevels,
paginationParams.page,
paginationParams.pageSize,
)
resp: Dict[str, Any] = {
"items": page_items,
"pagination": PaginationMetadata(
currentPage=paginationParams.page,
pageSize=paginationParams.pageSize,
totalItems=totalItems,
totalPages=totalPages,
sort=paginationParams.sort,
filters=paginationParams.filters,
).model_dump(),
}
if group_layout:
resp["groupLayout"] = group_layout.model_dump()
if viewMeta:
resp["appliedView"] = viewMeta.model_dump()
return JSONResponse(content=resp)
transactions = billingService.getTransactionHistory(limit=offset + limit)
result: List[TransactionResponse] = []
for t in transactions[offset : offset + limit]:
result.append(
TransactionResponse(
id=t.get("id"),
accountId=t.get("accountId"),
transactionType=TransactionTypeEnum(t.get("transactionType", "DEBIT")),
amount=t.get("amount", 0.0),
description=t.get("description", ""),
referenceType=ReferenceTypeEnum(t["referenceType"]) if t.get("referenceType") else None,
workflowId=t.get("workflowId"),
featureCode=t.get("featureCode"),
featureInstanceId=t.get("featureInstanceId"),
aicoreProvider=t.get("aicoreProvider"),
aicoreModel=t.get("aicoreModel"),
createdByUserId=t.get("createdByUserId"),
sysCreatedAt=t.get("sysCreatedAt"),
mandateId=t.get("mandateId"),
mandateName=t.get("mandateName"),
)
)
return result
except HTTPException:
raise
except Exception as e:
logger.error(f"Error getting billing transactions: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/statistics", response_model=UsageReportResponse)
@limiter.limit("30/minute")
@ -756,7 +598,6 @@ def createOrUpdateSettings(
return result or existingSettings
return existingSettings
else:
from modules.datamodels.datamodelBilling import BillingSettings
newSettings = BillingSettings(
mandateId=targetMandateId,
@ -821,7 +662,6 @@ def addCredit(
if creditRequest.amount == 0:
raise HTTPException(status_code=400, detail=routeApiMsg("Amount must not be zero"))
from modules.datamodels.datamodelBilling import BillingTransaction
isDeduction = creditRequest.amount < 0
txType = TransactionTypeEnum.DEBIT if isDeduction else TransactionTypeEnum.CREDIT
@ -1375,51 +1215,6 @@ def getMandateViewTransactions(
# User View Endpoints (RBAC-based)
# =============================================================================
@router.get("/view/users/balances", response_model=List[UserBalanceResponse])
@limiter.limit("30/minute")
def getUserViewBalances(
request: Request,
ctx: RequestContext = Depends(getRequestContext)
):
"""
Get user-level balances.
RBAC filtering:
- SysAdmin: sees all user balances across all mandates
- Mandate-Admin: sees user balances for mandates they administrate
- Regular user: sees only their own balances
"""
try:
billingInterface = getBillingInterface(ctx.user, ctx.mandateId)
# Evaluate RBAC scope
scope = _getBillingDataScope(ctx.user)
# Determine mandate IDs for data loading
if scope.isGlobalAdmin:
mandateIds = None
else:
mandateIds = scope.adminMandateIds + scope.memberMandateIds
if not mandateIds:
return []
allBalances = billingInterface.getUserBalancesForMandates(mandateIds)
# RBAC filter: mandate admins see all in their mandates, regular users only own
if not scope.isGlobalAdmin:
adminMandateSet = set(scope.adminMandateIds)
allBalances = [
b for b in allBalances
if b.get("mandateId") in adminMandateSet or b.get("userId") == scope.userId
]
return [UserBalanceResponse(**b) for b in allBalances]
except Exception as e:
logger.error(f"Error getting user view balances: {e}")
raise HTTPException(status_code=500, detail=str(e))
class ViewStatisticsResponse(BaseModel):
"""Aggregated statistics across all user's mandates."""
totalCost: float = 0.0
@ -1732,25 +1527,48 @@ def getUserViewTransactions(
resp["appliedView"] = viewMeta.model_dump(mode="json")
return JSONResponse(content=resp)
result = billingInterface.getTransactionsForMandatesPaginated(
mandateIds=loadMandateIds,
pagination=paginationParams,
scope=effectiveScope,
userId=personalUserId,
_ENRICHED_FILTER_COLS = {"mandateName", "userName", "mandateId", "userId"}
_hasEnrichedFilters = paginationParams.filters and any(
k in _ENRICHED_FILTER_COLS for k in paginationParams.filters
)
if _hasEnrichedFilters:
all_items = _view_user_transactions_filtered_list(
billingInterface,
loadMandateIds,
effectiveScope,
personalUserId,
paginationParams,
ctx.user,
)
totalItems = len(all_items)
totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0
pstart = (paginationParams.page - 1) * paginationParams.pageSize
page_items = all_items[pstart : pstart + paginationParams.pageSize]
else:
result = billingInterface.getTransactionsForMandatesPaginated(
mandateIds=loadMandateIds,
pagination=paginationParams,
scope=effectiveScope,
userId=personalUserId,
)
page_items = result.items
totalItems = result.totalItems
totalPages = result.totalPages
logger.debug(
f"SQL-paginated {result.totalItems} transactions for user {ctx.user.id} "
f"(scope={scope}, mandateId={mandateId}, page={paginationParams.page})"
f"Paginated {totalItems} transactions for user {ctx.user.id} "
f"(scope={scope}, mandateId={mandateId}, page={paginationParams.page}, "
f"enrichedFilter={_hasEnrichedFilters})"
)
return PaginatedResponse(
items=[_toResponse(d) for d in result.items],
items=[_toResponse(d) for d in page_items],
pagination=PaginationMetadata(
currentPage=paginationParams.page,
pageSize=paginationParams.pageSize,
totalItems=result.totalItems,
totalPages=result.totalPages,
totalItems=totalItems,
totalPages=totalPages,
sort=paginationParams.sort,
filters=paginationParams.filters,
),

7
modules/routes/routeDataConnections.py
View file

@ -24,8 +24,8 @@ from modules.datamodels.datamodelSecurity import Token
from modules.auth import getCurrentUser, limiter
from modules.auth.oauthConnectTicket import issue_connect_ticket
from modules.auth.tokenRefreshService import token_refresh_service
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
from modules.interfaces.interfaceDbApp import getInterface
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta
from modules.interfaces.interfaceDbApp import getInterface, getRootInterface
from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp
from modules.interfaces.interfaceDbManagement import ComponentObjects
from modules.shared.i18nRegistry import apiRouteContext
@ -161,7 +161,6 @@ async def get_connections(
from modules.interfaces.interfaceTableHelpers import (
resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels,
)
from modules.datamodels.datamodelPagination import AppliedViewMeta
CONTEXT_KEY = "connections"
@ -782,7 +781,6 @@ async def _updateKnowledgeConsent(
if not connection:
raise HTTPException(status_code=404, detail=routeApiMsg("Connection not found"))
from modules.interfaces.interfaceDbApp import getRootInterface
rootIf = getRootInterface()
rootIf.db.recordModify(UserConnection, connectionId, {"knowledgeIngestionEnabled": enabled})
@ -861,7 +859,6 @@ def _updateKnowledgePreferences(
cleaned = {k: v for k, v in preferences.items() if k in _ALLOWED_KEYS}
merged = {**existing, **cleaned, "schemaVersion": 1}
from modules.interfaces.interfaceDbApp import getRootInterface
getRootInterface().db.recordModify(UserConnection, connectionId, {"knowledgePreferences": merged})
logger.info("Knowledge preferences updated for connection %s", connectionId)

262
modules/routes/routeDataFiles.py
View file

@ -15,11 +15,11 @@ import zipfile
from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext
# Import interfaces
from modules.interfaces import interfaceDbManagement
from modules.interfaces import interfaceDbManagement, interfaceDbKnowledge
from modules.datamodels.datamodelFiles import FileItem, FilePreview, FileFolder
from modules.shared.attributeUtils import getModelAttributeDefinitions
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta
from modules.shared.i18nRegistry import apiRouteContext
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
routeApiMsg = apiRouteContext("routeDataFiles")
@ -311,72 +311,6 @@ def get_folder_tree(
raise HTTPException(status_code=500, detail=str(e))
@router.post("/attributes")
@limiter.limit("120/minute")
def getAttributesForIds(
request: Request,
body: Dict[str, Any] = Body(...),
currentUser: User = Depends(getCurrentUser),
context: RequestContext = Depends(getRequestContext),
):
"""Return current attribute values (neutralize, scope, ragIndexEnabled) for
a list of node IDs. For folder IDs, computes 'mixed' by checking direct
children. The frontend sends this after every toggle to refresh visible
nodes without reloading the tree structure."""
ids = body.get("ids", [])
if not isinstance(ids, list) or len(ids) == 0:
return {}
if len(ids) > 500:
raise HTTPException(status_code=400, detail="Max 500 IDs per request")
try:
managementInterface = interfaceDbManagement.getInterface(
currentUser,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
db = managementInterface.db
userId = str(currentUser.id)
allFolders = db.getRecordset(FileFolder, recordFilter={"sysCreatedBy": userId}) or []
allFiles = db.getRecordset(FileItem, recordFilter={"sysCreatedBy": userId}) or []
folderById = {f["id"]: f for f in allFolders}
fileById = {f["id"]: f for f in allFiles}
logger.info(
"getAttributesForIds: %d ids requested, %d folders found, %d files found",
len(ids), len(allFolders), len(allFiles),
)
result: Dict[str, Dict[str, Any]] = {}
for nodeId in ids:
if nodeId.startswith("__filesRoot:"):
attrs = _computeSyntheticRootAttrs(allFolders, allFiles)
result[nodeId] = attrs
elif nodeId in folderById:
folder = folderById[nodeId]
attrs = _computeFolderAttrs(folder, allFolders, allFiles)
result[nodeId] = attrs
elif nodeId in fileById:
f = fileById[nodeId]
result[nodeId] = {
"neutralize": bool(f.get("neutralize", False)),
"scope": f.get("scope", "personal"),
}
else:
logger.debug("getAttributesForIds: unknown id=%s", nodeId)
logger.info("getAttributesForIds: returning %d entries", len(result))
return result
except HTTPException:
raise
except Exception as e:
logger.error(f"getAttributesForIds error: {e}")
raise HTTPException(status_code=500, detail=str(e))
def _enrichFoldersWithMixed(
db, userId: str, folders: List[Dict[str, Any]], ownerMode: str,
) -> None:
@ -480,43 +414,6 @@ def _effectiveScope(
return childVals.pop()
def _computeSyntheticRootAttrs(
allFolders: List[Dict[str, Any]],
allFiles: List[Dict[str, Any]],
) -> Dict[str, Any]:
"""Compute attributes for the synthetic root by recursively checking the
entire tree. If ANY item at any depth diverges, root shows 'mixed'."""
topFolders = [f for f in allFolders if not f.get("parentId")]
topFiles = [f for f in allFiles if not f.get("folderId")]
neutralizeVals = set()
scopeVals = set()
for cf in topFolders:
nEff = _effectiveNeutralize(cf["id"], allFolders, allFiles)
if nEff == "mixed":
neutralizeVals.add(True)
neutralizeVals.add(False)
else:
neutralizeVals.add(nEff)
sEff = _effectiveScope(cf["id"], allFolders, allFiles)
if sEff == "mixed":
scopeVals.add("__mixed_a__")
scopeVals.add("__mixed_b__")
else:
scopeVals.add(sEff)
for cf in topFiles:
neutralizeVals.add(bool(cf.get("neutralize", False)))
scopeVals.add(cf.get("scope", "personal"))
if not neutralizeVals and not scopeVals:
return {"neutralize": False, "scope": "personal"}
return {
"neutralize": "mixed" if len(neutralizeVals) > 1 else (neutralizeVals.pop() if neutralizeVals else False),
"scope": "mixed" if len(scopeVals) > 1 else (scopeVals.pop() if scopeVals else "personal"),
}
@router.post("/folders", status_code=status.HTTP_201_CREATED)
@limiter.limit("30/minute")
def create_folder(
@ -700,6 +597,7 @@ def get_files(
pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"),
mode: Optional[str] = Query(None, description="'filterValues' for distinct column values, 'ids' for all filtered IDs"),
column: Optional[str] = Query(None, description="Column key (required when mode=filterValues)"),
owner: Optional[str] = Query(None, description="'me' for own files, 'shared' for files from others"),
currentUser: User = Depends(getCurrentUser),
context: RequestContext = Depends(getRequestContext)
):
@ -737,7 +635,6 @@ def get_files(
resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels,
)
import modules.interfaces.interfaceDbApp as _appIface
from modules.datamodels.datamodelPagination import AppliedViewMeta
managementInterface = interfaceDbManagement.getInterface(
currentUser,
@ -756,6 +653,21 @@ def get_files(
def _filesToDicts(fileItems):
return [f.model_dump() if hasattr(f, "model_dump") else (dict(f) if not isinstance(f, dict) else f) for f in fileItems]
ownerRecordFilter = None
ownerExcludeOwnFiles = False
ownerNorm = (owner or "").strip().lower()
if ownerNorm == "me":
ownerRecordFilter = {"sysCreatedBy": managementInterface.userId}
elif ownerNorm == "shared":
ownerExcludeOwnFiles = True
def _applyOwnerFilter(items):
"""Post-filter for owner=shared: exclude files created by current user."""
if not ownerExcludeOwnFiles:
return items
uid = managementInterface.userId
return [f for f in items if (f.get("sysCreatedBy") if isinstance(f, dict) else getattr(f, "sysCreatedBy", None)) != uid]
if mode == "groupSummary":
if not pagination:
raise HTTPException(status_code=400, detail="pagination required for groupSummary")
@ -794,10 +706,12 @@ def get_files(
return handleIdsMode(managementInterface.db, FileItem, pagination, recordFilter)
if not groupByLevels:
# No grouping: let DB handle pagination directly (fastest path)
result = managementInterface.getAllFiles(pagination=paginationParams)
result = managementInterface.getAllFiles(
pagination=paginationParams,
recordFilter=ownerRecordFilter,
)
if paginationParams and hasattr(result, 'items'):
enriched = enrichRowsWithFkLabels(_filesToDicts(result.items), FileItem, db=appInterface.db)
enriched = _applyOwnerFilter(enrichRowsWithFkLabels(_filesToDicts(result.items), FileItem, db=appInterface.db))
resp: dict = {
"items": enriched,
"pagination": PaginationMetadata(
@ -811,7 +725,8 @@ def get_files(
}
else:
items = result if isinstance(result, list) else (result.items if hasattr(result, "items") else [result])
resp = {"items": enrichRowsWithFkLabels(_filesToDicts(items), FileItem, db=appInterface.db), "pagination": None}
enriched = _applyOwnerFilter(enrichRowsWithFkLabels(_filesToDicts(items), FileItem, db=appInterface.db))
resp = {"items": enriched, "pagination": None}
if viewMeta:
resp["appliedView"] = viewMeta.model_dump()
return resp
@ -1115,133 +1030,6 @@ def batchDownload(
raise HTTPException(status_code=500, detail=str(e))
# ── Bulk file operations (replace former group-based bulk routes) ─────────────
@router.post("/bulk/scope")
@limiter.limit("30/minute")
def bulk_set_scope(
request: Request,
body: dict = Body(...),
currentUser: User = Depends(getCurrentUser),
context: RequestContext = Depends(getRequestContext),
):
"""Set scope for a list of files by their IDs."""
fileIds: list = body.get("fileIds") or []
scope: str = body.get("scope") or ""
if not fileIds:
raise HTTPException(status_code=400, detail="fileIds is required")
validScopes = {"personal", "featureInstance", "mandate", "global"}
if scope not in validScopes:
raise HTTPException(status_code=400, detail=f"Invalid scope. Must be one of {validScopes}")
if scope == "global" and not context.isSysAdmin:
raise HTTPException(status_code=403, detail="Only sysadmins can set global scope")
try:
managementInterface = interfaceDbManagement.getInterface(
currentUser,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
updated = 0
for fid in fileIds:
try:
managementInterface.updateFile(fid, {"scope": scope})
updated += 1
except Exception as e:
logger.error(f"bulk_set_scope: failed for file {fid}: {e}")
return {"scope": scope, "filesUpdated": updated}
except HTTPException:
raise
except Exception as e:
logger.error(f"bulk_set_scope error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/bulk/neutralize")
@limiter.limit("30/minute")
def bulk_set_neutralize(
request: Request,
body: dict = Body(...),
currentUser: User = Depends(getCurrentUser),
context: RequestContext = Depends(getRequestContext),
):
"""Set neutralize flag for a list of files by their IDs (incl. knowledge purge/reindex)."""
fileIds: list = body.get("fileIds") or []
neutralize = body.get("neutralize")
if not fileIds:
raise HTTPException(status_code=400, detail="fileIds is required")
if neutralize is None:
raise HTTPException(status_code=400, detail="neutralize is required")
try:
managementInterface = interfaceDbManagement.getInterface(
currentUser,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
updated = 0
for fid in fileIds:
try:
managementInterface.updateFile(fid, {"neutralize": neutralize})
if not neutralize:
try:
from modules.interfaces import interfaceDbKnowledge
kIface = interfaceDbKnowledge.getInterface(currentUser)
kIface.purgeFileKnowledge(fid)
except Exception as ke:
logger.warning(f"bulk_set_neutralize: knowledge purge failed for {fid}: {ke}")
updated += 1
except Exception as e:
logger.error(f"bulk_set_neutralize: failed for file {fid}: {e}")
return {"neutralize": neutralize, "filesUpdated": updated}
except HTTPException:
raise
except Exception as e:
logger.error(f"bulk_set_neutralize error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/bulk/download-zip")
@limiter.limit("10/minute")
async def bulk_download_zip(
request: Request,
body: dict = Body(...),
currentUser: User = Depends(getCurrentUser),
context: RequestContext = Depends(getRequestContext),
):
"""Download a list of files as a ZIP archive."""
fileIds: list = body.get("fileIds") or []
if not fileIds:
raise HTTPException(status_code=400, detail="fileIds is required")
try:
managementInterface = interfaceDbManagement.getInterface(
currentUser,
mandateId=str(context.mandateId) if context.mandateId else None,
featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None,
)
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
for fid in fileIds:
try:
fileMeta = managementInterface.getFile(fid)
fileData = managementInterface.getFileData(fid)
if fileMeta and fileData:
name = (getattr(fileMeta, "fileName", None) or fid)
zf.writestr(name, fileData)
except Exception as fe:
logger.warning(f"bulk_download_zip: skipping file {fid}: {fe}")
buf.seek(0)
from fastapi.responses import StreamingResponse
return StreamingResponse(
buf,
media_type="application/zip",
headers={"Content-Disposition": 'attachment; filename="files.zip"'},
)
except HTTPException:
raise
except Exception as e:
logger.error(f"bulk_download_zip error: {e}")
raise HTTPException(status_code=500, detail=str(e))
# ── Scope & neutralize tagging endpoints (before /{fileId} catch-all) ─────────
@router.patch("/{fileId}/scope")

3
modules/routes/routeDataPrompts.py
View file

@ -16,7 +16,7 @@ from modules.auth import limiter, getCurrentUser
from modules.interfaces import interfaceDbManagement
from modules.datamodels.datamodelUtils import Prompt
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict, AppliedViewMeta
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeDataPrompts")
@ -55,7 +55,6 @@ def get_prompts(
resolveView, applyViewToParams, buildGroupLayout, effective_group_by_levels,
)
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
from modules.datamodels.datamodelPagination import AppliedViewMeta
CONTEXT_KEY = "prompts"

32
modules/routes/routeJobs.py
View file

@ -4,7 +4,6 @@
Endpoints:
- GET /api/jobs/{jobId} -> single job status
- GET /api/jobs -> list (filter by jobType, instanceId)
Access control: a caller may read a job iff they are a member of its mandate
(or PlatformAdmin). Jobs without a mandateId (system-wide) are restricted to
@ -12,14 +11,13 @@ PlatformAdmin only.
"""
import logging
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Optional
from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request
from fastapi import APIRouter, Depends, HTTPException, Path, Request
from modules.auth import getRequestContext, RequestContext, limiter
from modules.serviceCenter.services.serviceBackgroundJobs import (
getJobStatus,
listJobs,
)
from modules.shared.i18nRegistry import apiRouteContext, resolveJobMessage
@ -91,29 +89,3 @@ def get_job(
return _serialiseJob(job)
@router.get("")
@limiter.limit("30/minute")
def list_jobs(
request: Request,
jobType: Optional[str] = Query(None),
mandateId: Optional[str] = Query(None),
instanceId: Optional[str] = Query(None, description="Feature instance scope"),
limit: int = Query(20, ge=1, le=100),
context: RequestContext = Depends(getRequestContext),
) -> Dict[str, List[Dict[str, Any]]]:
"""List recent jobs filtered by scope. Newest first."""
if mandateId is None:
if not context or not context.isPlatformAdmin:
raise HTTPException(
status_code=400,
detail=routeApiMsg("mandateId is required (only PlatformAdmin may list system-wide)"),
)
elif not _userHasMandateAccess(context, mandateId):
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
jobs = listJobs(
mandateId=mandateId,
featureInstanceId=instanceId,
jobType=jobType,
limit=limit,
)
return {"items": [_serialiseJob(j) for j in jobs]}

25
modules/routes/routeMfa.py
View file

@ -26,13 +26,14 @@ from modules.auth import (
setAccessTokenCookie,
setRefreshTokenCookie,
)
from modules.auth.homeMandateService import ensureHomeMandate
from modules.auth.mfaService import (
generateSetup,
confirmSetup,
verifyCode,
isMfaRequired,
)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.interfaces.interfaceDbApp import getRootInterface, getInterface
from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, Mandate
from modules.datamodels.datamodelSecurity import Token, TokenPurpose
from modules.shared.configuration import APP_CONFIG
@ -215,7 +216,6 @@ def mfaVerify(
jti = jwt.decode(accessToken, SECRET_KEY, algorithms=[ALGORITHM]).get("jti")
from modules.interfaces.interfaceDbApp import getInterface
user = User.model_validate(userRecord)
userInterface = getInterface(user)
dbToken = Token(
@ -230,8 +230,29 @@ def mfaVerify(
)
userInterface.saveAccessToken(dbToken)
try:
ensureHomeMandate(rootInterface, user)
except Exception as homeErr:
logger.error(f"Error ensuring Home mandate for user {username}: {homeErr}")
try:
activatedCount = rootInterface._activatePendingSubscriptions(userId)
if activatedCount > 0:
logger.info(
f"Activated {activatedCount} pending subscription(s) for user {username} after MFA"
)
except Exception as subErr:
logger.error(f"Error activating subscriptions after MFA verify: {subErr}")
logger.info("MFA verify successful for user %s", username)
# Mark device as trusted so MFA is skipped on next login from this device
try:
from modules.auth.trustedDeviceService import createTrustedDevice
createTrustedDevice(userId, request, response, rootInterface.db)
except Exception as e:
logger.warning(f"Failed to create trusted device after MFA verify: {e}")
try:
from modules.dbHelpers.auditLogger import audit_logger
audit_logger.logUserAccess(

1
modules/routes/routeNotifications.py
View file

@ -411,7 +411,6 @@ def _handleInvitationAction(
) -> str:
"""Handle accept/decline actions for invitation notifications."""
from modules.datamodels.datamodelInvitation import Invitation
from modules.datamodels.datamodelUam import Mandate
from modules.datamodels.datamodelMembership import UserMandate
invitationId = notification.referenceId

3
modules/routes/routeRagInventory.py
View file

@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Depends, Request
from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelUam import User, UserConnection
from modules.shared.i18nRegistry import apiRouteContext, resolveJobMessage
routeApiMsg = apiRouteContext("routeRagInventory")
@ -485,7 +485,6 @@ def _getInventoryPlatform(
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface
from modules.serviceCenter.services.serviceBackgroundJobs import mainBackgroundJobService as jobService
from modules.datamodels.datamodelUam import UserConnection
rootIf = getRootInterface()
knowledgeIf = getKnowledgeInterface(None)

6
modules/routes/routeSecurityGoogle.py
View file

@ -34,6 +34,7 @@ from modules.auth import (
)
from modules.auth.tokenManager import TokenManager
from modules.auth.oauthProviderConfig import googleAuthScopes, googleDataScopes
from modules.auth.homeMandateService import ensureHomeMandate
from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeSecurityGoogle")
@ -278,6 +279,11 @@ async def auth_login_callback(
)
# --- end MFA gate -----------------------------------------------------
try:
ensureHomeMandate(rootInterface, user)
except Exception as homeErr:
logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}")
jwt_token_data = {
"sub": user.username,
"userId": str(user.id),

93
modules/routes/routeSecurityLocal.py
View file

@ -22,6 +22,7 @@ from modules.datamodels.datamodelSecurity import Token, TokenPurpose
from modules.shared.configuration import APP_CONFIG
from modules.shared.timeUtils import getUtcTimestamp
from modules.shared.i18nRegistry import apiRouteContext
from modules.auth.homeMandateService import ensureHomeMandate
routeApiMsg = apiRouteContext("routeSecurityLocal")
# Configure logger
@ -70,7 +71,6 @@ def buildAuthEmailHtml(
operatorLine = ""
try:
from modules.shared.configuration import APP_CONFIG
parts = [p for p in [
APP_CONFIG.get("Operator_CompanyName", ""),
APP_CONFIG.get("Operator_Address", ""),
@ -175,50 +175,6 @@ router = APIRouter(
}
)
def _ensureHomeMandate(rootInterface, user) -> None:
"""Ensure user has a Home mandate, but only if they have no mandate memberships
AND no pending invitations.
Invited users should NOT get a Home mandate — they join existing mandates via
invitation acceptance and can create their own later via onboarding.
"""
userId = str(user.id)
userMandates = rootInterface.getUserMandates(userId)
if userMandates:
for um in userMandates:
mandate = rootInterface.getMandate(um.mandateId)
if mandate and (mandate.name or "").startswith("Home ") and not mandate.isSystem:
return
logger.debug(f"User {user.username} has {len(userMandates)} mandate(s) but no Home — skipping auto-creation")
return
try:
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIf
appIf = _getRootIf()
normalizedEmail = (user.email or "").strip().lower() if user.email else None
pendingByUsername = appIf.getInvitationsByTargetUsername(user.username)
pendingByEmail = appIf.getInvitationsByEmail(normalizedEmail) if normalizedEmail else []
seenIds = set()
for inv in pendingByUsername + pendingByEmail:
if inv.id in seenIds:
continue
seenIds.add(inv.id)
if not inv.revokedAt and (inv.currentUses or 0) < (inv.maxUses or 1):
logger.info(f"User {user.username} has pending invitation(s) — skipping Home mandate creation")
return
except Exception as e:
logger.warning(f"Could not check pending invitations for {user.username}: {e}")
homeMandateLabel = f"Home {user.username}"
rootInterface._provisionMandateForUser(
userId=userId,
mandateLabel=homeMandateLabel,
planKey="TRIAL_14D",
)
logger.info(f"Created Home mandate '{homeMandateLabel}' for user {user.username}")
@router.post("/login")
@limiter.limit("30/minute")
def login(
@ -256,6 +212,7 @@ def login(
# --- MFA gate --------------------------------------------------------
from modules.auth.mfaService import isMfaRequired as _isMfaRequired
from modules.auth.trustedDeviceService import isTrustedDevice as _isTrustedDevice
from modules.routes.routeMfa import createMfaPendingToken
userRecord = rootInterface._getUserForAuthentication(user.username)
@ -273,7 +230,14 @@ def login(
mfaRequired = _isMfaRequired(user, userMandates=userMandates, mandates=mandateObjs)
hasMfaSetup = bool(userRecord and userRecord.get("mfaSecret") and getattr(user, "mfaEnabled", False))
# Trusted device: skip MFA if the device was previously verified
_deviceTrusted = False
if mfaRequired or hasMfaSetup:
_deviceTrusted = _isTrustedDevice(request, str(user.id), rootInterface.db)
if _deviceTrusted:
logger.info(f"MFA skipped for user {user.username} (trusted device)")
if (mfaRequired or hasMfaSetup) and not _deviceTrusted:
_sid = str(uuid.uuid4())
pendingToken = createMfaPendingToken(
userId=str(user.id),
@ -358,7 +322,7 @@ def login(
# Ensure user has a Home mandate (created on first login if missing)
try:
_ensureHomeMandate(rootInterface, user)
ensureHomeMandate(rootInterface, user)
except Exception as homeErr:
logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}")
@ -659,31 +623,46 @@ def refresh_token(
logger.error(f"Failed to get user from database: {str(e)}")
raise HTTPException(status_code=500, detail=routeApiMsg("Failed to validate user"))
# Preserve sessionId from the refresh token so the session stays grouped
sessionId = payload.get("sid") or str(uuid.uuid4())
# Create new token data
# MULTI-TENANT: Token does NOT contain mandateId anymore
newJti = str(uuid.uuid4())
token_data = {
"sub": current_user.username,
"userId": str(current_user.id),
"authenticationAuthority": current_user.authenticationAuthority
# NO mandateId in token
"authenticationAuthority": current_user.authenticationAuthority,
"jti": newJti,
"sid": sessionId,
}
# Create new access token + set cookie
access_token, _expires = createAccessToken(token_data)
access_token, accessExpires = createAccessToken(token_data)
setAccessTokenCookie(response, access_token)
# Get expiration time
# Persist the new token in DB so _getUserBase() accepts it
authority = current_user.authenticationAuthority
if isinstance(authority, str):
authority = AuthAuthority(authority)
dbToken = Token(
id=newJti,
userId=str(current_user.id),
authority=authority,
tokenAccess=access_token,
tokenPurpose=TokenPurpose.AUTH_SESSION,
expiresAt=accessExpires.timestamp(),
sessionId=sessionId,
)
try:
payload = jwt.decode(access_token, SECRET_KEY, algorithms=[ALGORITHM])
expires_at = datetime.fromtimestamp(payload.get("exp"))
userInterface = getInterface(current_user)
userInterface.saveAccessToken(dbToken)
except Exception as e:
logger.error(f"Failed to decode new access token: {str(e)}")
raise HTTPException(status_code=500, detail=routeApiMsg("Failed to create new token"))
logger.warning(f"Failed to persist refreshed token in DB: {e}")
return {
"type": "token_refresh_success",
"message": "Token refreshed successfully",
"expires_at": expires_at.isoformat()
"expires_at": accessExpires.isoformat()
}
except HTTPException as e:
@ -1035,7 +1014,6 @@ def _getNeutralizationMappings(
):
"""List the current user's neutralization placeholder mappings."""
userId = str(context.user.id)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes
rootIf = getRootInterface()
records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"userId": userId})
@ -1051,7 +1029,6 @@ def _deleteNeutralizationMapping(
):
"""Delete a specific neutralization mapping owned by the current user."""
userId = str(context.user.id)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes
rootIf = getRootInterface()
records = rootIf.db.getRecordset(DataNeutralizerAttributes, recordFilter={"id": mappingId})

76
modules/routes/routeSecurityMsft.py
View file

@ -34,7 +34,8 @@ from modules.auth import (
clearRefreshTokenCookie,
)
from modules.auth.tokenManager import TokenManager
from modules.auth.oauthProviderConfig import msftAuthScopes, msftDataScopes, msftDataScopesForRefresh
from modules.auth.oauthProviderConfig import msftDataScopes, msftDataScopesForRefresh, msftGraphDefaultScopes
from modules.auth.homeMandateService import ensureHomeMandate
from modules.shared.timeUtils import createExpirationTimestamp, getUtcTimestamp, parseTimestamp
from modules.shared.i18nRegistry import apiRouteContext
routeApiMsg = apiRouteContext("routeSecurityMsft")
@ -122,7 +123,7 @@ def auth_login(request: Request) -> RedirectResponse:
)
state_jwt = _issue_oauth_state({"flow": _FLOW_LOGIN})
auth_url = msal_app.get_authorization_request_url(
scopes=msftAuthScopes,
scopes=msftGraphDefaultScopes(),
redirect_uri=AUTH_REDIRECT_URI,
state=state_jwt,
prompt="select_account",
@ -154,7 +155,7 @@ async def auth_login_callback(
)
token_response = msal_app.acquire_token_by_authorization_code(
code,
scopes=msftAuthScopes,
scopes=msftGraphDefaultScopes(),
redirect_uri=AUTH_REDIRECT_URI,
)
if "error" in token_response:
@ -251,6 +252,20 @@ async def auth_login_callback(
)
# --- end MFA gate -----------------------------------------------------
try:
ensureHomeMandate(rootInterface, user)
except Exception as homeErr:
logger.error(f"Error ensuring Home mandate for user {user.username}: {homeErr}")
try:
activatedCount = rootInterface._activatePendingSubscriptions(str(user.id))
if activatedCount > 0:
logger.info(
f"Activated {activatedCount} pending subscription(s) for user {user.username}"
)
except Exception as subErr:
logger.error(f"Error activating subscriptions on Microsoft login: {subErr}")
jwt_token_data = {
"sub": user.username,
"userId": str(user.id),
@ -305,16 +320,22 @@ def auth_connect(
request: Request,
connectionId: str = Query(..., description="UserConnection id"),
connectTicket: str = Query(..., description="Short-lived ticket from POST /api/connections/{id}/connect"),
reauth: Optional[int] = Query(0, description="If 1, force the consent screen so newly added scopes are granted"),
reauth: Optional[int] = Query(0, description="If 1, force account re-selection (prompt=select_account). Never forces consent."),
) -> RedirectResponse:
"""Start Microsoft Data OAuth for an existing connection.
Authenticated via ``connectTicket`` (issued by POST connect) so the popup
works when the UI uses Bearer tokens in localStorage instead of cookies.
With ``reauth=1`` the consent screen is forced (``prompt=consent``) so the
user re-grants permissions and any newly added scopes (e.g. Calendars.Read,
Contacts.Read) actually land on the access token.
We never force ``prompt=consent``: with the Graph ``.default`` scope the
tenant's admin-consented permissions (incl. newly added scopes) are pulled
automatically. Forcing consent would re-trigger an interactive consent that
admin-restricted scopes (Sites.ReadWrite.All, Mail.Send, …) escalate to
"Need admin approval" for non-admin users. Genuinely missing permissions
instead surface as AADSTS65001, which routes to the admin-consent flow.
With ``reauth=1`` we only force account re-selection so the user can pick a
different account when refreshing a connection.
"""
try:
_require_msft_data_config()
@ -336,10 +357,16 @@ def auth_connect(
login_kwargs["domain_hint"] = login_hint.split("@", 1)[1]
login_kwargs["prompt"] = "login"
if reauth:
login_kwargs["prompt"] = "consent"
# Refreshing a connection: let the user re-pick the account, but never
# force consent — prompt=consent would escalate admin-restricted scopes
# to "Need admin approval" for non-admin users even though tenant-wide
# admin consent already covers them via the ".default" scope.
login_kwargs["prompt"] = "select_account"
# ".default" pulls exactly the tenant-consented Graph permissions and
# avoids re-triggering the admin-consent screen for external tenants.
auth_url = msal_app.get_authorization_request_url(
scopes=msftDataScopes,
scopes=msftGraphDefaultScopes(),
redirect_uri=DATA_REDIRECT_URI,
**login_kwargs,
)
@ -374,7 +401,7 @@ async def auth_connect_callback(
)
token_response = msal_app.acquire_token_by_authorization_code(
code,
scopes=msftDataScopes,
scopes=msftGraphDefaultScopes(),
redirect_uri=DATA_REDIRECT_URI,
)
if "error" in token_response:
@ -593,24 +620,17 @@ def adminconsent_callback(
status_code=400,
)
if not state:
logger.error("Admin consent success callback missing state")
return HTMLResponse(
content="""
<html>
<head><title>Admin Consent Failed</title></head>
<body>
<h1>Admin Consent Failed</h1>
<p>Parameter „state“ fehlt.</p>
</body>
</html>
""",
status_code=400,
)
state_data = _parse_oauth_state(state)
if state_data.get("flow") != "admin_consent":
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid OAuth flow for this callback"))
# When admin consent is initiated outside our /adminconsent route (e.g.
# the "Grant admin consent" button in the Azure portal, or a raw consent
# URL), Microsoft redirects without our state JWT. The consent itself is
# still recorded server-side, so we must not hard-fail — only validate the
# flow claim when a state is actually present.
if state:
state_data = _parse_oauth_state(state)
if state_data.get("flow") != "admin_consent":
raise HTTPException(status_code=400, detail=routeApiMsg("Invalid OAuth flow for this callback"))
else:
logger.warning("Admin consent callback without state — accepting (consent initiated outside our route)")
granted = str(admin_consent or "").strip().lower() in ("true", "1", "yes")
if not granted:

68
modules/routes/routeVoiceGoogle.py
View file

@ -14,10 +14,8 @@ import secrets
import time
from fastapi import APIRouter, Depends, HTTPException, Query, Request, WebSocket, WebSocketDisconnect
from typing import Optional, Dict, Any, List
from modules.auth import getCurrentUser, getRequestContext, RequestContext, limiter
from modules.datamodels.datamodelUam import User
from modules.auth import getRequestContext, RequestContext, limiter
from modules.interfaces.interfaceVoiceObjects import getVoiceInterface, VoiceObjects
from modules.shared.voiceCatalog import getCatalogPayload
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/voice-google", tags=["Voice Google"])
@ -49,70 +47,6 @@ class ConnectionManager:
manager = ConnectionManager()
def _getVoiceInterface(currentUser: User) -> VoiceObjects:
"""Get voice interface instance with user context."""
try:
return getVoiceInterface(currentUser)
except Exception as e:
logger.error(f"Failed to initialize voice interface: {e}")
raise HTTPException(
status_code=500,
detail=f"Failed to initialize voice interface: {str(e)}"
)
@router.get("/languages")
async def get_available_languages(currentUser: User = Depends(getCurrentUser)):
"""Return the curated voice/language catalog (single source of truth).
Each entry: {bcp47, iso, label, flag, defaultVoice}. Same payload as
/api/voice/languages — both endpoints back the same catalog.
"""
return {
"success": True,
"languages": getCatalogPayload(),
}
@router.get("/voices")
async def get_available_voices(
languageCode: Optional[str] = None,
language_code: Optional[str] = None, # Accept both camelCase and snake_case
currentUser: User = Depends(getCurrentUser)
):
"""
Get available voices from Google Cloud Text-to-Speech.
Accepts languageCode (camelCase) or language_code (snake_case) query parameter.
"""
# Use language_code if provided (frontend sends this), otherwise use languageCode
if language_code:
languageCode = language_code
try:
logger.info(f"🎤 Getting available voices, language filter: {languageCode}")
voiceInterface = _getVoiceInterface(currentUser)
result = await voiceInterface.getAvailableVoices(languageCode=languageCode)
if result["success"]:
return {
"success": True,
"voices": result["voices"],
"language_filter": languageCode
}
else:
raise HTTPException(
status_code=400,
detail=f"Failed to get voices: {result.get('error', 'Unknown error')}"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"❌ Get voices error: {e}")
raise HTTPException(
status_code=500,
detail=f"Failed to get available voices: {str(e)}"
)
# =========================================================================
# STT Streaming WebSocket — generic, used by all features
# =========================================================================

133
modules/routes/routeWorkflowAutomation.py
View file

@ -31,8 +31,8 @@ from modules.datamodels.datamodelPagination import PaginationParams, PaginationM
from modules.datamodels.datamodelWorkflowAutomation import (
AutoWorkflow, AutoVersion, AutoRun, AutoStepLog, AutoTask,
)
from modules.dbHelpers.paginationHelpers import applyFiltersAndSort, paginateInMemory
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.dbHelpers.paginationHelpers import applyFiltersAndSort, paginateInMemory, handleFilterValuesInMemory, handleIdsInMemory
from modules.interfaces.interfaceDbApp import getRootInterface, getRootInterface as _getRootIface
from modules.shared.i18nRegistry import apiRouteContext, resolveText
from modules.workflowAutomation.helpers import (
_getWorkflowAutomationDb,
@ -64,17 +64,17 @@ async def _listWorkflows(
mandateId: Optional[str] = Query(default=None),
):
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface
db = _getWorkflowAutomationDb()
try:
db._ensureTableExists(AutoWorkflow)
scopeFilter = _scopedWorkflowFilter(request)
if mandateId and scopeFilter is not None:
if mandateId not in (scopeFilter.get("mandateId") or []):
if mandateId:
scopeMandates = scopeFilter.get("mandateId") or []
if isinstance(scopeMandates, str):
scopeMandates = [scopeMandates]
if mandateId not in scopeMandates:
return {"items": [], "total": 0}
scopeFilter = {"mandateId": mandateId}
elif mandateId and scopeFilter is None:
scopeFilter = {"mandateId": mandateId}
params = _parsePaginationOr400(pagination)
records = db.getRecordset(AutoWorkflow, recordFilter=scopeFilter)
@ -173,18 +173,17 @@ async def _listRuns(
workflowId: Optional[str] = Query(default=None),
):
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface
db = _getWorkflowAutomationDb()
try:
db._ensureTableExists(AutoRun)
scopeFilter = _scopedRunFilter(request)
if mandateId:
if scopeFilter is None:
scopeFilter = {"mandateId": mandateId}
elif "mandateId" in scopeFilter:
if mandateId not in scopeFilter["mandateId"]:
return {"items": [], "total": 0}
scopeFilter = {"mandateId": mandateId}
if mandateId and scopeFilter and "mandateId" in scopeFilter:
scopeMandates = scopeFilter["mandateId"]
if isinstance(scopeMandates, str):
scopeMandates = [scopeMandates]
if mandateId not in scopeMandates:
return {"items": [], "total": 0}
scopeFilter = {"mandateId": mandateId}
if workflowId:
scopeFilter = {**(scopeFilter or {}), "workflowId": workflowId}
@ -208,27 +207,6 @@ async def _listRuns(
db.close()
@router.get("/runs/{runId}")
async def _getRun(
runId: str,
request: RequestContext = Depends(getRequestContext),
):
db = _getWorkflowAutomationDb()
try:
db._ensureTableExists(AutoRun)
run = db.getRecord(AutoRun, runId)
if not run:
raise HTTPException(status_code=404, detail="Run not found")
wfId = run.get("workflowId")
if wfId:
wf = db.getRecord(AutoWorkflow, wfId)
_validateWorkflowAccess(request, wf, "read")
return run
finally:
db.close()
# ---------------------------------------------------------------------------
# Tasks
# ---------------------------------------------------------------------------
@ -242,16 +220,21 @@ async def _listTasks(
db = _getWorkflowAutomationDb()
try:
db._ensureTableExists(AutoTask)
scopeFilter: Optional[Dict[str, Any]] = None
userId = str(request.user.id) if request.user else None
if not userId:
return {"items": [], "total": 0}
userMandateIds = _getUserMandateIds(userId)
if not userMandateIds:
return {"items": [], "total": 0}
if not request.isPlatformAdmin:
userId = str(request.user.id) if request.user else None
if not userId:
return {"items": [], "total": 0}
scopeFilter = {"assigneeId": userId}
wfFilter = {"mandateId": userMandateIds, "isTemplate": False}
db._ensureTableExists(AutoWorkflow)
wfs = db.getRecordset(AutoWorkflow, recordFilter=wfFilter) or []
wfIds = [w.get("id") for w in wfs]
scopeFilter: Dict[str, Any] = {"workflowId": wfIds} if wfIds else {"workflowId": "__none__"}
if status:
scopeFilter = {**(scopeFilter or {}), "status": status}
scopeFilter["status"] = status
params = _parsePaginationOr400(pagination)
records = db.getRecordset(AutoTask, recordFilter=scopeFilter)
@ -286,6 +269,27 @@ async def _listVersions(
db.close()
# ---------------------------------------------------------------------------
# Editor chat history (stub — persistence not yet implemented)
# ---------------------------------------------------------------------------
@router.get("/{workflowId}/chat/messages")
async def _getEditorChatMessages(
workflowId: str,
request: RequestContext = Depends(getRequestContext),
):
db = _getWorkflowAutomationDb()
try:
db._ensureTableExists(AutoWorkflow)
wf = db.getRecord(AutoWorkflow, workflowId)
if not wf:
raise HTTPException(status_code=404, detail="Workflow not found")
_validateWorkflowAccess(request, wf, "read")
return {"messages": []}
finally:
db.close()
# ---------------------------------------------------------------------------
# Step logs
# ---------------------------------------------------------------------------
@ -424,9 +428,11 @@ def _listTemplates(
raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required"))
userId = str(context.user.id)
userMandateIds = _getUserMandateIds(userId) if not context.isPlatformAdmin else []
userMandateIds = _getUserMandateIds(userId)
if mandateId and mandateId not in userMandateIds:
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
effectiveMandateId = mandateId or (userMandateIds[0] if userMandateIds else None)
if not effectiveMandateId and not context.isPlatformAdmin:
if not effectiveMandateId:
return {"templates": []}
instanceId = None
@ -447,17 +453,14 @@ def _listTemplates(
templates = iface.getTemplates(scope=scope)
from modules.dbHelpers.fkLabelResolver import enrichRowsWithFkLabels
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface
enrichRowsWithFkLabels(templates, AutoWorkflow, db=_getRootIface().db)
if mode == "filterValues":
if not column:
raise HTTPException(status_code=400, detail="column parameter required for mode=filterValues")
from modules.dbHelpers.paginationHelpers import handleFilterValuesInMemory
return handleFilterValuesInMemory(templates, column, pagination)
if mode == "ids":
from modules.dbHelpers.paginationHelpers import handleIdsInMemory
return handleIdsInMemory(templates, pagination)
paginationParams = None
@ -535,8 +538,10 @@ def _copyTemplate(
mandateId = body.get("mandateId") if isinstance(body, dict) else None
userId = str(context.user.id)
userMandateIds = _getUserMandateIds(userId)
if mandateId and mandateId not in userMandateIds:
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
if not mandateId:
userMandateIds = _getUserMandateIds(userId)
mandateId = userMandateIds[0] if userMandateIds else ""
db = _getWorkflowAutomationDb()
@ -577,8 +582,10 @@ def _shareTemplate(
mandateId = body.get("mandateId", "")
userId = str(context.user.id)
userMandateIds = _getUserMandateIds(userId)
if mandateId and mandateId not in userMandateIds:
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
if not mandateId:
userMandateIds = _getUserMandateIds(userId)
mandateId = userMandateIds[0] if userMandateIds else ""
db = _getWorkflowAutomationDb()
@ -984,14 +991,12 @@ def _getMetrics(
raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required"))
userId = str(context.user.id)
userMandateIds = _getUserMandateIds(userId) if not context.isPlatformAdmin else []
userMandateIds = _getUserMandateIds(userId)
if mandateId:
if not context.isPlatformAdmin and mandateId not in userMandateIds:
if mandateId not in userMandateIds:
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
scopeFilter: Dict[str, Any] = {"mandateId": mandateId, "isTemplate": False}
elif context.isPlatformAdmin:
scopeFilter = {"isTemplate": False}
elif userMandateIds:
scopeFilter = {"mandateId": userMandateIds, "isTemplate": False}
else:
@ -1001,13 +1006,21 @@ def _getMetrics(
"totalTokens": 0, "totalCredits": 0.0,
}
runScope = _scopedRunFilter(context)
if mandateId and runScope:
if "mandateId" in runScope:
if mandateId not in (runScope["mandateId"] if isinstance(runScope["mandateId"], list) else [runScope["mandateId"]]):
runScope = {"mandateId": "__none__"}
else:
runScope = {"mandateId": mandateId}
db = _getWorkflowAutomationDb()
try:
workflows = db.getRecordset(AutoWorkflow, recordFilter=scopeFilter) or [] if db._ensureTableExists(AutoWorkflow) else []
wfIds = [w.get("id") for w in workflows]
runFilter = {"workflowId": wfIds} if wfIds else {"workflowId": "__none__"}
runs = db.getRecordset(AutoRun, recordFilter=runFilter) or [] if db._ensureTableExists(AutoRun) else []
tasks = db.getRecordset(AutoTask, recordFilter=runFilter) or [] if db._ensureTableExists(AutoTask) else []
workflows = (db.getRecordset(AutoWorkflow, recordFilter=scopeFilter) or []) if db._ensureTableExists(AutoWorkflow) else []
runs = (db.getRecordset(AutoRun, recordFilter=runScope) or []) if db._ensureTableExists(AutoRun) else []
runIds = [r.get("id") for r in runs]
taskFilter = {"runId": runIds} if runIds else {"runId": "__none__"}
tasks = (db.getRecordset(AutoTask, recordFilter=taskFilter) or []) if db._ensureTableExists(AutoTask) else []
finally:
db.close()
@ -1289,7 +1302,6 @@ def _getRunDetail(
if tid:
try:
from modules.dbHelpers.fkLabelResolver import resolveInstanceLabels
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface
labelMap = resolveInstanceLabels(_getRootIface().db, [tid])
targetInstanceLabel = labelMap.get(tid)
except Exception:
@ -1386,7 +1398,6 @@ def _startEmailPollerIfNeeded(result: dict) -> None:
if not isinstance(result, dict) or result.get("waitReason") != "email":
return
try:
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.workflowAutomation.scheduler.emailPoller import ensureRunning
root = getRootInterface()
eventUser = root.getUserByUsername("event") if root else None

11
modules/serviceCenter/services/serviceAgent/mainServiceAgent.py
View file

@ -9,10 +9,10 @@ from modules.datamodels.datamodelAi import (
AiCallRequest, AiCallOptions, AiCallResponse, OperationTypeEnum
)
from modules.serviceCenter.services.serviceAgent.datamodelAgent import (
AgentConfig, AgentEvent, AgentEventTypeEnum
AgentConfig, AgentEvent, AgentEventTypeEnum, ToolDefinition, ToolResult
)
from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry
from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop
from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop, classifyToolResult
from modules.serviceCenter.services.serviceAgent.actionToolAdapter import ActionToolAdapter
from modules.serviceCenter.services.serviceAgent.coreTools import registerCoreTools
import json
@ -425,7 +425,6 @@ class AgentService:
activeToolNames.update(tb.tools)
from modules.serviceCenter.services.serviceAgent.externalToolRegistry import getExternalTools
from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolDefinition
for tb in activeToolboxes:
extDefs = getExternalTools(tb.id)
if not extDefs:
@ -459,7 +458,6 @@ class AgentService:
from modules.serviceCenter.services.serviceAgent.toolboxRegistry import (
getToolboxRegistry, buildRequestToolboxDefinition, REQUEST_TOOLBOX_TOOL_NAME,
)
from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult
tbRegistry = getToolboxRegistry()
allIds = [tb.id for tb in tbRegistry.getAllToolboxes()]
@ -488,7 +486,6 @@ class AgentService:
activatedCount += 1
continue
try:
from modules.serviceCenter.services.serviceAgent.coreTools import registerCoreTools
registerCoreTools(registry, self.services)
if registry.isValidTool(toolName):
activatedCount += 1
@ -499,9 +496,6 @@ class AgentService:
try:
from modules.workflows.processing.shared.methodDiscovery import discoverMethods
from modules.workflows.processing.core.actionExecutor import ActionExecutor
from modules.serviceCenter.services.serviceAgent.actionToolAdapter import (
ActionToolAdapter,
)
discoverMethods(self.services)
adapter = ActionToolAdapter(ActionExecutor(self.services))
@ -622,7 +616,6 @@ class AgentService:
def _createPersistRoundMemoryFn(self, workflowId: str):
"""Create callback that persists RoundMemory entries after tool execution."""
from modules.serviceCenter.services.serviceAgent.agentLoop import classifyToolResult
from modules.datamodels.datamodelKnowledge import RoundMemory
async def _persistRoundMemory(

4
modules/serviceCenter/services/serviceAi/mainServiceAi.py
View file

@ -7,7 +7,7 @@ import time
import base64
from typing import Dict, Any, List, Optional, Tuple, Callable
from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument, WorkflowModeEnum
from modules.datamodels.datamodelAi import AiCallRequest, AiCallResponse, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
from modules.datamodels.datamodelAi import AiCallRequest, AiCallResponse, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum, AiModelCall
from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent
from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData
from modules.datamodels.datamodelDocument import RenderedDocument
@ -335,7 +335,6 @@ class AiService:
Returns:
AiCallResponse with content as JSON string (SpeechTeamsResponse format)
"""
from modules.datamodels.datamodelAi import AiCallResponse, AiModelCall, AiCallOptions, PriorityEnum
startTime = time.time()
@ -637,7 +636,6 @@ detectedIntent-Werte:
try:
from modules.aicore.aicoreModelRegistry import modelRegistry
from modules.aicore.aicoreModelSelector import modelSelector as _modSel
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum
_models = modelRegistry.getAvailableModels()
_providers = self._calculateEffectiveProviders()

6
modules/serviceCenter/services/serviceChat/mainServiceChat.py
View file

@ -3,8 +3,8 @@
"""Chat service for document processing, chat operations, and workflow management."""
import logging
from typing import Dict, Any, List, Optional, Callable
from modules.datamodels.datamodelUam import User, UserConnection
from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatLog
from modules.datamodels.datamodelUam import User, UserConnection, UserVoicePreferences
from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatLog, ActionItem
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
from modules.shared.progressLogger import ProgressLogger
import json
@ -615,7 +615,6 @@ class ChatService:
def getUserVoicePreferences(self, userId: str, mandateId: str = None) -> Optional[Dict[str, Any]]:
"""Get TTS voice preferences for a user, resolved by mandate scope."""
from modules.datamodels.datamodelUam import UserVoicePreferences
try:
prefRecords = self.interfaceDbApp.db.getRecordset(
UserVoicePreferences, recordFilter={"userId": userId}
@ -842,7 +841,6 @@ class ChatService:
"""Create an ActionItem record in the chat DB.
Encapsulates low-level _separateObjectFields + db.recordCreate so callers
never need direct interfaceDbChat access."""
from modules.datamodels.datamodelChat import ActionItem
simpleFields, _objectFields = self.interfaceDbChat._separateObjectFields(ActionItem, actionData)
return self.interfaceDbChat.db.recordCreate(ActionItem, simpleFields)

3
modules/serviceCenter/services/serviceExtraction/extractors/extractorContainer.py
View file

@ -22,7 +22,7 @@ import tarfile
from ..subUtils import makeId
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelContent import ContainerLimitError, ContentContextRef
from ..subRegistry import Extractor
from ..subRegistry import Extractor, getExtractorRegistry
import base64
logger = logging.getLogger(__name__)
@ -204,7 +204,6 @@ def _addFilePart(
entryPath = f"{containerPath}/{fileName}" if containerPath else fileName
detectedMime = _detectMimeType(fileName)
from ..subRegistry import getExtractorRegistry
registry = getExtractorRegistry()
extractor = registry.resolve(detectedMime, fileName)

3
modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py
View file

@ -18,7 +18,7 @@ import mimetypes
from modules.datamodels.datamodelExtraction import ContentPart
from ..subUtils import makeId
from ..subRegistry import Extractor
from ..subRegistry import Extractor, getExtractorRegistry
import base64
logger = logging.getLogger(__name__)
@ -255,7 +255,6 @@ def _delegateAttachment(attachData: bytes, attachName: str, parentId: str, depth
guessedMime, _ = mimetypes.guess_type(attachName)
detectedMime = guessedMime or "application/octet-stream"
from ..subRegistry import getExtractorRegistry
registry = getExtractorRegistry()
extractor = registry.resolve(detectedMime, attachName)

3
modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py
View file

@ -16,7 +16,7 @@ from pathlib import Path
from ..subUtils import makeId
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelContent import ContainerLimitError, ContentContextRef
from ..subRegistry import Extractor
from ..subRegistry import Extractor, ExtractorRegistry
logger = logging.getLogger(__name__)
@ -141,7 +141,6 @@ def _walkFolder(
guessedMime, _ = mimetypes.guess_type(entry.name)
detectedMime = guessedMime or "application/octet-stream"
from ..subRegistry import ExtractorRegistry
registry = ExtractorRegistry()
extractor = registry.resolve(detectedMime, entry.name)

5
modules/serviceCenter/services/serviceExtraction/subRegistry.py
View file

@ -3,7 +3,7 @@
from typing import Any, Dict, List, Optional, TYPE_CHECKING
import logging
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelExtraction import ContentPart, ContentExtracted
import os
import traceback
from pathlib import Path
@ -50,9 +50,8 @@ class Extractor:
precomputedParts: Optional[List[ContentPart]] = None,
) -> "UdmDocument":
"""Build UDM from extracted parts (default: heuristic grouping). Override for format-specific trees."""
from modules.datamodels.datamodelUdm import contentPartsToUdm, mimeToUdmSourceType
from modules.datamodels.datamodelExtraction import ContentExtracted
from .subUtils import makeId
from modules.datamodels.datamodelUdm import contentPartsToUdm, mimeToUdmSourceType
parts = precomputedParts if precomputedParts is not None else self.extract(fileBytes, context)
eid = context.get("extractionId") or makeId()

3
modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py
View file

@ -28,7 +28,7 @@ except ImportError:
import re
from ._pdfFontFallback import wrapEmojiSpansInXml as _wrapEmojiSpansInXml
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle
import os
import tempfile
@ -230,7 +230,6 @@ class RendererPdf(BaseRenderer):
# memory simultaneously. Collected here, deleted after the build.
self._tempImageFiles = []
try:
from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle
self._unifiedStyle = unifiedStyle or resolveStyle(None)
styles = self._convertUnifiedStyleToInternal(self._unifiedStyle)
for level in range(1, 7):

3
modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py
View file

@ -8,7 +8,7 @@ from datetime import datetime, UTC
from typing import Dict, Any, Optional, List
from .documentRendererBaseTemplate import BaseRenderer
from modules.datamodels.datamodelDocument import RenderedDocument
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle
logger = logging.getLogger(__name__)
@ -90,7 +90,6 @@ class RendererPptx(BaseRenderer):
from pptx.dml.color import RGBColor
if not style:
from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle
style = resolveStyle(None)
internalStyle = self._convertUnifiedStyleToInternal(style)
styles = internalStyle

3
modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py
View file

@ -6,7 +6,7 @@ Excel renderer for report generation using openpyxl.
from .documentRendererBaseTemplate import BaseRenderer
from modules.datamodels.datamodelDocument import RenderedDocument
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge
from modules.serviceCenter.services.serviceGeneration.styleDefaults import deepMerge, resolveStyle
from typing import Dict, Any, List, Optional
import io
import base64
@ -137,7 +137,6 @@ class RendererXlsx(BaseRenderer):
self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(jsonContent.keys()) if isinstance(jsonContent, dict) else 'Not a dict'}", "EXCEL_RENDERER")
if not style:
from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle
style = resolveStyle(None)
self._unifiedStyle = style

26
modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py
View file

@ -638,7 +638,8 @@ class KnowledgeService:
Returns:
Formatted context string for injection into the agent's system prompt.
"""
queryVector = await self._embedSingle(currentPrompt)
queryText = _extractUserQuery(currentPrompt)
queryVector = await self._embedSingle(queryText) if queryText else []
logger.debug(
"buildAgentContext.start userId=%s featureInstanceId=%s mandateId=%s isSysAdmin=%s prompt=%r",
userId, featureInstanceId, mandateId, isSysAdmin, (currentPrompt or "")[:120],
@ -960,6 +961,29 @@ class KnowledgeService:
# Internal helpers
# =============================================================================
# Markers added by the prompt enrichment layers (_enrichPromptWithFiles in
# mainServiceAgent, data source sections in routeFeatureWorkspace). Used to
# isolate the user's own words for the semantic search query.
_USER_REQUEST_MARKER = "\n\nUser request: "
_PROMPT_SECTION_MARKERS = ("\n\n[Active Data Sources]\n", "\n\n[Attached Feature Data Sources]\n")
def _extractUserQuery(prompt: str) -> str:
"""Isolate the user's actual question from an enriched agent prompt.
Enriched prompts wrap the user request in file metadata headers and
data-source sections. Only the user's own words form a meaningful semantic
search query - embedding the metadata would dilute the vector and can
exceed the embedding model's input limit.
"""
if not prompt:
return ""
text = prompt.rsplit(_USER_REQUEST_MARKER, 1)[-1]
for marker in _PROMPT_SECTION_MARKERS:
text = text.split(marker, 1)[0]
return text.strip()
def _estimateTokens(text: str) -> int:
"""Estimate token count using character-based heuristic (1 token ~ 4 chars)."""
return max(1, len(text) // CHARS_PER_TOKEN)

5
modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py
View file

@ -26,6 +26,7 @@ from typing import Any, Callable, Dict, List, Optional
from modules.serviceCenter.services.serviceKnowledge.subTextClean import cleanEmailBody
from modules.serviceCenter.services.serviceKnowledge.subWalkerHelpers import (
WalkerTimeout,
extractWithTimeout as _extractWithTimeout,
ingestWithTimeout,
logItemStart,
)
@ -564,10 +565,6 @@ async def _ingestAttachments(
attLabel = f"{messageId}/att:{stub['attachmentId']}/{fileName}"
logItemStart("gmail-attachment", attLabel, sizeBytes=stub.get("size") or None, mime=mimeType)
from modules.serviceCenter.services.serviceKnowledge.subWalkerHelpers import (
extractWithTimeout as _extractWithTimeout,
)
def _runAttExtraction():
return runExtraction(
extractorRegistry, chunkerRegistry,

5
modules/serviceCenter/services/serviceSubscription/mainServiceSubscription.py
View file

@ -12,7 +12,7 @@ import time
from typing import Dict, Any, List, Optional
from datetime import datetime, timezone, timedelta
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelUam import User, Mandate
from modules.datamodels.datamodelSubscription import (
SubscriptionPlan,
MandateSubscription,
@ -24,6 +24,7 @@ from modules.datamodels.datamodelSubscription import (
)
from modules.interfaces.interfaceDbSubscription import (
getInterface as getSubscriptionInterface,
getRootInterface as getSubRootInterface,
InvalidTransitionError,
)
from modules.shared.i18nRegistry import t
@ -414,7 +415,6 @@ class SubscriptionService:
mandateLabel = mandateId
try:
from modules.datamodels.datamodelUam import Mandate
from modules.security.rootAccess import getRootDbAppConnector
appDb = getRootDbAppConnector()
rows = appDb.getRecordset(Mandate, recordFilter={"id": mandateId})
@ -937,7 +937,6 @@ def _buildInvoiceSummaryHtml(
) -> str:
"""Build an HTML invoice summary block for inclusion in the activation email."""
import html as htmlmod
from modules.interfaces.interfaceDbSubscription import getRootInterface as getSubRootInterface
subInterface = getSubRootInterface()
userCount = subInterface.countActiveUsers(mandateId)

4
modules/system/databaseHealth.py
View file

@ -20,7 +20,7 @@ import psycopg2.extras
from modules.shared.configuration import APP_CONFIG
from modules.dbHelpers.dbRegistry import getRegisteredDatabases
from modules.dbHelpers.fkRegistry import getFkRelationships, FkRelationship
from modules.dbHelpers.fkRegistry import getFkRelationships, FkRelationship, ensureModelsLoaded
logger = logging.getLogger(__name__)
@ -805,7 +805,6 @@ def _discoverLegacyTables(dbFilter: Optional[str] = None) -> List[dict]:
Returns a list of dicts: {db, table, rowCount, sizeBytes}.
"""
from modules.datamodels.datamodelBase import MODEL_REGISTRY
from modules.dbHelpers.fkRegistry import ensureModelsLoaded
ensureModelsLoaded()
registeredDbs = getRegisteredDatabases()
@ -854,7 +853,6 @@ def _dropLegacyTable(dbName: str, tableName: str) -> dict:
Raises ValueError if the table is model-backed (safety guard).
"""
from modules.datamodels.datamodelBase import MODEL_REGISTRY
from modules.dbHelpers.fkRegistry import ensureModelsLoaded
ensureModelsLoaded()
if tableName in MODEL_REGISTRY:

2
modules/workflowAutomation/engine/executors/actionNodeExecutor.py
View file

@ -17,6 +17,7 @@ from typing import Any, Dict, Optional
from modules.nodeCatalog.portTypes import (
_normalizeError,
normalizeToSchema,
PORT_TYPE_CATALOG,
)
from modules.datamodels.serviceExceptions import SubscriptionInactiveException as _SubscriptionInactiveException, BillingContextError as _BillingContextError
from modules.workflows.methods.methodContext.actions.extractContent import (
@ -305,7 +306,6 @@ def _buildConnectionRefDict(connRef: str, chatService, services) -> Optional[Dic
def _schemaCarriesConnectionProvenance(outputSchema: str) -> bool:
"""True iff the port schema declares ``carriesConnectionProvenance`` in the catalog."""
from modules.nodeCatalog.portTypes import PORT_TYPE_CATALOG
schema = PORT_TYPE_CATALOG.get(outputSchema)
return bool(getattr(schema, "carriesConnectionProvenance", False))

18
modules/workflowAutomation/helpers.py
View file

@ -24,7 +24,8 @@ from modules.datamodels.datamodelWorkflowAutomation import (
AutoRun, AutoStepLog, AutoWorkflow, AutoTask, AutoVersion,
WORKFLOW_AUTOMATION_DATABASE,
)
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface
from modules.interfaces.interfaceDbApp import getRootInterface as _getRootIface, getRootInterface
from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields
from modules.shared.configuration import APP_CONFIG
logger = logging.getLogger(__name__)
@ -126,9 +127,7 @@ def _isUserMandateAdmin(userId: str, mandateId: str) -> bool:
def _scopedWorkflowFilter(context: RequestContext) -> Optional[Dict[str, Any]]:
"""Build DB filter for listing workflows: mandate-scoped for members, None for sysadmin."""
if context.isPlatformAdmin:
return None
"""Build DB filter for listing workflows: always mandate-scoped by membership."""
userId = str(context.user.id) if context.user else None
if not userId:
return {"mandateId": "__impossible__"}
@ -139,14 +138,17 @@ def _scopedWorkflowFilter(context: RequestContext) -> Optional[Dict[str, Any]]:
def _scopedRunFilter(context: RequestContext) -> Optional[Dict[str, Any]]:
"""Build DB filter for listing runs: admin sees mandate runs, user sees own."""
if context.isPlatformAdmin:
return None
"""Build DB filter for listing runs: always mandate-scoped by membership.
Mandate admins see all runs in their mandates, regular members see own."""
userId = str(context.user.id) if context.user else None
if not userId:
return {"ownerId": "__impossible__"}
mandateIds = _getUserMandateIds(userId)
if not mandateIds:
return {"ownerId": "__impossible__"}
adminMandateIds = _getAdminMandateIds(userId, mandateIds)
if context.isPlatformAdmin:
return {"mandateId": mandateIds}
if adminMandateIds:
return {"mandateId": adminMandateIds}
return {"ownerId": userId}
@ -202,7 +204,6 @@ def _validateWorkflowAccess(
if action == "execute":
targetInstanceId = workflow.get("targetFeatureInstanceId")
if targetInstanceId:
from modules.interfaces.interfaceDbApp import getRootInterface
access = getRootInterface().getFeatureAccess(userId, targetInstanceId)
if access and access.get("enabled"):
return
@ -581,7 +582,6 @@ def _getWorkflowsJoinedPaginated(
paginationParams: PaginationParams,
) -> dict:
"""SQL listing: AutoWorkflow LEFT JOIN aggregated AutoRun stats (one query + count)."""
from modules.connectors.connectorDbPostgre import getModelFields, parseRecordFields
wfFields = getModelFields(AutoWorkflow)
whereClause, orderClause, limitClause, values = _buildJoinedWorkflowWhereOrderLimit(

3
modules/workflowAutomation/mainWorkflowAutomation.py
View file

@ -12,7 +12,7 @@ import logging
import uuid
from typing import Dict, List, Any, Optional
from modules.shared.i18nRegistry import t
from modules.shared.i18nRegistry import t, resolveText
logger = logging.getLogger(__name__)
@ -291,7 +291,6 @@ def onInstanceCreate(mandateId: str, instanceId: str, featureCode: str, template
"""Create workflow instances from template definitions when a feature instance is created."""
from modules.interfaces.interfaceWorkflowAutomation import _getWorkflowAutomationInterface
from modules.security.rootAccess import getRootUser
from modules.shared.i18nRegistry import resolveText
rootUser = getRootUser()
waInterface = _getWorkflowAutomationInterface(rootUser, mandateId, instanceId)

4
modules/workflows/methods/methodAi/actions/process.py
View file

@ -9,7 +9,7 @@ import uuid
from typing import Dict, Any, List, Optional
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum
from modules.datamodels.datamodelExtraction import ContentPart
from modules.datamodels.datamodelExtraction import ContentPart, ExtractionOptions, MergeStrategy
from modules.datamodels.serviceExceptions import SubscriptionInactiveException, BillingContextError
logger = logging.getLogger(__name__)
@ -37,7 +37,6 @@ def _action_docs_to_content_parts(services, docs: List[Any]) -> List[ContentPart
"""Extract content from ActionDocument-like objects in memory (no persistence).
Decodes base64, runs extraction pipeline, returns ContentParts for AI.
"""
from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy
all_parts = []
extraction = services.extraction
@ -78,7 +77,6 @@ def _resolve_file_refs_to_content_parts(services, fileIdRefs) -> List[ContentPar
references, not chat message attachments. In the agent/chat context,
``DocumentItemReference`` holds ChatDocument IDs that must be resolved
via ``getChatDocumentsFromDocumentList`` instead."""
from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy
extraction = services.extraction
if not extraction:

568
scripts/script_analyze_platform_module_graph.py Normal file
View file

@ -0,0 +1,568 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""
Deep platform-core module import graph analysis.
Output: local/notes/refernce-analysis/import-analysis-platform-modules.md
Usage:
python platform-core/scripts/script_analyze_platform_module_graph.py
"""
from __future__ import annotations
import ast
import os
import sys
from collections import defaultdict
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from typing import Dict, List, Optional, Set, Tuple
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from script_analyze_porta_imports import ( # noqa: E402
OUTPUT_ROOT,
PLATFORM_ROOT,
SKIP_DIR_NAMES,
_collectPlatformModules,
_getPlatformContainer,
_platformModuleId,
_resolvePlatformImportTarget,
_resolvePlatformRelativeImport,
_writeText,
)
OUTPUT_FILE = OUTPUT_ROOT / "import-analysis-platform-modules.md"
LAYER_ORDER = {
"shared": 0,
"datamodels": 1,
"connectors": 2,
"nodeCatalog": 2,
"dbHelpers": 3,
"interfaces": 4,
"system": 4,
"security": 4,
"auth": 4,
"aicore": 4,
"demoConfigs": 4,
"serviceCenter": 5,
"workflows": 5,
"workflowAutomation": 5,
"features.commcoach": 5,
"features.neutralization": 5,
"features.realEstate": 5,
"features.realestate": 5,
"features.redmine": 5,
"features.teamsbot": 5,
"features.trustee": 5,
"features.workspace": 5,
"routes": 6,
"app": 7,
}
@dataclass
class ScopedImport:
target: str
rawModule: str
position: str
scope: str
isInternal: bool
isStdLib: bool
def _shortModule(moduleId: str) -> str:
parts = moduleId.replace("platform-core.", "").split(".")
if len(parts) <= 3:
return ".".join(parts)
return ".".join(parts[-3:])
LIFECYCLE_SCOPE_MARKERS = (
"lifespan",
"onBootstrap",
"onStart",
"onStop",
"onInstanceCreate",
"onMandateDelete",
"registerFeature",
"preWarm",
)
def _layerOf(moduleId: str) -> Optional[int]:
container = _getPlatformContainer(moduleId)
if container is None:
return None
return LAYER_ORDER.get(container)
def _isStdLibModule(moduleName: str) -> bool:
root = moduleName.split(".")[0]
if root.startswith("_"):
return False
if root in sys.builtin_module_names:
return True
if hasattr(sys, "stdlib_module_names") and root in sys.stdlib_module_names:
return True
return False
class _DetailedImportVisitor(ast.NodeVisitor):
def __init__(self, filePath: Path):
self.filePath = filePath
self.imports: List[ScopedImport] = []
self._scopeStack: List[str] = []
@property
def _currentScope(self) -> str:
return self._scopeStack[-1] if self._scopeStack else ""
def _position(self) -> str:
return "code" if self._scopeStack else "header"
def _add(self, rawModule: str, resolved: str, isInternal: bool) -> None:
self.imports.append(
ScopedImport(
target=resolved,
rawModule=rawModule,
position=self._position(),
scope=self._currentScope,
isInternal=isInternal,
isStdLib=_isStdLibModule(rawModule) if not isInternal else False,
)
)
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
self._scopeStack.append(f"function {node.name}")
self.generic_visit(node)
self._scopeStack.pop()
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
self._scopeStack.append(f"function {node.name}")
self.generic_visit(node)
self._scopeStack.pop()
def visit_ClassDef(self, node: ast.ClassDef) -> None:
self._scopeStack.append(f"class {node.name}")
self.generic_visit(node)
self._scopeStack.pop()
def visit_Import(self, node: ast.Import) -> None:
for alias in node.names:
resolved, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name)
self._add(alias.name, resolved, isInternal)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
if node.level > 0:
resolved = _resolvePlatformRelativeImport(self.filePath, node)
if resolved:
suffix = node.module or ""
raw = ("." * node.level) + suffix
self._add(raw, resolved, True)
return
if not node.module:
return
resolved, isInternal = _resolvePlatformImportTarget(self.filePath, node.module)
self._add(node.module, resolved, isInternal)
def _collectDetailedImports() -> Dict[str, List[ScopedImport]]:
byModule: Dict[str, List[ScopedImport]] = {}
pyFiles: List[Path] = []
appFile = PLATFORM_ROOT / "app.py"
if appFile.exists():
pyFiles.append(appFile)
for root, dirs, files in os.walk(PLATFORM_ROOT / "modules"):
dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES]
for fileName in files:
if fileName.endswith(".py"):
pyFiles.append(Path(root) / fileName)
for filePath in pyFiles:
moduleId = _platformModuleId(filePath)
if _getPlatformContainer(moduleId) is None:
continue
try:
tree = ast.parse(filePath.read_text(encoding="utf-8"), filename=str(filePath))
except (SyntaxError, UnicodeDecodeError):
continue
visitor = _DetailedImportVisitor(filePath)
visitor.visit(tree)
byModule[moduleId] = visitor.imports
return byModule
def _internalGraph(importsByModule: Dict[str, List[ScopedImport]]) -> Dict[str, Set[str]]:
graph: Dict[str, Set[str]] = defaultdict(set)
for source, items in importsByModule.items():
for item in items:
if item.isInternal and item.target.startswith("platform-core."):
graph[source].add(item.target)
return dict(graph)
def _mutualPairs(
graph: Dict[str, Set[str]],
moduleFilter: Optional[Set[str]] = None,
) -> List[Tuple[str, str]]:
pairs: List[Tuple[str, str]] = []
seen: Set[Tuple[str, str]] = set()
sources = moduleFilter if moduleFilter is not None else set(graph.keys())
for source in sorted(sources):
for target in graph.get(source, set()):
if moduleFilter is not None and target not in moduleFilter:
continue
if target not in graph or source not in graph[target]:
continue
key = tuple(sorted((source, target)))
if key not in seen:
seen.add(key)
pairs.append(key)
return pairs
def _tarjanScc(graph: Dict[str, Set[str]]) -> List[List[str]]:
index = 0
stack: List[str] = []
onStack: Set[str] = set()
indices: Dict[str, int] = {}
lowLink: Dict[str, int] = {}
result: List[List[str]] = []
nodes = set(graph.keys())
for targets in graph.values():
nodes.update(targets)
def strongConnect(node: str) -> None:
nonlocal index
indices[node] = index
lowLink[node] = index
index += 1
stack.append(node)
onStack.add(node)
for neighbor in graph.get(node, set()):
if neighbor not in indices:
strongConnect(neighbor)
lowLink[node] = min(lowLink[node], lowLink[neighbor])
elif neighbor in onStack:
lowLink[node] = min(lowLink[node], indices[neighbor])
if lowLink[node] == indices[node]:
component: List[str] = []
while True:
w = stack.pop()
onStack.remove(w)
component.append(w)
if w == node:
break
if len(component) > 1 or (len(component) == 1 and component[0] in graph.get(component[0], set())):
result.append(sorted(component))
for node in sorted(nodes):
if node not in indices:
strongConnect(node)
return sorted(result, key=lambda c: (len(c), c[0]), reverse=True)
def _canReach(graph: Dict[str, Set[str]], start: str, goal: str, skipEdge: Optional[Tuple[str, str]] = None) -> bool:
visited: Set[str] = set()
def dfs(node: str) -> bool:
if node == goal:
return True
if node in visited:
return False
visited.add(node)
for nxt in graph.get(node, set()):
if skipEdge and node == skipEdge[0] and nxt == skipEdge[1]:
continue
if dfs(nxt):
return True
return False
return dfs(start)
def _assessMutualPair(a: str, b: str) -> str:
containerA = _getPlatformContainer(a)
containerB = _getPlatformContainer(b)
layerA = _layerOf(a)
layerB = _layerOf(b)
sameContainer = containerA == containerB
if sameContainer:
if layerA is not None and layerA >= 5:
return "Prüfen — Feature/Service-interner Gegenimport; oft Lazy-Import-Workaround, Zyklus im Container."
return "Prüfen — gegenseitiger Import im gleichen Container; meist absichtlicher Lazy-Import gegen Zyklus."
if layerA is not None and layerB is not None:
if layerA < layerB and layerB < layerA:
pass
upward = (layerA > layerB and layerB is not None) or (layerB > layerA and layerA is not None)
if upward:
return "Refactor-Kandidat — untere Schicht importiert obere und umgekehrt (Layer-Verletzung)."
return "Refactor-Kandidat — Cross-Container-Gegenimport; Layer-Grenze prüfen."
def _assessCycle(component: List[str]) -> str:
if len(component) == 1:
return "OK — Package-Reexport/Self-Import (__init__ ↔ Submodul); typisch für Barrel-Module."
containers = {_getPlatformContainer(m) for m in component}
containers.discard(None)
layers = [layer for m in component if (layer := _layerOf(m)) is not None]
if len(containers) == 1:
container = next(iter(containers))
if LAYER_ORDER.get(container or "", 99) >= 5:
return "Prüfen — Zyklus innerhalb Feature/Service-Cluster; oft bekanntes Deferred-Coupling."
return "Prüfen — Intra-Container-Loop; Lazy-Imports prüfen ob extrahierbar."
if layers and max(layers) - min(layers) >= 2:
return "Refactor-Kandidat — Loop über mehrere Layer/Container; Architektur-Grenze verletzt."
return "Prüfen — Cross-Container-Loop; Abhängigkeit entkoppeln oder Typ/Protocol extrahieren."
def _assessLazyStdLib(moduleId: str, item: ScopedImport) -> str:
heavy = {"json", "csv", "xml", "pickle", "sqlite3", "subprocess", "multiprocessing"}
root = item.rawModule.split(".")[0]
if root in heavy:
return "OK — schwere Stdlib lazy (Startup/optional)."
if "TYPE_CHECKING" in item.scope:
return "OK — typing-only Kontext."
return "Harmlos — Stdlib lazy in Code-Scope; kein Architektur-Risiko."
def _assessMovable(moduleId: str, item: ScopedImport, graph: Dict[str, Set[str]], headerTargets: Set[str]) -> str:
if item.target in headerTargets:
return "Redundant — bereits im Header importiert; Lazy-Import entfernen."
if any(marker in item.scope for marker in LIFECYCLE_SCOPE_MARKERS):
return "Beabsichtigt lazy — Startup/Lifecycle-Hook; nicht in Header verschieben."
if _canReach(graph, item.target, moduleId):
return "Muss lazy bleiben — Header-Import würde Zyklus erzeugen."
return "Verschiebbar — kann vermutlich in den Header."
def _renderMarkdown(
importsByModule: Dict[str, List[ScopedImport]],
graph: Dict[str, Set[str]],
) -> str:
modulesByContainer: Dict[str, Set[str]] = defaultdict(set)
for moduleId in importsByModule:
container = _getPlatformContainer(moduleId)
if container:
modulesByContainer[container].add(moduleId)
lines = [
"# Import-Analyse Platform — Modul-Graph",
"",
f"- **Generiert:** {date.today().isoformat()}",
"- **Script:** `platform-core/scripts/script_analyze_platform_module_graph.py`",
"- **Scope:** interne `modules.*`-Imports (inkl. lazy)",
"",
"## Legende Beurteilung",
"",
"| Stufe | Bedeutung |",
"|-------|-----------|",
"| OK / Harmlos | kein Handlungsbedarf |",
"| Verschiebbar | Lazy-Import kann vermutlich in Header |",
"| Redundant | doppelter Import (Header + Code) |",
"| Prüfen | bekannt möglich, bewusst prüfen |",
"| Beabsichtigt lazy | Startup/Lifecycle — nicht in Header |",
"| Muss lazy bleiben | Zyklusvermeidung |",
"| Refactor-Kandidat | Layer-/Architektur-Thema |",
"",
]
# --- Mutual pairs per container ---
lines.extend(["## Gegenseitige Modul-Imports (Paare)", ""])
totalPairs = 0
for container in sorted(modulesByContainer.keys()):
moduleSet = modulesByContainer[container]
pairs = _mutualPairs(graph, moduleSet)
if not pairs:
continue
totalPairs += len(pairs)
lines.append(f"### Container `{container}`")
lines.append("")
lines.append("| Modul A | Modul B | Beurteilung |")
lines.append("|---------|---------|-------------|")
for a, b in pairs:
lines.append(
f"| `{_shortModule(a)}` | `{_shortModule(b)}` | {_assessMutualPair(a, b)} |"
)
lines.append("")
crossPairs = [
p for p in _mutualPairs(graph)
if _getPlatformContainer(p[0]) != _getPlatformContainer(p[1])
]
if crossPairs:
lines.extend(["### Cross-Container (gegenseitig)", ""])
lines.append("| Modul A | Container A | Modul B | Container B | Beurteilung |")
lines.append("|---------|-------------|---------|-------------|-------------|")
for a, b in crossPairs:
lines.append(
f"| `{_shortModule(a)}` | `{_getPlatformContainer(a)}` | "
f"`{_shortModule(b)}` | `{_getPlatformContainer(b)}` | {_assessMutualPair(a, b)} |"
)
lines.append("")
if totalPairs == 0 and not crossPairs:
lines.append("_Keine gegenseitigen Modul-Paare gefunden._")
lines.append("")
# --- Cycles ---
sccList = _tarjanScc(graph)
lines.extend(["## Import-Loops (über mehrere Module)", ""])
if not sccList:
lines.append("_Keine Strongly-Connected Components (>1 Knoten) gefunden._")
lines.append("")
else:
lines.append(f"**{len(sccList)} Loop-Gruppe(n)** (Tarjan SCC, nur interne Module).")
lines.append("")
for index, component in enumerate(sccList, start=1):
containers = sorted({c for m in component if (c := _getPlatformContainer(m))})
lines.append(f"### Loop {index} — {len(component)} Module")
lines.append("")
lines.append(f"- **Container:** {', '.join(f'`{c}`' for c in containers)}")
lines.append(f"- **Beurteilung:** {_assessCycle(component)}")
lines.append("- **Module:**")
for moduleId in component:
lines.append(f" - `{moduleId}`")
if len(component) <= 8:
chainHint = " → ".join(_shortModule(m) for m in component) + f" → `{_shortModule(component[0])}`"
lines.append(f"- **Ring (Auszug):** {chainHint}")
lines.append("")
# --- Lazy stdlib ---
lines.extend(["## Lazy Stdlib-Imports (in Code-Scope)", ""])
stdlibRows: List[Tuple[str, str, str, str, str]] = []
for moduleId, items in sorted(importsByModule.items()):
for item in items:
if item.position == "code" and item.isStdLib:
stdlibRows.append(
(
_shortModule(moduleId),
_getPlatformContainer(moduleId) or "",
item.rawModule,
item.scope or "(class/function)",
_assessLazyStdLib(moduleId, item),
)
)
if stdlibRows:
lines.append("| Modul | Container | Import | Scope | Beurteilung |")
lines.append("|-------|-----------|--------|-------|-------------|")
for row in stdlibRows:
lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |")
lines.append("")
else:
lines.append("_Keine lazy Stdlib-Imports in Code-Scope._")
lines.append("")
# --- Lazy internal movable ---
lines.extend(["## Lazy interne Imports — Header möglich?", ""])
movableRows: List[Tuple[str, str, str, str, str]] = []
intentionalRows: List[Tuple[str, str, str, str, str]] = []
mustStayRows: List[Tuple[str, str, str, str, str]] = []
redundantRows: List[Tuple[str, str, str, str, str]] = []
for moduleId, items in sorted(importsByModule.items()):
headerTargets = {i.target for i in items if i.position == "header" and i.isInternal}
for item in items:
if item.position != "code" or not item.isInternal:
continue
verdict = _assessMovable(moduleId, item, graph, headerTargets)
row = (
_shortModule(moduleId),
_getPlatformContainer(moduleId) or "",
_shortModule(item.target),
item.scope or "(code)",
verdict,
)
if verdict.startswith("Verschiebbar"):
movableRows.append(row)
elif verdict.startswith("Beabsichtigt"):
intentionalRows.append(row)
elif verdict.startswith("Redundant"):
redundantRows.append(row)
elif verdict.startswith("Muss lazy"):
mustStayRows.append(row)
if intentionalRows:
lines.append("### Beabsichtigt lazy (Startup/Lifecycle)")
lines.append("")
lines.append(f"**{len(intentionalRows)}** Einträge — lazy in lifespan/onBootstrap/…; kein Refactor nötig.")
lines.append("")
if movableRows:
lines.append("### Verschiebbar in Header")
lines.append("")
lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |")
lines.append("|-------|-----------|-------------|-------|-------------|")
for row in movableRows:
lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |")
lines.append("")
if mustStayRows:
lines.append("### Muss lazy bleiben (Zyklus)")
lines.append("")
lines.append(f"**{len(mustStayRows)}** Einträge — Auszug (max. 40):")
lines.append("")
lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |")
lines.append("|-------|-----------|-------------|-------|-------------|")
for row in mustStayRows[:40]:
lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |")
if len(mustStayRows) > 40:
lines.append("")
lines.append(f"_… und {len(mustStayRows) - 40} weitere._")
lines.append("")
if redundantRows:
lines.append("### Redundant (Header + Code)")
lines.append("")
lines.append("| Modul | Container | Import-Ziel | Scope | Beurteilung |")
lines.append("|-------|-----------|-------------|-------|-------------|")
for row in redundantRows:
lines.append(f"| `{row[0]}` | `{row[1]}` | `{row[2]}` | {row[3]} | {row[4]} |")
lines.append("")
if not movableRows and not mustStayRows and not redundantRows and not intentionalRows:
lines.append("_Keine lazy internen Imports gefunden._")
lines.append("")
lines.extend(
[
"## Kurzfassung",
"",
f"- Gegenseitige Modul-Paare (intra-container): **{totalPairs}**",
f"- Gegenseitige Modul-Paare (cross-container): **{len(crossPairs)}**",
f"- Import-Loop-Gruppen (SCC): **{len(sccList)}** (davon Self-Loop: **{sum(1 for c in sccList if len(c) == 1)}**)",
f"- Lazy Stdlib-Imports: **{len(stdlibRows)}**",
f"- Lazy intern / beabsichtigt (Lifecycle): **{len(intentionalRows)}**",
f"- Lazy intern / verschiebbar: **{len(movableRows)}**",
f"- Lazy intern / Zyklus (muss bleiben): **{len(mustStayRows)}**",
f"- Lazy intern / redundant: **{len(redundantRows)}**",
"",
]
)
return "\n".join(lines)
def main() -> None:
print("Collecting detailed platform imports...")
importsByModule = _collectDetailedImports()
graph = _internalGraph(importsByModule)
print(f" modules: {len(importsByModule)}")
print(f" internal edges: {sum(len(v) for v in graph.values())}")
markdown = _renderMarkdown(importsByModule, graph)
_writeText(OUTPUT_FILE, markdown)
print(f"Written: {OUTPUT_FILE}")
if __name__ == "__main__":
main()

898
scripts/script_analyze_porta_imports.py Normal file
View file

@ -0,0 +1,898 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""
Analyze all imports (including lazy/dynamic) for PowerOn PORTA UI and platform-core.
Outputs under local/notes/refernce-analysis/:
platform/modules/*.md one file per Python module
platform/containers/*.md aggregated stats per container
platform/container-network.drawio
ui/modules/*.md
ui/containers/*.md
ui/container-network.drawio
README.md
Usage:
python platform-core/scripts/script_analyze_porta_imports.py
"""
from __future__ import annotations
import ast
import html
import math
import os
import re
from collections import defaultdict
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Set, Tuple
SCRIPT_DIR = Path(__file__).resolve().parent
PLATFORM_ROOT = SCRIPT_DIR.parent
REPO_ROOT = PLATFORM_ROOT.parent
UI_ROOT = REPO_ROOT / "ui-nyla"
OUTPUT_ROOT = REPO_ROOT / "local" / "notes" / "refernce-analysis"
SKIP_DIR_NAMES = {
"__pycache__",
"node_modules",
".git",
"dist",
"build",
".venv",
"venv",
".tox",
".mypy_cache",
".pytest_cache",
}
UI_SKIP_GLOBS = ("**/*.test.ts", "**/*.test.tsx", "test/**")
@dataclass
class ImportRecord:
importedModule: str
position: str # "header" | "code"
isInternal: bool
sourceContainer: Optional[str] = None
targetContainer: Optional[str] = None
@dataclass
class ModuleAnalysis:
context: str # "platform" | "ui"
moduleId: str
filePath: Path
container: str
containerPath: str
imports: List[ImportRecord] = field(default_factory=list)
def _sanitizeFileName(value: str) -> str:
return re.sub(r"[^A-Za-z0-9._-]+", "_", value)
def _writeText(path: Path, content: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
# ---------------------------------------------------------------------------
# Platform (Python)
# ---------------------------------------------------------------------------
def _platformModuleId(filePath: Path) -> str:
rel = filePath.relative_to(PLATFORM_ROOT)
if filePath.name == "__init__.py":
parts = rel.parent.parts
else:
parts = rel.with_suffix("").parts
return "platform-core." + ".".join(parts)
def _platformContainerPath(container: str) -> str:
if container == "app":
return "platform-core/app.py"
if container.startswith("features."):
featureCode = container.split(".", 1)[1]
return f"platform-core/modules/features/{featureCode}"
return f"platform-core/modules/{container}"
def _getPlatformContainer(moduleId: str) -> Optional[str]:
if moduleId == "platform-core.app":
return "app"
if not moduleId.startswith("platform-core."):
return None
parts = moduleId.replace("platform-core.", "").split(".")
if not parts:
return "app"
if parts[0] in ("tests", "scripts") or parts[0].startswith("script_"):
return None
if parts[0] != "modules" or len(parts) < 2:
return "app"
container = parts[1]
if container == "features" and len(parts) > 2:
return f"features.{parts[2]}"
return container
def _resolvePlatformRelativeImport(currentFile: Path, importNode: ast.ImportFrom) -> Optional[str]:
dotCount = importNode.level
moduleSuffix = importNode.module or ""
currentDir = currentFile.parent
baseDir = currentDir
for _ in range(dotCount - 1):
baseDir = baseDir.parent
if moduleSuffix:
candidate = baseDir / Path(moduleSuffix.replace(".", os.sep))
else:
candidate = baseDir
pyFile = candidate.with_suffix(".py")
if pyFile.exists():
return _platformModuleId(pyFile)
initFile = candidate / "__init__.py"
if initFile.exists():
return _platformModuleId(initFile)
rel = candidate.relative_to(PLATFORM_ROOT) if candidate.is_relative_to(PLATFORM_ROOT) else None
if rel is None:
return None
return "platform-core." + ".".join(rel.with_suffix("").parts)
def _resolvePlatformImportTarget(currentFile: Path, importedName: str) -> Tuple[str, bool]:
if importedName.startswith("."):
return importedName, False
if importedName.startswith("modules."):
parts = importedName.split(".")
checkPath = PLATFORM_ROOT
for part in parts:
checkPath = checkPath / part
if checkPath.with_suffix(".py").exists():
return _platformModuleId(checkPath.with_suffix(".py")), True
if checkPath.is_dir() and (checkPath / "__init__.py").exists():
return _platformModuleId(checkPath / "__init__.py"), True
return f"platform-core.{importedName.replace('.', '.')}", True
return importedName, False
class _PythonImportVisitor(ast.NodeVisitor):
def __init__(self, filePath: Path):
self.filePath = filePath
self.imports: List[ImportRecord] = []
self._inCodeScope = False
def _addImport(self, importedModule: str, isInternal: bool) -> None:
position = "code" if self._inCodeScope else "header"
sourceContainer = _getPlatformContainer(_platformModuleId(self.filePath))
targetContainer = _getPlatformContainer(importedModule) if isInternal else None
self.imports.append(
ImportRecord(
importedModule=importedModule,
position=position,
isInternal=isInternal,
sourceContainer=sourceContainer,
targetContainer=targetContainer,
)
)
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
previous = self._inCodeScope
self._inCodeScope = True
self.generic_visit(node)
self._inCodeScope = previous
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
previous = self._inCodeScope
self._inCodeScope = True
self.generic_visit(node)
self._inCodeScope = previous
def visit_ClassDef(self, node: ast.ClassDef) -> None:
previous = self._inCodeScope
self._inCodeScope = True
self.generic_visit(node)
self._inCodeScope = previous
def visit_Import(self, node: ast.Import) -> None:
for alias in node.names:
resolved, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name)
self._addImport(resolved, isInternal)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
if node.level > 0:
resolved = _resolvePlatformRelativeImport(self.filePath, node)
if resolved:
self._addImport(resolved, True)
else:
suffix = node.module or ""
display = ("." * node.level) + suffix
self._addImport(f"(relative-unresolved) {display}", False)
return
if not node.module:
return
resolved, isInternal = _resolvePlatformImportTarget(self.filePath, node.module)
self._addImport(resolved, isInternal)
def _analyzePythonFile(filePath: Path) -> Optional[ModuleAnalysis]:
container = _getPlatformContainer(_platformModuleId(filePath))
if container is None:
return None
try:
source = filePath.read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(filePath))
except (SyntaxError, UnicodeDecodeError) as error:
print(f"WARN parse failed: {filePath}: {error}")
return None
visitor = _PythonImportVisitor(filePath)
visitor.visit(tree)
moduleId = _platformModuleId(filePath)
return ModuleAnalysis(
context="platform",
moduleId=moduleId,
filePath=filePath,
container=container,
containerPath=_platformContainerPath(container),
imports=visitor.imports,
)
def _collectPlatformModules() -> List[ModuleAnalysis]:
modules: List[ModuleAnalysis] = []
scanRoots = [PLATFORM_ROOT / "modules", PLATFORM_ROOT / "app.py"]
pyFiles: List[Path] = []
if scanRoots[1].exists():
pyFiles.append(scanRoots[1])
for root, dirs, files in os.walk(scanRoots[0]):
dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES]
for fileName in files:
if fileName.endswith(".py"):
pyFiles.append(Path(root) / fileName)
for filePath in pyFiles:
analysis = _analyzePythonFile(filePath)
if analysis:
modules.append(analysis)
return modules
# ---------------------------------------------------------------------------
# UI (TypeScript)
# ---------------------------------------------------------------------------
TS_IMPORT_FROM_RE = re.compile(
r"""(?:^|\n)\s*(?:import|export)\s+(?:type\s+)?(?:[\w*\s{},\n\r]+?\sfrom\s+)?['"]([^'"]+)['"]""",
re.MULTILINE,
)
TS_SIDE_EFFECT_IMPORT_RE = re.compile(
r"""(?:^|\n)\s*import\s+['"]([^'"]+)['"]\s*;""",
re.MULTILINE,
)
TS_DYNAMIC_IMPORT_RE = re.compile(r"""import\s*\(\s*['"]([^'"]+)['"]\s*\)""")
def _uiModuleId(filePath: Path) -> str:
rel = filePath.relative_to(UI_ROOT / "src")
if filePath.name == "index.ts" or filePath.name == "index.tsx":
parts = rel.parent.parts
else:
parts = rel.with_suffix("").parts
return "ui-nyla.src." + ".".join(parts)
def _uiContainerPath(container: str) -> str:
if container.startswith("pages."):
suffix = container.split(".", 1)[1]
if suffix in ("admin", "basedata", "billing", "settings", "workflowAutomation"):
return f"ui-nyla/src/pages/{suffix}"
return f"ui-nyla/src/pages/views/{suffix}"
if container.startswith("components."):
suffix = container.split(".", 1)[1]
return f"ui-nyla/src/components/{suffix}"
return f"ui-nyla/src/{container}"
def _getUiContainer(moduleId: str) -> Optional[str]:
if not moduleId.startswith("ui-nyla.src."):
return None
parts = moduleId.replace("ui-nyla.src.", "").split(".")
if not parts:
return None
top = parts[0]
if top == "test":
return None
if top == "pages":
if len(parts) >= 3 and parts[1] == "views":
return f"pages.{parts[2]}"
if len(parts) >= 2:
return f"pages.{parts[1]}"
return "pages"
if top == "components" and len(parts) >= 2:
return f"components.{parts[1]}"
return top
def _resolveUiImport(currentFile: Path, spec: str) -> Tuple[str, bool]:
if spec.startswith("."):
resolvedPath = (currentFile.parent / spec).resolve()
candidates = [
resolvedPath,
resolvedPath.with_suffix(".ts"),
resolvedPath.with_suffix(".tsx"),
resolvedPath / "index.ts",
resolvedPath / "index.tsx",
]
for candidate in candidates:
if candidate.exists() and candidate.is_relative_to(UI_ROOT / "src"):
return _uiModuleId(candidate), True
relDisplay = spec
return relDisplay, False
return spec, False
def _findTsImportPosition(source: str, matchStart: int) -> str:
depth = 0
inFunction = False
functionDepth = 0
i = 0
while i < matchStart:
char = source[i]
if char == "{":
depth += 1
elif char == "}":
depth = max(0, depth - 1)
if inFunction and depth < functionDepth:
inFunction = False
i += 1
lookback = source[max(0, matchStart - 400):matchStart]
if re.search(r"(?:function\s*\w*\s*\(|=>\s*\{|(?:async\s+)?function\s+\w+\s*\()", lookback):
tail = lookback[lookback.rfind("\n") + 1:]
if "=>" in tail or "function" in tail:
bracePos = source.find("{", max(0, matchStart - 120), matchStart)
if bracePos >= 0:
return "code"
return "header" if depth == 0 and not inFunction else "code"
def _analyzeTypeScriptFile(filePath: Path) -> Optional[ModuleAnalysis]:
container = _getUiContainer(_uiModuleId(filePath))
if container is None:
return None
try:
source = filePath.read_text(encoding="utf-8")
except UnicodeDecodeError as error:
print(f"WARN read failed: {filePath}: {error}")
return None
imports: List[ImportRecord] = []
seen: Set[Tuple[str, str, str]] = set()
def _register(spec: str, position: str) -> None:
resolved, isInternal = _resolveUiImport(filePath, spec)
key = (resolved, position, spec)
if key in seen:
return
seen.add(key)
sourceContainer = container
targetContainer = _getUiContainer(resolved) if isInternal else None
imports.append(
ImportRecord(
importedModule=resolved,
position=position,
isInternal=isInternal,
sourceContainer=sourceContainer,
targetContainer=targetContainer,
)
)
for match in TS_IMPORT_FROM_RE.finditer(source):
position = _findTsImportPosition(source, match.start())
_register(match.group(1), position)
for match in TS_SIDE_EFFECT_IMPORT_RE.finditer(source):
if match.group(1) in {m.group(1) for m in TS_IMPORT_FROM_RE.finditer(source)}:
continue
position = _findTsImportPosition(source, match.start())
_register(match.group(1), position)
for match in TS_DYNAMIC_IMPORT_RE.finditer(source):
position = _findTsImportPosition(source, match.start())
_register(match.group(1), position)
moduleId = _uiModuleId(filePath)
return ModuleAnalysis(
context="ui",
moduleId=moduleId,
filePath=filePath,
container=container,
containerPath=_uiContainerPath(container),
imports=imports,
)
def _collectUiModules() -> List[ModuleAnalysis]:
srcRoot = UI_ROOT / "src"
modules: List[ModuleAnalysis] = []
for filePath in srcRoot.rglob("*"):
if not filePath.is_file():
continue
if filePath.suffix not in (".ts", ".tsx"):
continue
rel = filePath.relative_to(srcRoot).as_posix()
if rel.startswith("test/") or rel.endswith(".test.ts") or rel.endswith(".test.tsx"):
continue
analysis = _analyzeTypeScriptFile(filePath)
if analysis:
modules.append(analysis)
return modules
# ---------------------------------------------------------------------------
# Markdown output
# ---------------------------------------------------------------------------
def _renderModuleMarkdown(module: ModuleAnalysis) -> str:
lines = [
f"# Module Import Analysis: `{module.moduleId}`",
"",
f"- **Kontext:** {module.context}",
f"- **Container:** `{module.container}`",
f"- **Container-Pfad:** `{module.containerPath}`",
f"- **Datei:** `{module.filePath.relative_to(REPO_ROOT).as_posix()}`",
f"- **Import-Anzahl:** {len(module.imports)}",
"",
"## Imports",
"",
"| Modul | Position | Intern |",
"|-------|----------|--------|",
]
for item in sorted(module.imports, key=lambda x: (x.importedModule, x.position)):
internal = "ja" if item.isInternal else "nein"
lines.append(f"| `{item.importedModule}` | {item.position} | {internal} |")
if not module.imports:
lines.append("| _keine_ | | |")
lines.append("")
return "\n".join(lines)
@dataclass
class ContainerStats:
container: str
containerPath: str
importsFrom: Dict[str, int] = field(default_factory=lambda: defaultdict(int))
exportedTo: Dict[str, int] = field(default_factory=lambda: defaultdict(int))
mixedWith: Dict[str, Tuple[int, int]] = field(default_factory=dict)
def _buildContainerStats(modules: Iterable[ModuleAnalysis]) -> Dict[str, ContainerStats]:
statsByContainer: Dict[str, ContainerStats] = {}
for module in modules:
if module.container not in statsByContainer:
statsByContainer[module.container] = ContainerStats(
container=module.container,
containerPath=module.containerPath,
)
for item in module.imports:
if not item.isInternal:
continue
if not item.sourceContainer or not item.targetContainer:
continue
if item.sourceContainer == item.targetContainer:
continue
stats = statsByContainer[item.sourceContainer]
stats.importsFrom[item.targetContainer] += 1
targetStats = statsByContainer.get(item.targetContainer)
if targetStats is None:
targetStats = ContainerStats(
container=item.targetContainer,
containerPath=_platformContainerPath(item.targetContainer)
if module.context == "platform"
else _uiContainerPath(item.targetContainer),
)
statsByContainer[item.targetContainer] = targetStats
targetStats.exportedTo[item.sourceContainer] += 1
for containerName, stats in statsByContainer.items():
mixed: Dict[str, Tuple[int, int]] = {}
for other, outCount in stats.importsFrom.items():
inCount = stats.exportedTo.get(other, 0)
if inCount > 0:
mixed[other] = (outCount, inCount)
stats.mixedWith = mixed
return statsByContainer
def _renderContainerMarkdown(context: str, stats: ContainerStats) -> str:
importsTotal = sum(stats.importsFrom.values())
exportsTotal = sum(stats.exportedTo.values())
mixedTotal = sum(min(pair[0], pair[1]) for pair in stats.mixedWith.values())
lines = [
f"# Container Import Analysis: `{stats.container}`",
"",
f"- **Kontext:** {context}",
f"- **Container-Pfad:** `{stats.containerPath}`",
"",
"## Imports aus anderen Containern",
"",
f"- **Anzahl:** {importsTotal}",
f"- **Container ({len(stats.importsFrom)}):** "
+ (", ".join(f"`{name}` ({count})" for name, count in sorted(stats.importsFrom.items())) or "_keine_"),
"",
"## Exports zu anderen Containern",
"",
f"- **Anzahl:** {exportsTotal}",
f"- **Container ({len(stats.exportedTo)}):** "
+ (", ".join(f"`{name}` ({count})" for name, count in sorted(stats.exportedTo.items())) or "_keine_"),
"",
"## Cross (mixed Import/Export)",
"",
f"- **Anzahl bidirektionaler Paare:** {len(stats.mixedWith)}",
f"- **Mindest-Wechselzahl (min je Richtung):** {mixedTotal}",
]
if stats.mixedWith:
lines.extend(["", "| Container | Importe hinaus | Importe herein |", "|-----------|----------------|----------------|"])
for other, (outCount, inCount) in sorted(stats.mixedWith.items()):
lines.append(f"| `{other}` | {outCount} | {inCount} |")
else:
lines.append("- **Container:** _keine_")
lines.append("")
return "\n".join(lines)
def _renderReadme(platformModules: int, uiModules: int, platformContainers: int, uiContainers: int) -> str:
return f"""# PORTA Import-Analyse
Generiert am {date.today().isoformat()} durch `platform-core/scripts/script_analyze_porta_imports.py`.
## Umfang
| Kontext | Module | Container |
|---------|--------|-----------|
| platform | {platformModules} | {platformContainers} |
| ui | {uiModules} | {uiContainers} |
## Struktur
- `import-analysis-platform.md` — konsolidierte Platform-Übersicht (Tabelle)
- `import-analysis-platform-modules.md` — Modul-Graph: Gegenimporte, Loops, lazy Imports
- `import-analysis-ui.md` — konsolidierte UI-Übersicht (Tabelle)
- `platform/modules/` — ein Markdown pro Python-Modul (alle Imports inkl. lazy)
- `platform/containers/` — aggregierte Container-Statistik
- `platform/container-network.drawio` — Container-Vernetzung (schwarz=einweg, rot=mixed)
- `ui/modules/` — ein Markdown pro TS/TSX-Modul
- `ui/containers/` — aggregierte Container-Statistik
- `ui/container-network.drawio` — Container-Vernetzung
- `container-network.drawio` — kombiniert (2 Diagramm-Tabs: platform + ui)
## Position
- `header` — Import auf Modulebene (Top-Level)
- `code` — Import innerhalb von Funktion/Klasse oder dynamisch (`import()`)
## Regenerieren
```bash
python platform-core/scripts/script_analyze_porta_imports.py
```
"""
# ---------------------------------------------------------------------------
# draw.io
# ---------------------------------------------------------------------------
CONTAINER_COLORS = {
"app": "#dae8fc",
"aicore": "#d5e8d4",
"auth": "#ffe6cc",
"connectors": "#e1d5e7",
"datamodels": "#fff2cc",
"interfaces": "#f8cecc",
"routes": "#d0cee2",
"security": "#fad7ac",
"serviceCenter": "#b1ddf0",
"shared": "#f0fff0",
"workflows": "#f5f5f5",
"workflowAutomation": "#e6d0de",
"system": "#cce5ff",
"dbHelpers": "#fff0f5",
"nodeCatalog": "#f5fffa",
"pages": "#dae8fc",
"components": "#d5e8d4",
"hooks": "#ffe6cc",
"contexts": "#e1d5e7",
"api": "#fff2cc",
"layouts": "#f8cecc",
"providers": "#d0cee2",
"config": "#fad7ac",
"utils": "#b1ddf0",
"types": "#f0fff0",
"locales": "#f5f5f5",
"stores": "#e2efda",
"styles": "#fce5cd",
}
def _aggregateContainerEdges(statsByContainer: Dict[str, ContainerStats]) -> Dict[Tuple[str, str], Tuple[int, int, bool]]:
pairCounts: Dict[Tuple[str, str], Tuple[int, int]] = {}
for stats in statsByContainer.values():
for target, count in stats.importsFrom.items():
key = (stats.container, target)
outCount, inCount = pairCounts.get(key, (0, 0))
pairCounts[key] = (outCount + count, inCount)
edges: Dict[Tuple[str, str], Tuple[int, int, bool]] = {}
processed: Set[Tuple[str, str]] = set()
for (source, target), (forward, _) in list(pairCounts.items()):
pairKey = tuple(sorted((source, target)))
if pairKey in processed:
continue
processed.add(pairKey)
a, b = pairKey
aToB = pairCounts.get((a, b), (0, 0))[0]
bToA = pairCounts.get((b, a), (0, 0))[0]
if aToB == 0 and bToA == 0:
continue
if aToB > 0 and bToA > 0:
edges[(a, b)] = (aToB, bToA, True)
elif aToB > 0:
edges[(a, b)] = (aToB, 0, False)
else:
edges[(b, a)] = (bToA, 0, False)
return edges
def _generateDrawio(context: str, statsByContainer: Dict[str, ContainerStats]) -> str:
containers = sorted(statsByContainer.keys())
edges = _aggregateContainerEdges(statsByContainer)
centerX = 700
centerY = 550
radius = 430
nodeWidth = 170
nodeHeight = 62
containerPositions: Dict[str, Tuple[int, int]] = {}
for index, container in enumerate(containers):
angle = (2 * math.pi * index / max(len(containers), 1)) - math.pi / 2
x = int(centerX + radius * math.cos(angle) - nodeWidth / 2)
y = int(centerY + radius * math.sin(angle) - nodeHeight / 2)
containerPositions[container] = (x, y)
cells: List[str] = []
for container in containers:
x, y = containerPositions[container]
base = container.split(".")[0]
color = CONTAINER_COLORS.get(base, "#ffffff")
label = f"{container}\\n({sum(statsByContainer[container].importsFrom.values())} out / "
label += f"{sum(statsByContainer[container].exportedTo.values())} in)"
cellId = f"container_{container.replace('.', '_')}"
cells.append(
f""" <mxCell id="{cellId}" value="{html.escape(label)}" """
f"""style="rounded=1;whiteSpace=wrap;html=1;fillColor={color};strokeColor=#666666;fontStyle=1;fontSize=11;" """
f"""vertex="1" parent="1">
<mxGeometry x="{x}" y="{y}" width="{nodeWidth}" height="{nodeHeight}" as="geometry" />
</mxCell>"""
)
edgeId = 1000
for (source, target), (forward, backward, isMixed) in sorted(edges.items(), key=lambda item: -(item[1][0] + item[1][1])):
sourceId = f"container_{source.replace('.', '_')}"
targetId = f"container_{target.replace('.', '_')}"
if isMixed:
label = f"{forward} / {backward}"
strokeColor = "#CC0000"
else:
label = str(forward)
strokeColor = "#000000"
strokeWidth = min(1 + (forward + backward) // 15, 6)
cells.append(
f""" <mxCell id="edge_{edgeId}" value="{html.escape(label)}" """
f"""style="edgeStyle=none;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;"""
f"""endArrow=block;endFill=1;strokeWidth={strokeWidth};strokeColor={strokeColor};"""
f"""fontSize=10;labelBackgroundColor=#ffffff;fontStyle=1;" """
f"""edge="1" parent="1" source="{sourceId}" target="{targetId}">
<mxGeometry relative="1" as="geometry" />
</mxCell>"""
)
edgeId += 1
innerXml = f""" <mxGraphModel dx="1434" dy="780" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1600" pageHeight="1200" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
{chr(10).join(cells)}
</root>
</mxGraphModel>"""
return _wrapDrawioDiagram(context, innerXml)
def _wrapDrawioDiagram(context: str, innerXml: str) -> str:
return f"""<?xml version="1.0" encoding="UTF-8"?>
<mxfile host="app.diagrams.net" modified="{date.today().isoformat()}T00:00:00.000Z" agent="script_analyze_porta_imports.py" version="21.0.0" type="device">
<diagram id="{context}-container-network" name="{context} container imports">
{innerXml}
</diagram>
</mxfile>
"""
def _extractDrawioDiagramBody(drawioXml: str) -> str:
start = drawioXml.index("<mxGraphModel")
end = drawioXml.index("</diagram>")
return drawioXml[start:end]
def _combineDrawioFiles(platformDrawio: str, uiDrawio: str) -> str:
return f"""<?xml version="1.0" encoding="UTF-8"?>
<mxfile host="app.diagrams.net" modified="{date.today().isoformat()}T00:00:00.000Z" agent="script_analyze_porta_imports.py" version="21.0.0" type="device">
<diagram id="platform-container-network" name="platform container imports">
{_extractDrawioDiagramBody(platformDrawio)}
</diagram>
<diagram id="ui-container-network" name="ui container imports">
{_extractDrawioDiagramBody(uiDrawio)}
</diagram>
</mxfile>
"""
SUMMARY_FILE_PLATFORM = "import-analysis-platform.md"
SUMMARY_FILE_UI = "import-analysis-ui.md"
def _renderConsolidatedSummary(
title: str,
context: str,
detailFolder: str,
statsByContainer: Dict[str, ContainerStats],
diagramPath: str,
) -> str:
lines = [
f"# {title}",
"",
f"- **Kontext:** {context}",
f"- **Generiert:** {date.today().isoformat()}",
f"- **Detail-Dateien:** `{detailFolder}/`",
"",
"## Container",
"",
"| Container | Imports out | Exports in | Mixed | Detail |",
"|-----------|------------:|-----------:|------:|--------|",
]
for containerName in sorted(statsByContainer.keys()):
stats = statsByContainer[containerName]
importsOut = sum(stats.importsFrom.values())
exportsIn = sum(stats.exportedTo.values())
mixedCount = len(stats.mixedWith)
detailLink = f"[Detail]({detailFolder}/containers/{_sanitizeFileName(containerName)}.md)"
lines.append(
f"| `{containerName}` | {importsOut} | {exportsIn} | {mixedCount} | {detailLink} |"
)
lines.extend(
[
"",
f"Diagramm: [{diagramPath}]({diagramPath})",
"",
]
)
return "\n".join(lines)
def _writeContextOutput(context: str, modules: List[ModuleAnalysis]) -> Tuple[int, str]:
contextRoot = OUTPUT_ROOT / context
modulesDir = contextRoot / "modules"
containersDir = contextRoot / "containers"
for module in modules:
fileName = _sanitizeFileName(module.moduleId) + ".md"
_writeText(modulesDir / fileName, _renderModuleMarkdown(module))
statsByContainer = _buildContainerStats(modules)
for containerName, stats in sorted(statsByContainer.items()):
fileName = _sanitizeFileName(containerName) + ".md"
_writeText(containersDir / fileName, _renderContainerMarkdown(context, stats))
drawio = _generateDrawio(context, statsByContainer)
_writeText(contextRoot / "container-network.drawio", drawio)
return len(statsByContainer), drawio
def main() -> None:
print("Analyzing platform-core (Python)...")
platformModules = _collectPlatformModules()
print(f" modules: {len(platformModules)}")
print("Analyzing ui-nyla (TypeScript)...")
uiModules = _collectUiModules()
print(f" modules: {len(uiModules)}")
platformContainerCount, platformDrawio = _writeContextOutput("platform", platformModules)
uiContainerCount, uiDrawio = _writeContextOutput("ui", uiModules)
combinedDrawio = _combineDrawioFiles(platformDrawio, uiDrawio)
_writeText(OUTPUT_ROOT / "container-network.drawio", combinedDrawio)
readme = _renderReadme(
platformModules=len(platformModules),
uiModules=len(uiModules),
platformContainers=platformContainerCount,
uiContainers=uiContainerCount,
)
_writeText(OUTPUT_ROOT / "README.md", readme)
platformStats = _buildContainerStats(platformModules)
platformSummary = _renderConsolidatedSummary(
title="Import-Analyse Platform Core",
context="platform",
detailFolder="platform",
statsByContainer=platformStats,
diagramPath="platform/container-network.drawio",
)
_writeText(OUTPUT_ROOT / SUMMARY_FILE_PLATFORM, platformSummary)
uiStats = _buildContainerStats(uiModules)
uiSummary = _renderConsolidatedSummary(
title="Import-Analyse UI Nyla",
context="ui",
detailFolder="ui",
statsByContainer=uiStats,
diagramPath="ui/container-network.drawio",
)
_writeText(OUTPUT_ROOT / SUMMARY_FILE_UI, uiSummary)
print(f"\nOutput written to: {OUTPUT_ROOT}")
print(f" platform containers: {platformContainerCount}")
print(f" ui containers: {uiContainerCount}")
if __name__ == "__main__":
main()

165
scripts/script_remove_redundant_platform_imports.py Normal file
View file

@ -0,0 +1,165 @@
# Copyright (c) 2026 PowerOn AG
# All rights reserved.
"""
Remove redundant lazy imports in platform-core when the same internal module
is already imported at module header level.
Usage:
python platform-core/scripts/script_remove_redundant_platform_imports.py
python platform-core/scripts/script_remove_redundant_platform_imports.py --dry-run
"""
from __future__ import annotations
import argparse
import ast
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from script_analyze_porta_imports import ( # noqa: E402
PLATFORM_ROOT,
SKIP_DIR_NAMES,
_getPlatformContainer,
_platformModuleId,
_resolvePlatformImportTarget,
_resolvePlatformRelativeImport,
)
class _RedundantImportFinder(ast.NodeVisitor):
def __init__(self, filePath: Path):
self.filePath = filePath
self.headerTargets: Set[str] = set()
self.linesToRemove: Set[int] = set()
self._scopeDepth = 0
def _resolveImportNode(self, node: ast.Import | ast.ImportFrom) -> List[Tuple[str, bool]]:
resolved: List[Tuple[str, bool]] = []
if isinstance(node, ast.ImportFrom):
if node.level > 0:
target = _resolvePlatformRelativeImport(self.filePath, node)
if target:
resolved.append((target, True))
return resolved
if not node.module:
return resolved
target, isInternal = _resolvePlatformImportTarget(self.filePath, node.module)
resolved.append((target, isInternal))
return resolved
for alias in node.names:
target, isInternal = _resolvePlatformImportTarget(self.filePath, alias.name)
resolved.append((target, isInternal))
return resolved
def _handleImportNode(self, node: ast.Import | ast.ImportFrom) -> None:
for target, isInternal in self._resolveImportNode(node):
if not isInternal or not target.startswith("platform-core."):
continue
if self._scopeDepth == 0:
self.headerTargets.add(target)
elif target in self.headerTargets:
endLine = getattr(node, "end_lineno", None) or node.lineno
for lineNo in range(node.lineno, endLine + 1):
self.linesToRemove.add(lineNo)
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
self._scopeDepth += 1
self.generic_visit(node)
self._scopeDepth -= 1
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
self._scopeDepth += 1
self.generic_visit(node)
self._scopeDepth -= 1
def visit_ClassDef(self, node: ast.ClassDef) -> None:
self._scopeDepth += 1
self.generic_visit(node)
self._scopeDepth -= 1
def visit_Import(self, node: ast.Import) -> None:
self._handleImportNode(node)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
self._handleImportNode(node)
def _moduleIdToFilePath(moduleId: str) -> Path:
rel = moduleId.replace("platform-core.", "")
parts = rel.split(".")
candidate = PLATFORM_ROOT.joinpath(*parts).with_suffix(".py")
if candidate.exists():
return candidate
initFile = PLATFORM_ROOT.joinpath(*parts, "__init__.py")
return initFile
def _collectPythonFiles() -> List[Path]:
pyFiles: List[Path] = []
appFile = PLATFORM_ROOT / "app.py"
if appFile.exists():
pyFiles.append(appFile)
for root, dirs, files in os.walk(PLATFORM_ROOT / "modules"):
dirs[:] = [d for d in dirs if d not in SKIP_DIR_NAMES]
for fileName in files:
if fileName.endswith(".py"):
pyFiles.append(Path(root) / fileName)
return pyFiles
def _removeLines(filePath: Path, linesToRemove: Set[int], dryRun: bool) -> int:
if not linesToRemove:
return 0
lines = filePath.read_text(encoding="utf-8").splitlines(keepends=True)
newLines = [line for index, line in enumerate(lines, start=1) if index not in linesToRemove]
if not dryRun:
filePath.write_text("".join(newLines), encoding="utf-8")
return len(linesToRemove)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
totalRemoved = 0
filesChanged = 0
details: List[Tuple[str, int]] = []
for filePath in _collectPythonFiles():
moduleId = _platformModuleId(filePath)
if _getPlatformContainer(moduleId) is None:
continue
try:
tree = ast.parse(filePath.read_text(encoding="utf-8"), filename=str(filePath))
except (SyntaxError, UnicodeDecodeError) as error:
print(f"WARN skip {filePath}: {error}")
continue
finder = _RedundantImportFinder(filePath)
finder.visit(tree)
if not finder.linesToRemove:
continue
removed = _removeLines(filePath, finder.linesToRemove, args.dry_run)
totalRemoved += removed
filesChanged += 1
rel = filePath.relative_to(PLATFORM_ROOT.parent).as_posix()
details.append((rel, removed))
action = "would remove" if args.dry_run else "removed"
print(f"{action} {removed} from {rel}")
print(f"\nFiles touched: {filesChanged}")
print(f"Import lines {('would be ' if args.dry_run else '')}removed: {totalRemoved}")
if __name__ == "__main__":
main()

36
tests/demo/README.md
View file

@ -2,33 +2,33 @@
Automated tests for the investor demo configuration.
## Prerequisites
## SAFETY RULE (critical)
1. Gateway DB must be running and accessible
2. Demo config must be loaded first: Admin UI → `/admin/demo-config` → Load "Investor Demo April 2026"
3. RMA credentials must be set in `gateway/config.ini`
Tests in this suite MUST be strictly read-only towards the database.
Pytest runs against the **real dev database** (there is no separate test DB).
Tests must NEVER load or remove demo configs — neither via direct module
calls (`cfg.load()` / `cfg.remove()`) nor via HTTP calls to
`/api/admin/demo-config/<code>/load|remove` (not even to assert 401/403).
Background: on 2026-06-09 a demo test reloaded `investor-demo-2026` during a
pytest run. The demo mandates (HappyLife AG, Alpina Treuhand AG) were deleted
and recreated with new UUIDs, orphaning all real feature data (trustee
accounting incl. RMA connection, workflows, documents). The data had to be
recovered by remapping ~15'000 rows across 9 databases.
Demo configs are loaded/removed manually via Admin UI → `/admin/demo-config`.
## Run
```bash
cd gateway/
# All demo tests (structural, no AI calls):
cd platform-core/
pytest tests/demo/ -v
# Only bootstrap tests:
pytest tests/demo/test_demo_bootstrap.py -v
# Only UC1 trustee:
pytest tests/demo/test_demo_uc1_trustee.py -v
```
## Test files
| File | What it tests |
|------|--------------|
| `test_demo_bootstrap.py` | Idempotent load/remove, mandates, user, features, RMA, neutralization |
| `test_demo_uc1_trustee.py` | Trustee instances, RMA config, system workflow templates |
| `test_demo_uc2_realestate.py` | Workspace instances for agent demo |
| `test_demo_uc4_i18n.py` | i18n readiness, Spanish not pre-installed |
| `test_demo_neutralization.py` | Neutralization config enabled, test PDF exists |
| `test_demo_api.py` | Config auto-discovery (read-only), list endpoint rejects unauthenticated |
| `test_demo_data_files.py` | Demo data files exist in `demoData/` (filesystem only) |

16
tests/demo/test_demo_api.py
View file

@ -44,7 +44,13 @@ class TestDemoConfigDiscovery:
class TestDemoConfigApiEndpoints:
"""Test API endpoints via TestClient."""
"""Test API endpoints via TestClient.
SAFETY: Never call the load/remove endpoints here - not even to assert
401/403. Tests run against the real dev database; if auth/CSRF ever lets
a request through, demo mandates get deleted and recreated with new UUIDs,
orphaning all real feature data (happened on 2026-06-09).
"""
@pytest.fixture(scope="class")
def client(self):
@ -55,11 +61,3 @@ class TestDemoConfigApiEndpoints:
def test_listEndpointRejectsUnauthenticated(self, client):
response = client.get("/api/admin/demo-config")
assert response.status_code in (401, 403)
def test_loadEndpointRejectsUnauthenticated(self, client):
response = client.post("/api/admin/demo-config/investor-demo-2026/load")
assert response.status_code in (401, 403)
def test_removeEndpointRejectsUnauthenticated(self, client):
response = client.post("/api/admin/demo-config/investor-demo-2026/remove")
assert response.status_code in (401, 403)