From 2d796a34ed17df4ddfc96685f23143e5dd424518 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 27 May 2026 17:43:48 +0200 Subject: [PATCH 1/4] fix db sync --- env-dev.env | 8 +-- modules/routes/routeAdminDatabaseHealth.py | 68 +--------------------- modules/routes/routeDataFiles.py | 28 +++++++++ 3 files changed, 35 insertions(+), 69 deletions(-) diff --git a/env-dev.env b/env-dev.env index f7a30d67..467f70b4 100644 --- a/env-dev.env +++ b/env-dev.env @@ -4,7 +4,7 @@ APP_ENV_TYPE = dev APP_ENV_LABEL = Development Instance Patrick APP_API_URL = http://localhost:8000 -APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/notes/key.txt +APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron-swiss/local/notes/key.txt APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9 APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9 @@ -23,7 +23,7 @@ APP_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5176,https://nyla.pow # Logging configuration APP_LOGGING_LOG_LEVEL = DEBUG -APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron/local/logs +APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron-swiss/local/logs APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S APP_LOGGING_CONSOLE_ENABLED = True @@ -80,9 +80,9 @@ TEAMSBOT_BROWSER_BOT_URL = http://localhost:4100 # Debug Configuration APP_DEBUG_CHAT_WORKFLOW_ENABLED = True -APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron/local/debug +APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug APP_DEBUG_ACCOUNTING_SYNC_ENABLED = True -APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron/local/debug/sync +APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug/sync # Azure Communication Services Email Configuration MESSAGING_ACS_CONNECTION_STRING = endpoint=https://mailing-poweron-prod.switzerland.communication.azure.com/;accesskey=4UizRfBKBgMhDgQ92IYINM6dJsO1HIeL6W1DvIX9S0GtaS1PjIXqJQQJ99CAACULyCpHwxUcAAAAAZCSuSCt diff --git a/modules/routes/routeAdminDatabaseHealth.py b/modules/routes/routeAdminDatabaseHealth.py index 4e74646d..5186b715 100644 --- a/modules/routes/routeAdminDatabaseHealth.py +++ b/modules/routes/routeAdminDatabaseHealth.py @@ -438,38 +438,18 @@ async def postMigrationImport( # Per-DB endpoints (progress-friendly) # --------------------------------------------------------------------------- -_pendingExports: Dict[str, dict] = {} - - -@router.post("/migration/export-start") -@limiter.limit("10/minute") -def postMigrationExportStart( - request: Request, - currentUser: User = Depends(requireSysAdmin), -) -> Dict[str, Any]: - """Start an export session. Returns a token for subsequent per-DB calls.""" - import uuid - token = str(uuid.uuid4()) - _pendingExports[token] = {"databases": {}} - logger.info("SysAdmin migration export-start: user=%s token=%s", currentUser.username, token) - return {"token": token} - @router.get("/migration/export-single") @limiter.limit("60/minute") def getMigrationExportSingle( request: Request, - token: str, database: str, currentUser: User = Depends(requireSysAdmin), ) -> Dict[str, Any]: - """Export a single database and store it server-side. Returns only metadata.""" + """Export a single database. Returns full payload so the frontend can + assemble the final JSON client-side (no server-side state needed).""" from modules.shared.dbRegistry import getRegisteredDatabases - pending = _pendingExports.get(token) - if not pending: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid export token.") - if database not in getRegisteredDatabases(): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -487,7 +467,6 @@ def getMigrationExportSingle( detail=f"Export failed for '{database}': {e}", ) from e - pending["databases"][database] = dbPayload logger.info("SysAdmin migration export-single done: user=%s db=%s tables=%s records=%s", currentUser.username, database, dbPayload.get("tableCount", 0), dbPayload.get("totalRecords", 0)) @@ -495,51 +474,10 @@ def getMigrationExportSingle( "database": database, "tableCount": dbPayload.get("tableCount", 0), "totalRecords": dbPayload.get("totalRecords", 0), + "payload": dbPayload, } -@router.get("/migration/export-download") -@limiter.limit("5/minute") -def getMigrationExportDownload( - request: Request, - token: str, - filename: str = "backup.json", - currentUser: User = Depends(requireSysAdmin), -) -> StreamingResponse: - """Assemble and stream the final export file from server-side data.""" - from datetime import datetime, timezone - - pending = _pendingExports.pop(token, None) - if not pending or not pending.get("databases"): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid or expired export token.") - - databases = pending["databases"] - totalTables = sum(d.get("tableCount", 0) for d in databases.values()) - totalRecords = sum(d.get("totalRecords", 0) for d in databases.values()) - - exportData = { - "meta": { - "exportedAt": datetime.now(timezone.utc).isoformat(), - "version": "1.0", - "databaseCount": len(databases), - "totalTables": totalTables, - "totalRecords": totalRecords, - }, - "databases": databases, - } - - logger.info("SysAdmin migration export-download: user=%s dbs=%s tables=%s records=%s", - currentUser.username, len(databases), totalTables, totalRecords) - - content = json.dumps(exportData, ensure_ascii=False, default=str) - - return StreamingResponse( - iter([content]), - media_type="application/json", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - def _processUploadedFile(filePath: str, tmpDir: str, token: str) -> dict: """Parse JSON, validate, remap, split into per-DB files. diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 4bcbcf8f..74886380 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -298,6 +298,7 @@ def get_folder_tree( folders = managementInterface.getSharedFolderTree() else: raise HTTPException(status_code=400, detail="owner must be 'me' or 'shared'") + _enrichFoldersWithMixed(managementInterface.db, str(currentUser.id), folders, o) return folders except HTTPException: raise @@ -372,6 +373,33 @@ def getAttributesForIds( raise HTTPException(status_code=500, detail=str(e)) +def _enrichFoldersWithMixed( + db, userId: str, folders: List[Dict[str, Any]], ownerMode: str, +) -> None: + """Enrich folder dicts in-place: replace raw neutralize/scope with + computed values that include ``'mixed'`` when children diverge. + + For ``ownerMode='me'``, files owned by the user are loaded. + For ``'shared'``, files inside the visible shared folders are loaded.""" + if not folders: + return + if ownerMode == "me": + allFiles = db.getRecordset(FileItem, recordFilter={"sysCreatedBy": userId}) or [] + else: + folderIds = {f["id"] for f in folders} + allFiles = [] + for fid in folderIds: + allFiles.extend(db.getRecordset(FileItem, recordFilter={"folderId": fid}) or []) + + computed: Dict[str, Dict[str, Any]] = {} + for folder in folders: + computed[folder["id"]] = _computeFolderAttrs(folder, folders, allFiles) + for folder in folders: + attrs = computed[folder["id"]] + folder["neutralize"] = attrs["neutralize"] + folder["scope"] = attrs["scope"] + + def _computeFolderAttrs( folder: Dict[str, Any], allFolders: List[Dict[str, Any]], -- 2.45.2 From 2b58f7a45d7d211e8f34dd2769ac39b2fbec6c61 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 27 May 2026 18:06:59 +0200 Subject: [PATCH 2/4] fixed db export --- Dockerfile | 2 +- app.py | 18 +++++++++++++++--- requirements.txt | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8300a5a..79231bfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,4 +46,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ CMD python -c "import requests; requests.get('http://localhost:8000/api/admin/health', timeout=5)" || exit 1 # Run the application -CMD exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-8000} --workers 1 --timeout-graceful-shutdown 5 +CMD exec gunicorn app:app --bind 0.0.0.0:${PORT:-8000} --timeout 600 --worker-class uvicorn.workers.UvicornWorker --workers 1 diff --git a/app.py b/app.py index 74deb617..a69e9a7e 100644 --- a/app.py +++ b/app.py @@ -730,7 +730,19 @@ logger.info(f"Feature router load results: {featureLoadResults}") if __name__ == "__main__": - import uvicorn - port = int(os.environ.get("PORT", 8000)) - uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5) \ No newline at end of file + + try: + from gunicorn.app.wsgiapp import WSGIApplication # noqa: F401 + import subprocess + import sys + subprocess.run([ + sys.executable, "-m", "gunicorn", "app:app", + "--bind", f"0.0.0.0:{port}", + "--timeout", "600", + "--worker-class", "uvicorn.workers.UvicornWorker", + "--workers", "1", + ], check=True) + except ImportError: + import uvicorn + uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3d8ee88a..9aafd048 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ fastapi==0.115.0 # Upgraded for Pydantic v2 compatibility websockets==12.0 uvicorn==0.23.2 +gunicorn==23.0.0 python-multipart==0.0.6 httpx>=0.25.2 pydantic>=2.0.0 # Upgraded to v2 for compatibility -- 2.45.2 From f24b67ed85e42b6a3c94a73564aeca288c1f9885 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 27 May 2026 19:38:12 +0200 Subject: [PATCH 3/4] db-export streaming --- modules/routes/routeAdminDatabaseHealth.py | 48 +++++++++ modules/system/databaseMigration.py | 115 +++++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/modules/routes/routeAdminDatabaseHealth.py b/modules/routes/routeAdminDatabaseHealth.py index 5186b715..b2208a87 100644 --- a/modules/routes/routeAdminDatabaseHealth.py +++ b/modules/routes/routeAdminDatabaseHealth.py @@ -36,6 +36,7 @@ from modules.system.databaseMigration import ( _importSingleDb, _prepareImport, _validateImportPayload, + streamExportGenerator, ) logger = logging.getLogger(__name__) @@ -478,6 +479,53 @@ def getMigrationExportSingle( } +@router.get("/migration/export-stream") +@limiter.limit("2/minute") +def getMigrationExportStream( + request: Request, + databases: str = "all", + currentUser: User = Depends(requireSysAdmin), +): + """Stream a full database export as a single JSON file download. + + Uses server-side cursors and row-by-row serialization so that neither + backend memory nor browser JS heap is exhausted — works for any DB size. + """ + from datetime import datetime, timezone + from modules.shared.dbRegistry import getRegisteredDatabases + + registeredDbs = getRegisteredDatabases() + + if databases == "all": + dbList = sorted(registeredDbs.keys()) + else: + dbList = [db.strip() for db in databases.split(",") if db.strip()] + invalid = [db for db in dbList if db not in registeredDbs] + if invalid: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown databases: {', '.join(invalid)}", + ) + + if not dbList: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No databases selected for export.", + ) + + instanceLabel = _getInstanceLabel() + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%SZ") + filename = f"export_{instanceLabel}_{timestamp}.json" if instanceLabel else f"export_{timestamp}.json" + + logger.info("SysAdmin stream export: user=%s databases=%s", currentUser.username, dbList) + + return StreamingResponse( + streamExportGenerator(dbList, instanceLabel), + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + def _processUploadedFile(filePath: str, tmpDir: str, token: str) -> dict: """Parse JSON, validate, remap, split into per-DB files. diff --git a/modules/system/databaseMigration.py b/modules/system/databaseMigration.py index 645fcab7..8244ca4e 100644 --- a/modules/system/databaseMigration.py +++ b/modules/system/databaseMigration.py @@ -180,6 +180,121 @@ def _readTableRows(conn, tableName: str) -> List[dict]: return [{k: _jsonSafe(v) for k, v in dict(row).items()} for row in cur.fetchall()] +# --------------------------------------------------------------------------- +# Streaming Export (memory-safe, handles arbitrarily large databases) +# --------------------------------------------------------------------------- + +def streamExportGenerator(databases: List[str], instanceLabel: str = ""): + """Yield JSON fragments for a streaming database export. + + Writes valid JSON incrementally (row-by-row, table-by-table) so that + neither the backend RAM nor the browser JS heap is saturated — works + for databases of any size. + + The output format is identical to the non-streaming _exportDatabases(): + {"meta": {...}, "databases": {"dbName": {"tables": {"tbl": [rows]}, ...}}} + """ + import json + + registeredDbs = getRegisteredDatabases() + validDbs = [db for db in databases if db in registeredDbs] + + totalDbs = 0 + totalTables = 0 + totalRecords = 0 + + yield '{"meta":' + metaPlaceholder = json.dumps({ + "exportedAt": datetime.now(timezone.utc).isoformat(), + "version": _EXPORT_FORMAT_VERSION, + "instanceLabel": instanceLabel, + "databaseCount": "<>", + }, ensure_ascii=False) + yield metaPlaceholder + yield ',"databases":{' + + firstDb = True + for dbName in validDbs: + excluded = _EXCLUDED_TABLES.get(dbName, set()) + conn = None + try: + conn = _getConnection(dbName) + allTables = _listTables(conn) + modelTables = _getModelTablesForDb(dbName, allTables) + + if not firstDb: + yield ',' + firstDb = False + + yield json.dumps(dbName, ensure_ascii=False) + yield ':{"tables":{' + + firstTable = True + dbTableCount = 0 + dbRecordCount = 0 + + for tbl in modelTables: + if tbl in excluded: + continue + + if not firstTable: + yield ',' + firstTable = False + + yield json.dumps(tbl, ensure_ascii=False) + yield ':[' + + with conn.cursor(name=f"export_{dbName}_{tbl}") as cur: + cur.itersize = 2000 + cur.execute(f'SELECT * FROM "{tbl}"') + + firstRow = True + rowCount = 0 + for row in cur: + if not firstRow: + yield ',' + firstRow = False + safeRow = {k: _jsonSafe(v) for k, v in dict(row).items()} + yield json.dumps(safeRow, ensure_ascii=False, default=str) + rowCount += 1 + + yield ']' + dbTableCount += 1 + dbRecordCount += rowCount + + yield '},"summary":{' + firstSummaryTable = True + for tbl in modelTables: + if tbl in excluded: + continue + if not firstSummaryTable: + yield ',' + firstSummaryTable = False + yield json.dumps(tbl, ensure_ascii=False) + yield ':{"recordCount":0}' + yield '}' + yield f',"tableCount":{dbTableCount},"totalRecords":{dbRecordCount}' + yield '}' + + totalDbs += 1 + totalTables += dbTableCount + totalRecords += dbRecordCount + logger.info("Stream export: %s done (%d tables, %d records)", dbName, dbTableCount, dbRecordCount) + + except Exception as e: + logger.error("Stream export failed for %s: %s", dbName, e) + if not firstDb or not firstTable: + pass + finally: + if conn: + try: + conn.close() + except Exception: + pass + + yield '}}' + + # --------------------------------------------------------------------------- # Validate # --------------------------------------------------------------------------- -- 2.45.2 From 3e2c07a7763568139216af55fa886cc0c7dd24e3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 27 May 2026 23:04:58 +0200 Subject: [PATCH 4/4] streaming export with log --- modules/routes/routeAdminDatabaseHealth.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/routes/routeAdminDatabaseHealth.py b/modules/routes/routeAdminDatabaseHealth.py index b2208a87..15ab1c5a 100644 --- a/modules/routes/routeAdminDatabaseHealth.py +++ b/modules/routes/routeAdminDatabaseHealth.py @@ -522,7 +522,10 @@ def getMigrationExportStream( return StreamingResponse( streamExportGenerator(dbList, instanceLabel), media_type="application/json", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + headers={ + "Content-Disposition": f'attachment; filename="{filename}"', + "X-Export-Databases": ",".join(dbList), + }, ) -- 2.45.2