From afbb8177a3bf6589e46ffabdc09d4c7f9540fb21 Mon Sep 17 00:00:00 2001
From: ValueOn AG <p.motsch@poweron.swiss>
Date: Sun, 24 May 2026 17:34:24 +0200
Subject: [PATCH] swap for 2gb upload db

---
 modules/routes/routeAdminDatabaseHealth.py | 86 ++++++++++++++++------
 1 file changed, 64 insertions(+), 22 deletions(-)

diff --git a/modules/routes/routeAdminDatabaseHealth.py b/modules/routes/routeAdminDatabaseHealth.py
index 63f93996..a01c0648 100644
--- a/modules/routes/routeAdminDatabaseHealth.py
+++ b/modules/routes/routeAdminDatabaseHealth.py
@@ -26,14 +26,13 @@ from modules.system.databaseHealth import (
     _scanOrphans,
 )
 from modules.system.databaseMigration import (
-    _buildIdRemapFromPayload,
     _exportDatabases,
     _exportSingleDb,
     _getAvailableDatabases,
     _getInstanceLabel,
     _importDatabases,
     _importSingleDb,
-    _loadLiveSystemObjectIds,
+    _prepareImport,
     _validateImportPayload,
 )
 
@@ -487,9 +486,10 @@ async def postMigrationUploadImport(
     file: UploadFile = File(...),
     currentUser: User = Depends(requireSysAdmin),
 ) -> Dict[str, Any]:
-    """Upload a backup file to disk (chunked, no full RAM load), validate it,
-    and return a token + metadata for per-DB import.
+    """Upload a backup file to disk (chunked), validate, remap IDs,
+    split into per-DB temp files so the full payload doesn't stay in RAM.
     """
+    import gc
     import os
     import tempfile
     import uuid
@@ -516,7 +516,8 @@ async def postMigrationUploadImport(
             os.remove(filePath)
         raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Upload failed: {e}") from e
 
-    logger.info("SysAdmin migration upload-import: %s bytes written to disk", totalBytes)
+    logger.info("SysAdmin migration upload-import: %s bytes on disk (%.1f MB)",
+                totalBytes, totalBytes / 1024 / 1024)
 
     try:
         with open(filePath, "r", encoding="utf-8") as f:
@@ -525,22 +526,41 @@ async def postMigrationUploadImport(
         os.remove(filePath)
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Invalid JSON file: {e}") from e
 
-    from modules.system.databaseMigration import _prepareImport
+    try:
+        os.remove(filePath)
+    except OSError:
+        pass
+
     result = _prepareImport(payload)
 
-    liveIds = _loadLiveSystemObjectIds()
-    remap = _buildIdRemapFromPayload(payload, liveIds)
-    if remap:
-        logger.info("System-object ID remap: %s", remap)
-        from modules.system.databaseMigration import _remapSystemObjectIds
-        _remapSystemObjectIds(payload, remap)
+    if not result.get("valid"):
+        del payload
+        gc.collect()
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"message": "Payload validation failed", "warnings": result.get("warnings", [])},
+        )
 
-    protectedIds = list(set(liveIds.values()))
+    protectedIds = result.get("protectedIds", [])
+
+    dbFiles = {}
+    databases = payload.get("databases", {})
+    for dbName, dbData in databases.items():
+        dbPath = os.path.join(tmpDir, f"poweron_import_{token}_{dbName}.json")
+        with open(dbPath, "w", encoding="utf-8") as dbF:
+            json.dump(dbData, dbF, ensure_ascii=False, default=str)
+        dbFiles[dbName] = dbPath
+
+    del payload
+    del databases
+    gc.collect()
+
+    logger.info("SysAdmin migration upload-import: split into %d per-DB files, payload freed",
+                len(dbFiles))
 
     _pendingImports[token] = {
-        "payload": payload,
+        "dbFiles": dbFiles,
         "protectedIds": protectedIds,
-        "filePath": filePath,
     }
 
     return {
@@ -566,6 +586,8 @@ def postMigrationImportSingle(
 
     Body: ``{token, database, mode}``
     """
+    import os
+
     token = body.get("token", "")
     database = body.get("database", "")
     mode = body.get("mode", "merge")
@@ -577,10 +599,29 @@ def postMigrationImportSingle(
     if not pending:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid or expired import token.")
 
+    dbFiles = pending.get("dbFiles", {})
+    dbFilePath = dbFiles.get(database)
+    if not dbFilePath or not os.path.exists(dbFilePath):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"No data for database '{database}'.",
+        )
+
     logger.info("SysAdmin migration import-single: user=%s db=%s mode=%s", currentUser.username, database, mode)
 
     try:
-        result = _importSingleDb(pending["payload"], database, mode, pending["protectedIds"])
+        with open(dbFilePath, "r", encoding="utf-8") as f:
+            dbData = json.load(f)
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to read import data for '{database}': {e}",
+        ) from e
+
+    payload = {"databases": {database: dbData}}
+
+    try:
+        result = _importSingleDb(payload, database, mode, pending["protectedIds"])
     except Exception as e:
         logger.error("Import-single failed for %s: %s", database, e)
         raise HTTPException(
@@ -598,14 +639,15 @@ def postMigrationImportDone(
     body: dict,
     currentUser: User = Depends(requireSysAdmin),
 ) -> Dict[str, Any]:
-    """Clean up the server-side payload cache and temp file."""
+    """Clean up the per-DB temp files."""
     import os
 
     token = body.get("token", "")
     pending = _pendingImports.pop(token, None)
-    if pending and pending.get("filePath"):
-        try:
-            os.remove(pending["filePath"])
-        except OSError:
-            pass
+    if pending:
+        for dbPath in pending.get("dbFiles", {}).values():
+            try:
+                os.remove(dbPath)
+            except OSError:
+                pass
     return {"ok": True}