diff --git a/modules/routes/routeAdminDatabaseHealth.py b/modules/routes/routeAdminDatabaseHealth.py index a01c0648..b035eeda 100644 --- a/modules/routes/routeAdminDatabaseHealth.py +++ b/modules/routes/routeAdminDatabaseHealth.py @@ -479,6 +479,47 @@ def getMigrationExportDownload( ) +def _processUploadedFile(filePath: str, tmpDir: str, token: str) -> dict: + """Parse JSON, validate, remap, split into per-DB files. + + Runs in a thread pool to avoid blocking the asyncio event loop + during the CPU-heavy json.load() of large (500+ MB) files. + """ + import gc + import os + + with open(filePath, "r", encoding="utf-8") as f: + payload = json.load(f) + + try: + os.remove(filePath) + except OSError: + pass + + result = _prepareImport(payload) + + if not result.get("valid"): + del payload + gc.collect() + return {"result": result, "dbFiles": {}} + + protectedIds = result.get("protectedIds", []) + + dbFiles = {} + databases = payload.get("databases", {}) + for dbName, dbData in databases.items(): + dbPath = os.path.join(tmpDir, f"poweron_import_{token}_{dbName}.json") + with open(dbPath, "w", encoding="utf-8") as dbF: + json.dump(dbData, dbF, ensure_ascii=False, default=str) + dbFiles[dbName] = dbPath + + del payload + del databases + gc.collect() + + return {"result": result, "dbFiles": dbFiles, "protectedIds": protectedIds} + + @router.post("/migration/upload-import") @limiter.limit("5/minute") async def postMigrationUploadImport( @@ -489,7 +530,7 @@ async def postMigrationUploadImport( """Upload a backup file to disk (chunked), validate, remap IDs, split into per-DB temp files so the full payload doesn't stay in RAM. """ - import gc + import asyncio import os import tempfile import uuid @@ -520,47 +561,31 @@ async def postMigrationUploadImport( totalBytes, totalBytes / 1024 / 1024) try: - with open(filePath, "r", encoding="utf-8") as f: - payload = json.load(f) + processed = await asyncio.to_thread(_processUploadedFile, filePath, tmpDir, token) except (json.JSONDecodeError, UnicodeDecodeError) as e: - os.remove(filePath) + if os.path.exists(filePath): + os.remove(filePath) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Invalid JSON file: {e}") from e + except Exception as e: + if os.path.exists(filePath): + os.remove(filePath) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Processing failed: {e}") from e - try: - os.remove(filePath) - except OSError: - pass - - result = _prepareImport(payload) + result = processed["result"] + dbFiles = processed.get("dbFiles", {}) if not result.get("valid"): - del payload - gc.collect() raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={"message": "Payload validation failed", "warnings": result.get("warnings", [])}, ) - protectedIds = result.get("protectedIds", []) - - dbFiles = {} - databases = payload.get("databases", {}) - for dbName, dbData in databases.items(): - dbPath = os.path.join(tmpDir, f"poweron_import_{token}_{dbName}.json") - with open(dbPath, "w", encoding="utf-8") as dbF: - json.dump(dbData, dbF, ensure_ascii=False, default=str) - dbFiles[dbName] = dbPath - - del payload - del databases - gc.collect() - logger.info("SysAdmin migration upload-import: split into %d per-DB files, payload freed", len(dbFiles)) _pendingImports[token] = { "dbFiles": dbFiles, - "protectedIds": protectedIds, + "protectedIds": processed.get("protectedIds", []), } return {