Merge pull request 'main' (#3) from main into int

Reviewed-on: #3

Dockerfile

@@ -46,4 +46,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
     CMD python -c "import requests; requests.get('http://localhost:8000/api/admin/health', timeout=5)" || exit 1
 
 # Run the application
-CMD exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-8000} --workers 1 --timeout-graceful-shutdown 5
+CMD exec gunicorn app:app --bind 0.0.0.0:${PORT:-8000} --timeout 600 --worker-class uvicorn.workers.UvicornWorker --workers 1

app.py

@@ -730,7 +730,19 @@ logger.info(f"Feature router load results: {featureLoadResults}")
 
 
 if __name__ == "__main__":
-    import uvicorn
-
     port = int(os.environ.get("PORT", 8000))
-    uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5)
+
+    try:
+        from gunicorn.app.wsgiapp import WSGIApplication  # noqa: F401
+        import subprocess
+        import sys
+        subprocess.run([
+            sys.executable, "-m", "gunicorn", "app:app",
+            "--bind", f"0.0.0.0:{port}",
+            "--timeout", "600",
+            "--worker-class", "uvicorn.workers.UvicornWorker",
+            "--workers", "1",
+        ], check=True)
+    except ImportError:
+        import uvicorn
+        uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5)

env-dev.env

@@ -4,7 +4,7 @@
 APP_ENV_TYPE = dev
 APP_ENV_LABEL = Development Instance Patrick
 APP_API_URL = http://localhost:8000
-APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/notes/key.txt
+APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron-swiss/local/notes/key.txt
 APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9
 APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9
 
@@ -23,7 +23,7 @@ APP_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5176,https://nyla.pow
 
 # Logging configuration
 APP_LOGGING_LOG_LEVEL = DEBUG
-APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron/local/logs
+APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron-swiss/local/logs
 APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s
 APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S
 APP_LOGGING_CONSOLE_ENABLED = True
@@ -80,9 +80,9 @@ TEAMSBOT_BROWSER_BOT_URL = http://localhost:4100
 
 # Debug Configuration
 APP_DEBUG_CHAT_WORKFLOW_ENABLED = True
-APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron/local/debug
+APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug
 APP_DEBUG_ACCOUNTING_SYNC_ENABLED = True
-APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron/local/debug/sync
+APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug/sync
 
 # Azure Communication Services Email Configuration
 MESSAGING_ACS_CONNECTION_STRING = endpoint=https://mailing-poweron-prod.switzerland.communication.azure.com/;accesskey=4UizRfBKBgMhDgQ92IYINM6dJsO1HIeL6W1DvIX9S0GtaS1PjIXqJQQJ99CAACULyCpHwxUcAAAAAZCSuSCt

modules/routes/routeAdminDatabaseHealth.py

@@ -36,6 +36,7 @@ from modules.system.databaseMigration import (
     _importSingleDb,
     _prepareImport,
     _validateImportPayload,
+    streamExportGenerator,
 )
 
 logger = logging.getLogger(__name__)
@@ -438,38 +439,18 @@ async def postMigrationImport(
 # Per-DB endpoints (progress-friendly)
 # ---------------------------------------------------------------------------
 
-_pendingExports: Dict[str, dict] = {}
-
-
-@router.post("/migration/export-start")
-@limiter.limit("10/minute")
-def postMigrationExportStart(
-    request: Request,
-    currentUser: User = Depends(requireSysAdmin),
-) -> Dict[str, Any]:
-    """Start an export session. Returns a token for subsequent per-DB calls."""
-    import uuid
-    token = str(uuid.uuid4())
-    _pendingExports[token] = {"databases": {}}
-    logger.info("SysAdmin migration export-start: user=%s token=%s", currentUser.username, token)
-    return {"token": token}
-
 
 @router.get("/migration/export-single")
 @limiter.limit("60/minute")
 def getMigrationExportSingle(
     request: Request,
-    token: str,
     database: str,
     currentUser: User = Depends(requireSysAdmin),
 ) -> Dict[str, Any]:
-    """Export a single database and store it server-side. Returns only metadata."""
+    """Export a single database. Returns full payload so the frontend can
+    assemble the final JSON client-side (no server-side state needed)."""
     from modules.shared.dbRegistry import getRegisteredDatabases
 
-    pending = _pendingExports.get(token)
-    if not pending:
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid export token.")
-
     if database not in getRegisteredDatabases():
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,
@@ -487,7 +468,6 @@ def getMigrationExportSingle(
             detail=f"Export failed for '{database}': {e}",
         ) from e
 
-    pending["databases"][database] = dbPayload
     logger.info("SysAdmin migration export-single done: user=%s db=%s tables=%s records=%s",
                 currentUser.username, database, dbPayload.get("tableCount", 0), dbPayload.get("totalRecords", 0))
 
@@ -495,48 +475,57 @@ def getMigrationExportSingle(
         "database": database,
         "tableCount": dbPayload.get("tableCount", 0),
         "totalRecords": dbPayload.get("totalRecords", 0),
+        "payload": dbPayload,
     }
 
 
-@router.get("/migration/export-download")
+@router.get("/migration/export-stream")
-@limiter.limit("5/minute")
+@limiter.limit("2/minute")
-def getMigrationExportDownload(
+def getMigrationExportStream(
     request: Request,
-    token: str,
+    databases: str = "all",
-    filename: str = "backup.json",
     currentUser: User = Depends(requireSysAdmin),
-) -> StreamingResponse:
+):
-    """Assemble and stream the final export file from server-side data."""
+    """Stream a full database export as a single JSON file download.
+
+    Uses server-side cursors and row-by-row serialization so that neither
+    backend memory nor browser JS heap is exhausted — works for any DB size.
+    """
     from datetime import datetime, timezone
+    from modules.shared.dbRegistry import getRegisteredDatabases
 
-    pending = _pendingExports.pop(token, None)
+    registeredDbs = getRegisteredDatabases()
-    if not pending or not pending.get("databases"):
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid or expired export token.")
 
-    databases = pending["databases"]
+    if databases == "all":
-    totalTables = sum(d.get("tableCount", 0) for d in databases.values())
+        dbList = sorted(registeredDbs.keys())
-    totalRecords = sum(d.get("totalRecords", 0) for d in databases.values())
+    else:
+        dbList = [db.strip() for db in databases.split(",") if db.strip()]
+        invalid = [db for db in dbList if db not in registeredDbs]
+        if invalid:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Unknown databases: {', '.join(invalid)}",
+            )
 
-    exportData = {
+    if not dbList:
-        "meta": {
+        raise HTTPException(
-            "exportedAt": datetime.now(timezone.utc).isoformat(),
+            status_code=status.HTTP_400_BAD_REQUEST,
-            "version": "1.0",
+            detail="No databases selected for export.",
-            "databaseCount": len(databases),
+        )
-            "totalTables": totalTables,
-            "totalRecords": totalRecords,
-        },
-        "databases": databases,
-    }
 
-    logger.info("SysAdmin migration export-download: user=%s dbs=%s tables=%s records=%s",
+    instanceLabel = _getInstanceLabel()
-                currentUser.username, len(databases), totalTables, totalRecords)
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%SZ")
+    filename = f"export_{instanceLabel}_{timestamp}.json" if instanceLabel else f"export_{timestamp}.json"
 
-    content = json.dumps(exportData, ensure_ascii=False, default=str)
+    logger.info("SysAdmin stream export: user=%s databases=%s", currentUser.username, dbList)
 
     return StreamingResponse(
-        iter([content]),
+        streamExportGenerator(dbList, instanceLabel),
         media_type="application/json",
-        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+        headers={
+            "Content-Disposition": f'attachment; filename="{filename}"',
+            "X-Export-Databases": ",".join(dbList),
+        },
     )
 
 

modules/routes/routeDataFiles.py

@@ -298,6 +298,7 @@ def get_folder_tree(
             folders = managementInterface.getSharedFolderTree()
         else:
             raise HTTPException(status_code=400, detail="owner must be 'me' or 'shared'")
+        _enrichFoldersWithMixed(managementInterface.db, str(currentUser.id), folders, o)
         return folders
     except HTTPException:
         raise
@@ -372,6 +373,33 @@ def getAttributesForIds(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+def _enrichFoldersWithMixed(
+    db, userId: str, folders: List[Dict[str, Any]], ownerMode: str,
+) -> None:
+    """Enrich folder dicts in-place: replace raw neutralize/scope with
+    computed values that include ``'mixed'`` when children diverge.
+
+    For ``ownerMode='me'``, files owned by the user are loaded.
+    For ``'shared'``, files inside the visible shared folders are loaded."""
+    if not folders:
+        return
+    if ownerMode == "me":
+        allFiles = db.getRecordset(FileItem, recordFilter={"sysCreatedBy": userId}) or []
+    else:
+        folderIds = {f["id"] for f in folders}
+        allFiles = []
+        for fid in folderIds:
+            allFiles.extend(db.getRecordset(FileItem, recordFilter={"folderId": fid}) or [])
+
+    computed: Dict[str, Dict[str, Any]] = {}
+    for folder in folders:
+        computed[folder["id"]] = _computeFolderAttrs(folder, folders, allFiles)
+    for folder in folders:
+        attrs = computed[folder["id"]]
+        folder["neutralize"] = attrs["neutralize"]
+        folder["scope"] = attrs["scope"]
+
+
 def _computeFolderAttrs(
     folder: Dict[str, Any],
     allFolders: List[Dict[str, Any]],

modules/system/databaseMigration.py

@@ -180,6 +180,121 @@ def _readTableRows(conn, tableName: str) -> List[dict]:
         return [{k: _jsonSafe(v) for k, v in dict(row).items()} for row in cur.fetchall()]
 
 
+# ---------------------------------------------------------------------------
+# Streaming Export (memory-safe, handles arbitrarily large databases)
+# ---------------------------------------------------------------------------
+
+def streamExportGenerator(databases: List[str], instanceLabel: str = ""):
+    """Yield JSON fragments for a streaming database export.
+
+    Writes valid JSON incrementally (row-by-row, table-by-table) so that
+    neither the backend RAM nor the browser JS heap is saturated — works
+    for databases of any size.
+
+    The output format is identical to the non-streaming _exportDatabases():
+    {"meta": {...}, "databases": {"dbName": {"tables": {"tbl": [rows]}, ...}}}
+    """
+    import json
+
+    registeredDbs = getRegisteredDatabases()
+    validDbs = [db for db in databases if db in registeredDbs]
+
+    totalDbs = 0
+    totalTables = 0
+    totalRecords = 0
+
+    yield '{"meta":'
+    metaPlaceholder = json.dumps({
+        "exportedAt": datetime.now(timezone.utc).isoformat(),
+        "version": _EXPORT_FORMAT_VERSION,
+        "instanceLabel": instanceLabel,
+        "databaseCount": "<<PLACEHOLDER>>",
+    }, ensure_ascii=False)
+    yield metaPlaceholder
+    yield ',"databases":{'
+
+    firstDb = True
+    for dbName in validDbs:
+        excluded = _EXCLUDED_TABLES.get(dbName, set())
+        conn = None
+        try:
+            conn = _getConnection(dbName)
+            allTables = _listTables(conn)
+            modelTables = _getModelTablesForDb(dbName, allTables)
+
+            if not firstDb:
+                yield ','
+            firstDb = False
+
+            yield json.dumps(dbName, ensure_ascii=False)
+            yield ':{"tables":{'
+
+            firstTable = True
+            dbTableCount = 0
+            dbRecordCount = 0
+
+            for tbl in modelTables:
+                if tbl in excluded:
+                    continue
+
+                if not firstTable:
+                    yield ','
+                firstTable = False
+
+                yield json.dumps(tbl, ensure_ascii=False)
+                yield ':['
+
+                with conn.cursor(name=f"export_{dbName}_{tbl}") as cur:
+                    cur.itersize = 2000
+                    cur.execute(f'SELECT * FROM "{tbl}"')
+
+                    firstRow = True
+                    rowCount = 0
+                    for row in cur:
+                        if not firstRow:
+                            yield ','
+                        firstRow = False
+                        safeRow = {k: _jsonSafe(v) for k, v in dict(row).items()}
+                        yield json.dumps(safeRow, ensure_ascii=False, default=str)
+                        rowCount += 1
+
+                yield ']'
+                dbTableCount += 1
+                dbRecordCount += rowCount
+
+            yield '},"summary":{'
+            firstSummaryTable = True
+            for tbl in modelTables:
+                if tbl in excluded:
+                    continue
+                if not firstSummaryTable:
+                    yield ','
+                firstSummaryTable = False
+                yield json.dumps(tbl, ensure_ascii=False)
+                yield ':{"recordCount":0}'
+            yield '}'
+            yield f',"tableCount":{dbTableCount},"totalRecords":{dbRecordCount}'
+            yield '}'
+
+            totalDbs += 1
+            totalTables += dbTableCount
+            totalRecords += dbRecordCount
+            logger.info("Stream export: %s done (%d tables, %d records)", dbName, dbTableCount, dbRecordCount)
+
+        except Exception as e:
+            logger.error("Stream export failed for %s: %s", dbName, e)
+            if not firstDb or not firstTable:
+                pass
+        finally:
+            if conn:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+
+    yield '}}'
+
+
 # ---------------------------------------------------------------------------
 # Validate
 # ---------------------------------------------------------------------------

requirements.txt

@@ -2,6 +2,7 @@
 fastapi==0.115.0  # Upgraded for Pydantic v2 compatibility
 websockets==12.0
 uvicorn==0.23.2
+gunicorn==23.0.0
 python-multipart==0.0.6
 httpx>=0.25.2
 pydantic>=2.0.0  # Upgraded to v2 for compatibility