PowerOn/platform-core
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

main #3

Merged
p.motsch merged 4 commits from main into int 2026-05-27 21:18:07 +00:00
Conversation 0 Commits 4 Files changed 7 +204 -59
7 changed files with 204 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -46,4 +46,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
CMD python -c "import requests; requests.get('http://localhost:8000/api/admin/health', timeout=5)" || exit 1
# Run the application
CMD exec uvicorn app:app --host 0.0.0.0 --port ${PORT:-8000} --workers 1 --timeout-graceful-shutdown 5
CMD exec gunicorn app:app --bind 0.0.0.0:${PORT:-8000} --timeout 600 --worker-class uvicorn.workers.UvicornWorker --workers 1

18
app.py
View file

@ -730,7 +730,19 @@ logger.info(f"Feature router load results: {featureLoadResults}")
if __name__ == "__main__":
import uvicorn
port = int(os.environ.get("PORT", 8000))
uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5)
try:
from gunicorn.app.wsgiapp import WSGIApplication # noqa: F401
import subprocess
import sys
subprocess.run([
sys.executable, "-m", "gunicorn", "app:app",
"--bind", f"0.0.0.0:{port}",
"--timeout", "600",
"--worker-class", "uvicorn.workers.UvicornWorker",
"--workers", "1",
], check=True)
except ImportError:
import uvicorn
uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5)

8
env-dev.env
View file

@ -4,7 +4,7 @@
APP_ENV_TYPE = dev
APP_ENV_LABEL = Development Instance Patrick
APP_API_URL = http://localhost:8000
APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/notes/key.txt
APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron-swiss/local/notes/key.txt
APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9
APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9
@ -23,7 +23,7 @@ APP_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5176,https://nyla.pow
# Logging configuration
APP_LOGGING_LOG_LEVEL = DEBUG
APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron/local/logs
APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron-swiss/local/logs
APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s
APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S
APP_LOGGING_CONSOLE_ENABLED = True
@ -80,9 +80,9 @@ TEAMSBOT_BROWSER_BOT_URL = http://localhost:4100
# Debug Configuration
APP_DEBUG_CHAT_WORKFLOW_ENABLED = True
APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron/local/debug
APP_DEBUG_CHAT_WORKFLOW_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug
APP_DEBUG_ACCOUNTING_SYNC_ENABLED = True
APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron/local/debug/sync
APP_DEBUG_ACCOUNTING_SYNC_DIR = D:/Athi/Local/Web/poweron-swiss/local/debug/sync
# Azure Communication Services Email Configuration
MESSAGING_ACS_CONNECTION_STRING = endpoint=https://mailing-poweron-prod.switzerland.communication.azure.com/;accesskey=4UizRfBKBgMhDgQ92IYINM6dJsO1HIeL6W1DvIX9S0GtaS1PjIXqJQQJ99CAACULyCpHwxUcAAAAAZCSuSCt

91
modules/routes/routeAdminDatabaseHealth.py
View file

@ -36,6 +36,7 @@ from modules.system.databaseMigration import (
_importSingleDb,
_prepareImport,
_validateImportPayload,
streamExportGenerator,
)
logger = logging.getLogger(__name__)
@ -438,38 +439,18 @@ async def postMigrationImport(
# Per-DB endpoints (progress-friendly)
# ---------------------------------------------------------------------------
_pendingExports: Dict[str, dict] = {}
@router.post("/migration/export-start")
@limiter.limit("10/minute")
def postMigrationExportStart(
request: Request,
currentUser: User = Depends(requireSysAdmin),
) -> Dict[str, Any]:
"""Start an export session. Returns a token for subsequent per-DB calls."""
import uuid
token = str(uuid.uuid4())
_pendingExports[token] = {"databases": {}}
logger.info("SysAdmin migration export-start: user=%s token=%s", currentUser.username, token)
return {"token": token}
@router.get("/migration/export-single")
@limiter.limit("60/minute")
def getMigrationExportSingle(
request: Request,
token: str,
database: str,
currentUser: User = Depends(requireSysAdmin),
) -> Dict[str, Any]:
"""Export a single database and store it server-side. Returns only metadata."""
"""Export a single database. Returns full payload so the frontend can
assemble the final JSON client-side (no server-side state needed)."""
from modules.shared.dbRegistry import getRegisteredDatabases
pending = _pendingExports.get(token)
if not pending:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid export token.")
if database not in getRegisteredDatabases():
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@ -487,7 +468,6 @@ def getMigrationExportSingle(
detail=f"Export failed for '{database}': {e}",
) from e
pending["databases"][database] = dbPayload
logger.info("SysAdmin migration export-single done: user=%s db=%s tables=%s records=%s",
currentUser.username, database, dbPayload.get("tableCount", 0), dbPayload.get("totalRecords", 0))
@ -495,48 +475,57 @@ def getMigrationExportSingle(
"database": database,
"tableCount": dbPayload.get("tableCount", 0),
"totalRecords": dbPayload.get("totalRecords", 0),
"payload": dbPayload,
}
@router.get("/migration/export-download")
@limiter.limit("5/minute")
def getMigrationExportDownload(
@router.get("/migration/export-stream")
@limiter.limit("2/minute")
def getMigrationExportStream(
request: Request,
token: str,
filename: str = "backup.json",
databases: str = "all",
currentUser: User = Depends(requireSysAdmin),
) -> StreamingResponse:
"""Assemble and stream the final export file from server-side data."""
):
"""Stream a full database export as a single JSON file download.
Uses server-side cursors and row-by-row serialization so that neither
backend memory nor browser JS heap is exhausted — works for any DB size.
"""
from datetime import datetime, timezone
from modules.shared.dbRegistry import getRegisteredDatabases
pending = _pendingExports.pop(token, None)
if not pending or not pending.get("databases"):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid or expired export token.")
registeredDbs = getRegisteredDatabases()
databases = pending["databases"]
totalTables = sum(d.get("tableCount", 0) for d in databases.values())
totalRecords = sum(d.get("totalRecords", 0) for d in databases.values())
if databases == "all":
dbList = sorted(registeredDbs.keys())
else:
dbList = [db.strip() for db in databases.split(",") if db.strip()]
invalid = [db for db in dbList if db not in registeredDbs]
if invalid:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unknown databases: {', '.join(invalid)}",
)
exportData = {
"meta": {
"exportedAt": datetime.now(timezone.utc).isoformat(),
"version": "1.0",
"databaseCount": len(databases),
"totalTables": totalTables,
"totalRecords": totalRecords,
},
"databases": databases,
}
if not dbList:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="No databases selected for export.",
)
logger.info("SysAdmin migration export-download: user=%s dbs=%s tables=%s records=%s",
currentUser.username, len(databases), totalTables, totalRecords)
instanceLabel = _getInstanceLabel()
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%SZ")
filename = f"export_{instanceLabel}_{timestamp}.json" if instanceLabel else f"export_{timestamp}.json"
content = json.dumps(exportData, ensure_ascii=False, default=str)
logger.info("SysAdmin stream export: user=%s databases=%s", currentUser.username, dbList)
return StreamingResponse(
iter([content]),
streamExportGenerator(dbList, instanceLabel),
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
headers={
"Content-Disposition": f'attachment; filename="{filename}"',
"X-Export-Databases": ",".join(dbList),
},
)

28
modules/routes/routeDataFiles.py
View file

@ -298,6 +298,7 @@ def get_folder_tree(
folders = managementInterface.getSharedFolderTree()
else:
raise HTTPException(status_code=400, detail="owner must be 'me' or 'shared'")
_enrichFoldersWithMixed(managementInterface.db, str(currentUser.id), folders, o)
return folders
except HTTPException:
raise
@ -372,6 +373,33 @@ def getAttributesForIds(
raise HTTPException(status_code=500, detail=str(e))
def _enrichFoldersWithMixed(
db, userId: str, folders: List[Dict[str, Any]], ownerMode: str,
) -> None:
"""Enrich folder dicts in-place: replace raw neutralize/scope with
computed values that include ``'mixed'`` when children diverge.
For ``ownerMode='me'``, files owned by the user are loaded.
For ``'shared'``, files inside the visible shared folders are loaded."""
if not folders:
return
if ownerMode == "me":
allFiles = db.getRecordset(FileItem, recordFilter={"sysCreatedBy": userId}) or []
else:
folderIds = {f["id"] for f in folders}
allFiles = []
for fid in folderIds:
allFiles.extend(db.getRecordset(FileItem, recordFilter={"folderId": fid}) or [])
computed: Dict[str, Dict[str, Any]] = {}
for folder in folders:
computed[folder["id"]] = _computeFolderAttrs(folder, folders, allFiles)
for folder in folders:
attrs = computed[folder["id"]]
folder["neutralize"] = attrs["neutralize"]
folder["scope"] = attrs["scope"]
def _computeFolderAttrs(
folder: Dict[str, Any],
allFolders: List[Dict[str, Any]],

115
modules/system/databaseMigration.py
View file

@ -180,6 +180,121 @@ def _readTableRows(conn, tableName: str) -> List[dict]:
return [{k: _jsonSafe(v) for k, v in dict(row).items()} for row in cur.fetchall()]
# ---------------------------------------------------------------------------
# Streaming Export (memory-safe, handles arbitrarily large databases)
# ---------------------------------------------------------------------------
def streamExportGenerator(databases: List[str], instanceLabel: str = ""):
"""Yield JSON fragments for a streaming database export.
Writes valid JSON incrementally (row-by-row, table-by-table) so that
neither the backend RAM nor the browser JS heap is saturated — works
for databases of any size.
The output format is identical to the non-streaming _exportDatabases():
{"meta": {...}, "databases": {"dbName": {"tables": {"tbl": [rows]}, ...}}}
"""
import json
registeredDbs = getRegisteredDatabases()
validDbs = [db for db in databases if db in registeredDbs]
totalDbs = 0
totalTables = 0
totalRecords = 0
yield '{"meta":'
metaPlaceholder = json.dumps({
"exportedAt": datetime.now(timezone.utc).isoformat(),
"version": _EXPORT_FORMAT_VERSION,
"instanceLabel": instanceLabel,
"databaseCount": "<<PLACEHOLDER>>",
}, ensure_ascii=False)
yield metaPlaceholder
yield ',"databases":{'
firstDb = True
for dbName in validDbs:
excluded = _EXCLUDED_TABLES.get(dbName, set())
conn = None
try:
conn = _getConnection(dbName)
allTables = _listTables(conn)
modelTables = _getModelTablesForDb(dbName, allTables)
if not firstDb:
yield ','
firstDb = False
yield json.dumps(dbName, ensure_ascii=False)
yield ':{"tables":{'
firstTable = True
dbTableCount = 0
dbRecordCount = 0
for tbl in modelTables:
if tbl in excluded:
continue
if not firstTable:
yield ','
firstTable = False
yield json.dumps(tbl, ensure_ascii=False)
yield ':['
with conn.cursor(name=f"export_{dbName}_{tbl}") as cur:
cur.itersize = 2000
cur.execute(f'SELECT * FROM "{tbl}"')
firstRow = True
rowCount = 0
for row in cur:
if not firstRow:
yield ','
firstRow = False
safeRow = {k: _jsonSafe(v) for k, v in dict(row).items()}
yield json.dumps(safeRow, ensure_ascii=False, default=str)
rowCount += 1
yield ']'
dbTableCount += 1
dbRecordCount += rowCount
yield '},"summary":{'
firstSummaryTable = True
for tbl in modelTables:
if tbl in excluded:
continue
if not firstSummaryTable:
yield ','
firstSummaryTable = False
yield json.dumps(tbl, ensure_ascii=False)
yield ':{"recordCount":0}'
yield '}'
yield f',"tableCount":{dbTableCount},"totalRecords":{dbRecordCount}'
yield '}'
totalDbs += 1
totalTables += dbTableCount
totalRecords += dbRecordCount
logger.info("Stream export: %s done (%d tables, %d records)", dbName, dbTableCount, dbRecordCount)
except Exception as e:
logger.error("Stream export failed for %s: %s", dbName, e)
if not firstDb or not firstTable:
pass
finally:
if conn:
try:
conn.close()
except Exception:
pass
yield '}}'
# ---------------------------------------------------------------------------
# Validate
# ---------------------------------------------------------------------------

1
requirements.txt
View file

@ -2,6 +2,7 @@
fastapi==0.115.0 # Upgraded for Pydantic v2 compatibility
websockets==12.0
uvicorn==0.23.2
gunicorn==23.0.0
python-multipart==0.0.6
httpx>=0.25.2
pydantic>=2.0.0 # Upgraded to v2 for compatibility