From c2443a77811a0a7e9290bbc152e8737db29b5309 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 25 May 2026 14:34:02 +0200 Subject: [PATCH] db backup-restore with fk --- modules/system/databaseMigration.py | 95 +++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/modules/system/databaseMigration.py b/modules/system/databaseMigration.py index 8d13041a..4f1e73aa 100644 --- a/modules/system/databaseMigration.py +++ b/modules/system/databaseMigration.py @@ -28,6 +28,10 @@ logger = logging.getLogger(__name__) _EXPORT_FORMAT_VERSION = "1.0" _SYSTEM_TABLE = "_system" +_EXCLUDED_TABLES: Dict[str, Set[str]] = { + "poweron_app": {"Token", "AuthEvent"}, +} + # --------------------------------------------------------------------------- # Instance label @@ -114,10 +118,14 @@ def _exportDatabases(databases: List[str]) -> dict: def _exportSingleDb(dbName: str) -> dict: conn = _getConnection(dbName) + excluded = _EXCLUDED_TABLES.get(dbName, set()) try: tables = _listTables(conn) dbPayload: dict = {"tables": {}, "summary": {}, "tableCount": 0, "totalRecords": 0} for tbl in tables: + if tbl in excluded: + logger.info("Export: skipping excluded table %s.%s", dbName, tbl) + continue rows = _readTableRows(conn, tbl) dbPayload["tables"][tbl] = rows dbPayload["summary"][tbl] = {"recordCount": len(rows)} @@ -627,9 +635,66 @@ def _createTableFromExport(conn, tableName: str, rows: List[dict]) -> None: logger.info("Created table %s with %d columns", tableName, len(allKeys)) +def _getTableImportOrder(conn, tableNames: List[str]) -> List[str]: + """Sort tables by FK dependencies (parents first) using topological sort. + + Queries ``information_schema`` for FK relationships, builds a dependency + graph, and returns the tables in an order that satisfies referential + integrity: parent tables before child tables. + """ + tableSet = set(tableNames) + + with conn.cursor() as cur: + cur.execute(""" + SELECT DISTINCT + tc.table_name AS child_table, + ccu.table_name AS parent_table + FROM information_schema.table_constraints tc + JOIN information_schema.constraint_column_usage ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = 'public' + AND tc.table_name != ccu.table_name + """) + fks = cur.fetchall() + + deps: Dict[str, Set[str]] = {t: set() for t in tableNames} + for fk in fks: + child = fk["child_table"] + parent = fk["parent_table"] + if child in tableSet and parent in tableSet: + deps[child].add(parent) + + inDegree = {t: len(deps[t]) for t in tableNames} + queue = sorted(t for t in tableNames if inDegree[t] == 0) + ordered: List[str] = [] + + while queue: + node = queue.pop(0) + ordered.append(node) + for t in tableNames: + if node in deps[t]: + deps[t].discard(node) + inDegree[t] -= 1 + if inDegree[t] == 0: + queue.append(t) + queue.sort() + + remaining = [t for t in tableNames if t not in set(ordered)] + if remaining: + logger.warning("FK cycle detected, appending without order guarantee: %s", remaining) + ordered.extend(sorted(remaining)) + + return ordered + + def _importSingleDb(payload: dict, dbName: str, mode: str, protectedIds: List[str]) -> dict: """Import a single database from the (already remapped) payload. + Tables are sorted by FK dependencies: parent tables are inserted first, + child tables are deleted first (reverse order) in replace mode. + Returns ``{database, tables: {tableName: insertedCount}, recordCount, warnings}``. """ if mode not in ("replace", "merge"): @@ -656,6 +721,7 @@ def _importSingleDb(payload: dict, dbName: str, mode: str, protectedIds: List[st tables = dbData.get("tables", {}) warnings: List[str] = [] dbResult: Dict[str, int] = {} + excluded = _EXCLUDED_TABLES.get(dbName, set()) if dbCreated: warnings.append(f"Datenbank '{dbName}' wurde neu erstellt") @@ -665,15 +731,37 @@ def _importSingleDb(payload: dict, dbName: str, mode: str, protectedIds: List[st conn.autocommit = False existingTables = set(_listTables(conn)) + # Pre-create missing tables so FK ordering can discover them for tableName, rows in tables.items(): - if not isinstance(rows, list): + if tableName in excluded or not isinstance(rows, list): continue - if tableName not in existingTables: _createTableFromExport(conn, tableName, rows) conn.commit() conn.autocommit = False + existingTables.add(tableName) + # Build importable table list and sort by FK dependencies + importable = [t for t in tables + if t not in excluded + and isinstance(tables.get(t), list) + and t in existingTables] + importOrder = _getTableImportOrder(conn, importable) + + logger.info("Import order for %s: %s", dbName, importOrder) + + for tableName in tables: + if tableName in excluded and isinstance(tables.get(tableName), list): + warnings.append(f"Table '{dbName}.{tableName}' excluded (security/transient)") + + # Phase 1 (replace only): DELETE children first (reverse topological order) + if mode == "replace": + for tableName in reversed(importOrder): + _deleteNonProtected(conn, tableName, protectedIdSet) + + # Phase 2: INSERT parents first (topological order) + for tableName in importOrder: + rows = tables[tableName] physicalCols = _getPhysicalColumns(conn, tableName) if not physicalCols: continue @@ -686,9 +774,6 @@ def _importSingleDb(payload: dict, dbName: str, mode: str, protectedIds: List[st continue filteredRows.append(row) - if mode == "replace": - _deleteNonProtected(conn, tableName, protectedIdSet) - insertedCount = _insertRows(conn, tableName, filteredRows, physicalCols, mode) dbResult[tableName] = insertedCount