# Copyright (c) 2025 Patrick Motsch # All rights reserved. """Orchestrates importing accounting data from external systems into TrusteeData* tables. Flow: load config → resolve connector → fetch data → clear old records → write new records → compute balances. """ import json as _json import logging import os import time from collections import defaultdict from pathlib import Path from typing import Dict, Any, List, Optional from .accountingConnectorBase import BaseAccountingConnector from .accountingRegistry import _getAccountingRegistry logger = logging.getLogger(__name__) _DEBUG_SYNC_DIR = Path("D:/Athi/Local/Web/poweron/local/debug/sync") def _debugSyncDir() -> Path: _DEBUG_SYNC_DIR.mkdir(parents=True, exist_ok=True) return _DEBUG_SYNC_DIR def _isDebugEnabled() -> bool: try: from modules.shared.configuration import APP_CONFIG return APP_CONFIG.get("APP_LOGGING_FILE_ENABLED", False) is True or str(APP_CONFIG.get("APP_LOGGING_FILE_ENABLED", "")).lower() == "true" except Exception: return False def _dumpSyncData(tag: str, rows: list): """Write raw connector data to a timestamped JSON file in local/debug/sync/.""" if not _isDebugEnabled(): return try: d = _debugSyncDir() ts = time.strftime("%Y%m%d-%H%M%S") path = d / f"{ts}_{tag}.json" serializable = [] for r in rows: if isinstance(r, dict): serializable.append(r) elif hasattr(r, "__dict__"): serializable.append({k: v for k, v in r.__dict__.items() if not k.startswith("_")}) else: serializable.append(str(r)) with open(path, "w", encoding="utf-8") as f: _json.dump({"count": len(serializable), "rows": serializable}, f, ensure_ascii=False, indent=2, default=str) logger.info(f"Debug sync dump: {path.name} ({len(serializable)} rows)") except Exception as e: logger.warning(f"Failed to write debug sync dump for {tag}: {e}") class AccountingDataSync: """Imports accounting data (read-only) from an external system into local TrusteeData* tables.""" def __init__(self, trusteeInterface): self._if = trusteeInterface self._registry = _getAccountingRegistry() async def importData( self, featureInstanceId: str, mandateId: str, dateFrom: Optional[str] = None, dateTo: Optional[str] = None, ) -> Dict[str, Any]: """Run a full data import for a feature instance. Returns a summary dict with counts per entity and any errors. """ from modules.features.trustee.datamodelFeatureTrustee import ( TrusteeAccountingConfig, TrusteeDataAccount, TrusteeDataJournalEntry, TrusteeDataJournalLine, TrusteeDataContact, TrusteeDataAccountBalance, ) from modules.shared.configuration import decryptValue summary: Dict[str, Any] = { "accounts": 0, "journalEntries": 0, "journalLines": 0, "contacts": 0, "accountBalances": 0, "errors": [], "startedAt": time.time(), } cfgRecords = self._if.db.getRecordset( TrusteeAccountingConfig, recordFilter={"featureInstanceId": featureInstanceId, "isActive": True}, ) if not cfgRecords: summary["errors"].append("No active accounting configuration found") return summary cfgRecord = cfgRecords[0] connectorType = cfgRecord.get("connectorType", "") encryptedConfig = cfgRecord.get("encryptedConfig", "") try: import json plainJson = decryptValue(encryptedConfig) connConfig = json.loads(plainJson) if plainJson else {} except Exception as e: summary["errors"].append(f"Failed to decrypt config: {e}") return summary connector = self._registry.getConnector(connectorType) if not connector: summary["errors"].append(f"Unknown connector type: {connectorType}") return summary scope = {"featureInstanceId": featureInstanceId, "mandateId": mandateId} logger.info(f"AccountingDataSync starting for {featureInstanceId}, connector={connectorType}, dateFrom={dateFrom}, dateTo={dateTo}") fetchedAccountNumbers: list = [] # 1) Chart of accounts try: charts = await connector.getChartOfAccounts(connConfig) _dumpSyncData("accounts", charts) fetchedAccountNumbers = [acc.accountNumber for acc in charts if acc.accountNumber] self._clearTable(TrusteeDataAccount, featureInstanceId) for acc in charts: self._if.db.recordCreate(TrusteeDataAccount, { "accountNumber": acc.accountNumber, "label": acc.label, "accountType": acc.accountType or "", "currency": "CHF", "isActive": True, **scope, }) summary["accounts"] = len(charts) except Exception as e: logger.error(f"Import accounts failed: {e}", exc_info=True) summary["errors"].append(f"Accounts: {e}") # 2) Journal entries + lines (pass already-fetched chart to avoid redundant API call) try: rawEntries = await connector.getJournalEntries(connConfig, dateFrom=dateFrom, dateTo=dateTo, accountNumbers=fetchedAccountNumbers or None) _dumpSyncData("journalEntries", rawEntries) self._clearTable(TrusteeDataJournalEntry, featureInstanceId) self._clearTable(TrusteeDataJournalLine, featureInstanceId) lineCount = 0 for raw in rawEntries: import uuid entryId = str(uuid.uuid4()) self._if.db.recordCreate(TrusteeDataJournalEntry, { "id": entryId, "externalId": raw.get("externalId"), "bookingDate": raw.get("bookingDate"), "reference": raw.get("reference"), "description": raw.get("description", ""), "currency": raw.get("currency", "CHF"), "totalAmount": float(raw.get("totalAmount", 0)), **scope, }) for line in (raw.get("lines") or []): self._if.db.recordCreate(TrusteeDataJournalLine, { "journalEntryId": entryId, "accountNumber": line.get("accountNumber", ""), "debitAmount": float(line.get("debitAmount", 0)), "creditAmount": float(line.get("creditAmount", 0)), "currency": line.get("currency", "CHF"), "taxCode": line.get("taxCode"), "costCenter": line.get("costCenter"), "description": line.get("description", ""), **scope, }) lineCount += 1 summary["journalEntries"] = len(rawEntries) summary["journalLines"] = lineCount except Exception as e: logger.error(f"Import journal entries failed: {e}") summary["errors"].append(f"Journal entries: {e}") # 3) Contacts (customers + vendors) try: self._clearTable(TrusteeDataContact, featureInstanceId) contactCount = 0 customers = await connector.getCustomers(connConfig) _dumpSyncData("customers", customers) for c in customers: self._if.db.recordCreate(TrusteeDataContact, self._mapContact(c, "customer", scope)) contactCount += 1 vendors = await connector.getVendors(connConfig) _dumpSyncData("vendors", vendors) for v in vendors: self._if.db.recordCreate(TrusteeDataContact, self._mapContact(v, "vendor", scope)) contactCount += 1 summary["contacts"] = contactCount except Exception as e: logger.error(f"Import contacts failed: {e}", exc_info=True) summary["errors"].append(f"Contacts: {e}") # 4) Compute account balances from journal lines try: self._clearTable(TrusteeDataAccountBalance, featureInstanceId) balanceCount = self._computeBalances(featureInstanceId, mandateId) summary["accountBalances"] = balanceCount except Exception as e: logger.error(f"Compute balances failed: {e}") summary["errors"].append(f"Balances: {e}") cfgId = cfgRecord.get("id") if cfgId: corePayload = { "lastSyncAt": time.time(), "lastSyncStatus": "success" if not summary["errors"] else "partial", "lastSyncErrorMessage": "; ".join(summary["errors"])[:500] if summary["errors"] else None, } try: self._if.db.recordModify(TrusteeAccountingConfig, cfgId, corePayload) except Exception as coreErr: logger.exception(f"AccountingDataSync: failed to write core lastSync* fields for cfg {cfgId}: {coreErr}") summary["errors"].append(f"Persist lastSync core: {coreErr}") extPayload = { "lastSyncDateFrom": dateFrom, "lastSyncDateTo": dateTo, "lastSyncCounts": { "accounts": int(summary.get("accounts", 0)), "journalEntries": int(summary.get("journalEntries", 0)), "journalLines": int(summary.get("journalLines", 0)), "contacts": int(summary.get("contacts", 0)), "accountBalances": int(summary.get("accountBalances", 0)), }, } try: self._if.db.recordModify(TrusteeAccountingConfig, cfgId, extPayload) except Exception as extErr: logger.exception(f"AccountingDataSync: failed to write extended lastSync* fields for cfg {cfgId}: {extErr}") summary["errors"].append(f"Persist lastSync ext: {extErr}") summary["finishedAt"] = time.time() summary["durationSeconds"] = round(summary["finishedAt"] - summary["startedAt"], 1) logger.info( f"AccountingDataSync completed for {featureInstanceId}: " f"{summary['accounts']} accounts, {summary['journalEntries']} entries, " f"{summary['journalLines']} lines, {summary['contacts']} contacts, " f"{summary['accountBalances']} balances, {len(summary['errors'])} errors, " f"{summary['durationSeconds']}s" ) return summary @staticmethod def _safeStr(val: Any) -> str: """Convert a value to a safe string for DB storage, collapsing nested dicts/lists.""" if val is None: return "" if isinstance(val, (dict, list)): return "" return str(val) def _mapContact(self, raw: Dict[str, Any], contactType: str, scope: Dict[str, Any]) -> Dict[str, Any]: """Extract contact fields from a raw API dict, handling varying field names across connectors.""" s = self._safeStr return { "externalId": s(raw.get("id") or raw.get("Id") or raw.get("customer_nr") or raw.get("vendor_nr") or ""), "contactType": contactType, "contactNumber": s( raw.get("customernumber") or raw.get("customer_nr") or raw.get("vendornumber") or raw.get("vendor_nr") or raw.get("nr") or raw.get("ContactNumber") or raw.get("id") or "" ), "name": s(raw.get("name") or raw.get("Name") or raw.get("name_1") or ""), "address": s(raw.get("addr1") or raw.get("address") or raw.get("Address") or ""), "zip": s(raw.get("zipcode") or raw.get("postcode") or raw.get("Zip") or raw.get("zip") or ""), "city": s(raw.get("city") or raw.get("City") or ""), "country": s(raw.get("country") or raw.get("country_id") or raw.get("Country") or ""), "email": s(raw.get("email") or raw.get("mail") or raw.get("Email") or ""), "phone": s(raw.get("phone") or raw.get("phone_fixed") or raw.get("Phone") or ""), "vatNumber": s(raw.get("vat_identifier") or raw.get("vatNumber") or ""), **scope, } def _clearTable(self, model, featureInstanceId: str): """Delete all records for this feature instance from a TrusteeData* table.""" records = self._if.db.getRecordset(model, recordFilter={"featureInstanceId": featureInstanceId}) for r in (records or []): rid = r.get("id") if isinstance(r, dict) else getattr(r, "id", None) if rid: try: self._if.db.recordDelete(model, rid) except Exception: pass def _computeBalances(self, featureInstanceId: str, mandateId: str) -> int: """Aggregate journal lines into monthly + annual account balances.""" from modules.features.trustee.datamodelFeatureTrustee import ( TrusteeDataJournalEntry, TrusteeDataJournalLine, TrusteeDataAccountBalance, ) entries = self._if.db.getRecordset( TrusteeDataJournalEntry, recordFilter={"featureInstanceId": featureInstanceId}, ) or [] entryDates = {} for e in entries: eid = e.get("id") if isinstance(e, dict) else getattr(e, "id", None) bdate = e.get("bookingDate") if isinstance(e, dict) else getattr(e, "bookingDate", None) if eid and bdate: entryDates[eid] = bdate lines = self._if.db.getRecordset( TrusteeDataJournalLine, recordFilter={"featureInstanceId": featureInstanceId}, ) or [] # key: (accountNumber, year, month) buckets: Dict[tuple, Dict[str, float]] = defaultdict(lambda: {"debit": 0.0, "credit": 0.0}) for ln in lines: if isinstance(ln, dict): jeid = ln.get("journalEntryId", "") accNo = ln.get("accountNumber", "") debit = float(ln.get("debitAmount", 0)) credit = float(ln.get("creditAmount", 0)) else: jeid = getattr(ln, "journalEntryId", "") accNo = getattr(ln, "accountNumber", "") debit = float(getattr(ln, "debitAmount", 0)) credit = float(getattr(ln, "creditAmount", 0)) bdate = entryDates.get(jeid, "") if not accNo or not bdate: continue parts = bdate.split("-") if len(parts) < 2: continue year = int(parts[0]) month = int(parts[1]) buckets[(accNo, year, month)]["debit"] += debit buckets[(accNo, year, month)]["credit"] += credit buckets[(accNo, year, 0)]["debit"] += debit buckets[(accNo, year, 0)]["credit"] += credit count = 0 scope = {"featureInstanceId": featureInstanceId, "mandateId": mandateId} for (accNo, year, month), totals in buckets.items(): closing = totals["debit"] - totals["credit"] self._if.db.recordCreate(TrusteeDataAccountBalance, { "accountNumber": accNo, "periodYear": year, "periodMonth": month, "openingBalance": 0.0, "debitTotal": round(totals["debit"], 2), "creditTotal": round(totals["credit"], 2), "closingBalance": round(closing, 2), "currency": "CHF", **scope, }) count += 1 return count