306 lines
13 KiB
Python
306 lines
13 KiB
Python
# Copyright (c) 2025 Patrick Motsch
|
|
# All rights reserved.
|
|
"""Orchestrates importing accounting data from external systems into TrusteeData* tables.
|
|
|
|
Flow: load config → resolve connector → fetch data → clear old records → write new records → compute balances.
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
from collections import defaultdict
|
|
from typing import Dict, Any, Optional
|
|
|
|
from .accountingConnectorBase import BaseAccountingConnector
|
|
from .accountingRegistry import _getAccountingRegistry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class AccountingDataSync:
|
|
"""Imports accounting data (read-only) from an external system into local TrusteeData* tables."""
|
|
|
|
def __init__(self, trusteeInterface):
|
|
self._if = trusteeInterface
|
|
self._registry = _getAccountingRegistry()
|
|
|
|
async def importData(
|
|
self,
|
|
featureInstanceId: str,
|
|
mandateId: str,
|
|
dateFrom: Optional[str] = None,
|
|
dateTo: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Run a full data import for a feature instance.
|
|
|
|
Returns a summary dict with counts per entity and any errors.
|
|
"""
|
|
from modules.features.trustee.datamodelFeatureTrustee import (
|
|
TrusteeAccountingConfig,
|
|
TrusteeDataAccount,
|
|
TrusteeDataJournalEntry,
|
|
TrusteeDataJournalLine,
|
|
TrusteeDataContact,
|
|
TrusteeDataAccountBalance,
|
|
)
|
|
from modules.shared.configuration import decryptValue
|
|
|
|
summary: Dict[str, Any] = {
|
|
"accounts": 0,
|
|
"journalEntries": 0,
|
|
"journalLines": 0,
|
|
"contacts": 0,
|
|
"accountBalances": 0,
|
|
"errors": [],
|
|
"startedAt": time.time(),
|
|
}
|
|
|
|
cfgRecords = self._if.db.getRecordset(
|
|
TrusteeAccountingConfig,
|
|
recordFilter={"featureInstanceId": featureInstanceId, "isActive": True},
|
|
)
|
|
if not cfgRecords:
|
|
summary["errors"].append("No active accounting configuration found")
|
|
return summary
|
|
|
|
cfgRecord = cfgRecords[0]
|
|
connectorType = cfgRecord.get("connectorType", "")
|
|
encryptedConfig = cfgRecord.get("encryptedConfig", "")
|
|
|
|
try:
|
|
import json
|
|
plainJson = decryptValue(encryptedConfig)
|
|
connConfig = json.loads(plainJson) if plainJson else {}
|
|
except Exception as e:
|
|
summary["errors"].append(f"Failed to decrypt config: {e}")
|
|
return summary
|
|
|
|
connector = self._registry.getConnector(connectorType)
|
|
if not connector:
|
|
summary["errors"].append(f"Unknown connector type: {connectorType}")
|
|
return summary
|
|
|
|
scope = {"featureInstanceId": featureInstanceId, "mandateId": mandateId}
|
|
logger.info(f"AccountingDataSync starting for {featureInstanceId}, connector={connectorType}, dateFrom={dateFrom}, dateTo={dateTo}")
|
|
fetchedAccountNumbers: list = []
|
|
|
|
# 1) Chart of accounts
|
|
try:
|
|
charts = await connector.getChartOfAccounts(connConfig)
|
|
fetchedAccountNumbers = [acc.accountNumber for acc in charts if acc.accountNumber]
|
|
self._clearTable(TrusteeDataAccount, featureInstanceId)
|
|
for acc in charts:
|
|
self._if.db.recordCreate(TrusteeDataAccount, {
|
|
"accountNumber": acc.accountNumber,
|
|
"label": acc.label,
|
|
"accountType": acc.accountType or "",
|
|
"currency": "CHF",
|
|
"isActive": True,
|
|
**scope,
|
|
})
|
|
summary["accounts"] = len(charts)
|
|
except Exception as e:
|
|
logger.error(f"Import accounts failed: {e}", exc_info=True)
|
|
summary["errors"].append(f"Accounts: {e}")
|
|
|
|
# 2) Journal entries + lines (pass already-fetched chart to avoid redundant API call)
|
|
try:
|
|
rawEntries = await connector.getJournalEntries(connConfig, dateFrom=dateFrom, dateTo=dateTo, accountNumbers=fetchedAccountNumbers or None)
|
|
self._clearTable(TrusteeDataJournalEntry, featureInstanceId)
|
|
self._clearTable(TrusteeDataJournalLine, featureInstanceId)
|
|
lineCount = 0
|
|
for raw in rawEntries:
|
|
import uuid
|
|
entryId = str(uuid.uuid4())
|
|
self._if.db.recordCreate(TrusteeDataJournalEntry, {
|
|
"id": entryId,
|
|
"externalId": raw.get("externalId"),
|
|
"bookingDate": raw.get("bookingDate"),
|
|
"reference": raw.get("reference"),
|
|
"description": raw.get("description", ""),
|
|
"currency": raw.get("currency", "CHF"),
|
|
"totalAmount": float(raw.get("totalAmount", 0)),
|
|
**scope,
|
|
})
|
|
for line in (raw.get("lines") or []):
|
|
self._if.db.recordCreate(TrusteeDataJournalLine, {
|
|
"journalEntryId": entryId,
|
|
"accountNumber": line.get("accountNumber", ""),
|
|
"debitAmount": float(line.get("debitAmount", 0)),
|
|
"creditAmount": float(line.get("creditAmount", 0)),
|
|
"currency": line.get("currency", "CHF"),
|
|
"taxCode": line.get("taxCode"),
|
|
"costCenter": line.get("costCenter"),
|
|
"description": line.get("description", ""),
|
|
**scope,
|
|
})
|
|
lineCount += 1
|
|
summary["journalEntries"] = len(rawEntries)
|
|
summary["journalLines"] = lineCount
|
|
except Exception as e:
|
|
logger.error(f"Import journal entries failed: {e}")
|
|
summary["errors"].append(f"Journal entries: {e}")
|
|
|
|
# 3) Contacts (customers + vendors)
|
|
try:
|
|
self._clearTable(TrusteeDataContact, featureInstanceId)
|
|
contactCount = 0
|
|
|
|
customers = await connector.getCustomers(connConfig)
|
|
for c in customers:
|
|
self._if.db.recordCreate(TrusteeDataContact, self._mapContact(c, "customer", scope))
|
|
contactCount += 1
|
|
|
|
vendors = await connector.getVendors(connConfig)
|
|
for v in vendors:
|
|
self._if.db.recordCreate(TrusteeDataContact, self._mapContact(v, "vendor", scope))
|
|
contactCount += 1
|
|
|
|
summary["contacts"] = contactCount
|
|
except Exception as e:
|
|
logger.error(f"Import contacts failed: {e}", exc_info=True)
|
|
summary["errors"].append(f"Contacts: {e}")
|
|
|
|
# 4) Compute account balances from journal lines
|
|
try:
|
|
self._clearTable(TrusteeDataAccountBalance, featureInstanceId)
|
|
balanceCount = self._computeBalances(featureInstanceId, mandateId)
|
|
summary["accountBalances"] = balanceCount
|
|
except Exception as e:
|
|
logger.error(f"Compute balances failed: {e}")
|
|
summary["errors"].append(f"Balances: {e}")
|
|
|
|
# Update config with last import timestamp
|
|
try:
|
|
cfgId = cfgRecord.get("id")
|
|
if cfgId:
|
|
self._if.db.recordModify(TrusteeAccountingConfig, cfgId, {
|
|
"lastSyncAt": time.time(),
|
|
"lastSyncStatus": "success" if not summary["errors"] else "partial",
|
|
"lastSyncErrorMessage": "; ".join(summary["errors"])[:500] if summary["errors"] else None,
|
|
})
|
|
except Exception:
|
|
pass
|
|
|
|
summary["finishedAt"] = time.time()
|
|
summary["durationSeconds"] = round(summary["finishedAt"] - summary["startedAt"], 1)
|
|
logger.info(
|
|
f"AccountingDataSync completed for {featureInstanceId}: "
|
|
f"{summary['accounts']} accounts, {summary['journalEntries']} entries, "
|
|
f"{summary['journalLines']} lines, {summary['contacts']} contacts, "
|
|
f"{summary['accountBalances']} balances, {len(summary['errors'])} errors, "
|
|
f"{summary['durationSeconds']}s"
|
|
)
|
|
return summary
|
|
|
|
@staticmethod
|
|
def _safeStr(val: Any) -> str:
|
|
"""Convert a value to a safe string for DB storage, collapsing nested dicts/lists."""
|
|
if val is None:
|
|
return ""
|
|
if isinstance(val, (dict, list)):
|
|
return ""
|
|
return str(val)
|
|
|
|
def _mapContact(self, raw: Dict[str, Any], contactType: str, scope: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""Extract contact fields from a raw API dict, handling varying field names across connectors."""
|
|
s = self._safeStr
|
|
return {
|
|
"externalId": s(raw.get("id") or raw.get("Id") or raw.get("customer_nr") or raw.get("vendor_nr") or ""),
|
|
"contactType": contactType,
|
|
"contactNumber": s(
|
|
raw.get("customernumber") or raw.get("customer_nr")
|
|
or raw.get("vendornumber") or raw.get("vendor_nr")
|
|
or raw.get("nr") or raw.get("ContactNumber")
|
|
or raw.get("id") or ""
|
|
),
|
|
"name": s(raw.get("name") or raw.get("Name") or raw.get("name_1") or ""),
|
|
"address": s(raw.get("addr1") or raw.get("address") or raw.get("Address") or ""),
|
|
"zip": s(raw.get("zipcode") or raw.get("postcode") or raw.get("Zip") or raw.get("zip") or ""),
|
|
"city": s(raw.get("city") or raw.get("City") or ""),
|
|
"country": s(raw.get("country") or raw.get("country_id") or raw.get("Country") or ""),
|
|
"email": s(raw.get("email") or raw.get("mail") or raw.get("Email") or ""),
|
|
"phone": s(raw.get("phone") or raw.get("phone_fixed") or raw.get("Phone") or ""),
|
|
"vatNumber": s(raw.get("vat_identifier") or raw.get("vatNumber") or ""),
|
|
**scope,
|
|
}
|
|
|
|
def _clearTable(self, model, featureInstanceId: str):
|
|
"""Delete all records for this feature instance from a TrusteeData* table."""
|
|
records = self._if.db.getRecordset(model, recordFilter={"featureInstanceId": featureInstanceId})
|
|
for r in (records or []):
|
|
rid = r.get("id") if isinstance(r, dict) else getattr(r, "id", None)
|
|
if rid:
|
|
try:
|
|
self._if.db.recordDelete(model, rid)
|
|
except Exception:
|
|
pass
|
|
|
|
def _computeBalances(self, featureInstanceId: str, mandateId: str) -> int:
|
|
"""Aggregate journal lines into monthly + annual account balances."""
|
|
from modules.features.trustee.datamodelFeatureTrustee import (
|
|
TrusteeDataJournalEntry,
|
|
TrusteeDataJournalLine,
|
|
TrusteeDataAccountBalance,
|
|
)
|
|
|
|
entries = self._if.db.getRecordset(
|
|
TrusteeDataJournalEntry,
|
|
recordFilter={"featureInstanceId": featureInstanceId},
|
|
) or []
|
|
entryDates = {}
|
|
for e in entries:
|
|
eid = e.get("id") if isinstance(e, dict) else getattr(e, "id", None)
|
|
bdate = e.get("bookingDate") if isinstance(e, dict) else getattr(e, "bookingDate", None)
|
|
if eid and bdate:
|
|
entryDates[eid] = bdate
|
|
|
|
lines = self._if.db.getRecordset(
|
|
TrusteeDataJournalLine,
|
|
recordFilter={"featureInstanceId": featureInstanceId},
|
|
) or []
|
|
|
|
# key: (accountNumber, year, month)
|
|
buckets: Dict[tuple, Dict[str, float]] = defaultdict(lambda: {"debit": 0.0, "credit": 0.0})
|
|
for ln in lines:
|
|
if isinstance(ln, dict):
|
|
jeid = ln.get("journalEntryId", "")
|
|
accNo = ln.get("accountNumber", "")
|
|
debit = float(ln.get("debitAmount", 0))
|
|
credit = float(ln.get("creditAmount", 0))
|
|
else:
|
|
jeid = getattr(ln, "journalEntryId", "")
|
|
accNo = getattr(ln, "accountNumber", "")
|
|
debit = float(getattr(ln, "debitAmount", 0))
|
|
credit = float(getattr(ln, "creditAmount", 0))
|
|
|
|
bdate = entryDates.get(jeid, "")
|
|
if not accNo or not bdate:
|
|
continue
|
|
parts = bdate.split("-")
|
|
if len(parts) < 2:
|
|
continue
|
|
year = int(parts[0])
|
|
month = int(parts[1])
|
|
|
|
buckets[(accNo, year, month)]["debit"] += debit
|
|
buckets[(accNo, year, month)]["credit"] += credit
|
|
buckets[(accNo, year, 0)]["debit"] += debit
|
|
buckets[(accNo, year, 0)]["credit"] += credit
|
|
|
|
count = 0
|
|
scope = {"featureInstanceId": featureInstanceId, "mandateId": mandateId}
|
|
for (accNo, year, month), totals in buckets.items():
|
|
closing = totals["debit"] - totals["credit"]
|
|
self._if.db.recordCreate(TrusteeDataAccountBalance, {
|
|
"accountNumber": accNo,
|
|
"periodYear": year,
|
|
"periodMonth": month,
|
|
"openingBalance": 0.0,
|
|
"debitTotal": round(totals["debit"], 2),
|
|
"creditTotal": round(totals["credit"], 2),
|
|
"closingBalance": round(closing, 2),
|
|
"currency": "CHF",
|
|
**scope,
|
|
})
|
|
count += 1
|
|
return count
|