# Copyright (c) 2025 Patrick Motsch # All rights reserved. """ Public and authenticated routes for UI language sets (DB-backed i18n). AI translation pipeline: - create_language_set → background job translates all keys via AiObjects - update_language_set → synchronous AI pass for added keys - update_all → iterates non-de sets """ from __future__ import annotations import asyncio import json import logging import math import re from pathlib import Path from typing import Any, Dict, List, Optional, Set from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Request, UploadFile, status from fastapi.responses import Response from pydantic import BaseModel, Field from modules.auth import getCurrentUser, requireSysAdminRole from modules.connectors.connectorDbPostgre import _get_cached_connector from modules.datamodels.datamodelAi import ( AiCallOptions, AiCallRequest, AiCallResponse, OperationTypeEnum, PriorityEnum, ) from modules.datamodels.datamodelUiLanguage import UiLanguageSet from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelNotification import NotificationType from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface from modules.routes.routeNotifications import _createNotification from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) router = APIRouter( prefix="/api/i18n", tags=["i18n"], responses={404: {"description": "Not found"}}, ) _MIN_AI_BILLING_ESTIMATE_CHF = 0.01 _TRANSLATE_BATCH_SIZE = 80 def _publicMgmtDb(): return _get_cached_connector( dbHost=APP_CONFIG.get("DB_HOST", "localhost"), dbDatabase="poweron_management", dbUser=APP_CONFIG.get("DB_USER"), dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"), dbPort=int(APP_CONFIG.get("DB_PORT", 5432)), userId="__i18n_public__", ) def _row_to_public(row: dict) -> dict: keys = row.get("keys") or {} return { "code": row["id"], "label": row.get("label"), "status": row.get("status"), "keys": keys if isinstance(keys, dict) else {}, } def _load_master_de_keys(db) -> Dict[str, str]: rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"}) if not rows: return {} keys = rows[0].get("keys") or {} return dict(keys) if isinstance(keys, dict) else {} def _userMemberMandateIds(currentUser: User) -> List[str]: from modules.interfaces.interfaceDbApp import getRootInterface root = getRootInterface() memberships = root.getUserMandates(str(currentUser.id)) out = [] for um in memberships: mid = getattr(um, "mandateId", None) or ( um.get("mandateId") if isinstance(um, dict) else None ) if mid: out.append(str(mid)) return list(dict.fromkeys(out)) def _mandatePassesAiPoolBilling(currentUser: User, mandateId: str, userId: str) -> bool: from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface bi = getBillingInterface(currentUser, mandateId) res = bi.checkBalance(mandateId, userId, _MIN_AI_BILLING_ESTIMATE_CHF) return bool(res.allowed) # --------------------------------------------------------------------------- # AI Translation helpers # --------------------------------------------------------------------------- _aiObjectsSingleton = None async def _getAiObjects(): """Lazy singleton — same pattern as routeFeatureWorkspace.""" global _aiObjectsSingleton if _aiObjectsSingleton is None: from modules.interfaces.interfaceAiObjects import AiObjects _aiObjectsSingleton = await AiObjects.create() return _aiObjectsSingleton def _makeBillingCallback(currentUser: User, mandateId: str): """Return a billing callback that records each AI response cost.""" from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService billingService = getBillingService(currentUser, mandateId) def _cb(response: AiCallResponse) -> None: if not response or getattr(response, "errorCount", 0) > 0: return basePriceCHF = getattr(response, "priceCHF", 0.0) if not basePriceCHF or basePriceCHF <= 0: return provider = getattr(response, "provider", None) or "unknown" modelName = getattr(response, "modelName", None) or "unknown" try: billingService.recordUsage( priceCHF=basePriceCHF, aicoreProvider=provider, aicoreModel=modelName, description=f"i18n translation ({modelName})", processingTime=getattr(response, "processingTime", None), bytesSent=getattr(response, "bytesSent", None), bytesReceived=getattr(response, "bytesReceived", None), ) except Exception as e: logger.error("i18n billing callback failed: %s", e) return _cb async def _translateBatch( keysToTranslate: Dict[str, str], targetLanguageLabel: str, targetCode: str, billingCallback=None, ) -> Dict[str, str]: """Translate a batch of German-key → German-value pairs into *targetLanguageLabel*. Returns dict { germanKey: translatedValue }. Splits into sub-batches of _TRANSLATE_BATCH_SIZE to stay within token limits. """ if not keysToTranslate: return {} aiObjects = await _getAiObjects() allKeys = list(keysToTranslate.items()) totalBatches = math.ceil(len(allKeys) / _TRANSLATE_BATCH_SIZE) result: Dict[str, str] = {} for batchIdx in range(totalBatches): chunk = allKeys[batchIdx * _TRANSLATE_BATCH_SIZE : (batchIdx + 1) * _TRANSLATE_BATCH_SIZE] payload = {k: v for k, v in chunk} jsonPayload = json.dumps(payload, ensure_ascii=False) systemPrompt = ( f"Du bist ein professioneller Übersetzer für Software-UI-Texte. " f"Übersetze die folgenden deutschen UI-Labels ins {targetLanguageLabel} (ISO {targetCode}). " f"Behalte Platzhalter wie {{variable}} exakt bei. " f"Antworte NUR mit einem JSON-Objekt — gleiche Keys, übersetzte Values. Kein Markdown, kein Kommentar." ) request = AiCallRequest( prompt=f"Übersetze diese UI-Labels:\n{jsonPayload}", context=systemPrompt, options=AiCallOptions( operationType=OperationTypeEnum.DATA_GENERATE, priority=PriorityEnum.BALANCED, compressPrompt=False, compressContext=False, resultFormat="json", temperature=0.2, ), ) if billingCallback: aiObjects.billingCallback = billingCallback try: response = await aiObjects.callWithTextContext(request) if response and response.content: raw = response.content.strip() if raw.startswith("```"): raw = re.sub(r"^```[a-z]*\n?", "", raw) raw = re.sub(r"\n?```$", "", raw) parsed = json.loads(raw) if isinstance(parsed, dict): result.update(parsed) else: logger.warning("i18n AI batch %d/%d returned non-dict", batchIdx + 1, totalBatches) else: logger.warning("i18n AI batch %d/%d empty response", batchIdx + 1, totalBatches) except json.JSONDecodeError as je: logger.error("i18n AI batch %d/%d JSON parse error: %s", batchIdx + 1, totalBatches, je) except Exception as e: logger.error("i18n AI batch %d/%d failed: %s", batchIdx + 1, totalBatches, e) finally: aiObjects.billingCallback = None return result def _resolveMandateIdForAiI18n(request: Request, currentUser: User) -> str: userId = str(currentUser.id) memberIds = _userMemberMandateIds(currentUser) if not memberIds: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Mindestens eine Mandats-Mitgliedschaft ist für die AI-Nutzung erforderlich.", ) headerRaw = ( request.headers.get("X-Mandate-Id") or request.headers.get("x-mandate-id") or "" ).strip() if headerRaw: if headerRaw not in memberIds: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="X-Mandate-Id ist kein Mandat Ihrer Mitgliedschaft.", ) if _mandatePassesAiPoolBilling(currentUser, headerRaw, userId): return headerRaw for mid in memberIds: if _mandatePassesAiPoolBilling(currentUser, mid, userId): return mid raise HTTPException( status_code=status.HTTP_402_PAYMENT_REQUIRED, detail="Nicht genügend AI-Guthaben (Mandats-Pool) für diese Aktion.", ) # --------------------------------------------------------------------------- # de-Master sync from frontend codebase # --------------------------------------------------------------------------- _REPO_ROOT = Path(__file__).resolve().parents[3] _FRONTEND_SRC = _REPO_ROOT / "frontend_nyla" / "src" _T_CALL_RE = re.compile( r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))""" ) def _scanCodebaseKeys() -> Set[str]: """Scan all .tsx/.ts files under frontend_nyla/src for t('...') calls. Returns the set of German plaintext keys found in the codebase. """ keys: Set[str] = set() if not _FRONTEND_SRC.is_dir(): logger.warning("i18n codebase scan: %s not found", _FRONTEND_SRC) return keys for ext in ("*.tsx", "*.ts"): for filepath in _FRONTEND_SRC.rglob(ext): try: content = filepath.read_text(encoding="utf-8", errors="replace") except OSError: continue for m in _T_CALL_RE.finditer(content): raw = m.group(1) raw = raw.replace("\\'", "'") if raw: keys.add(raw) return keys def _syncDeMasterFromCodebase(db, userId: Optional[str]) -> Dict[str, Any]: """Synchronise the de master set with t()-keys found in the frontend codebase. - Keys in codebase but not in DB → add (key = value = German plaintext) - Keys in DB but not in codebase → remove (orphaned) Returns summary dict. """ codebaseKeys = _scanCodebaseKeys() if not codebaseKeys: logger.warning("i18n de-sync: codebase scan returned 0 keys — aborting") return {"added": [], "removed": [], "keysCount": 0, "error": "Codebase scan returned 0 keys"} rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"}) if not rows: raise HTTPException(status_code=503, detail="Deutsch-Master nicht in DB vorhanden.") row = dict(rows[0]) cur: Dict[str, str] = dict(row.get("keys") or {}) dbKeys = set(cur.keys()) added = sorted(codebaseKeys - dbKeys) removed = sorted(dbKeys - codebaseKeys) for k in removed: del cur[k] for k in added: cur[k] = k if not added and not removed: return {"added": [], "removed": [], "keysCount": len(cur)} now = getUtcTimestamp() row["keys"] = cur row["sysModifiedAt"] = now row["sysModifiedBy"] = userId db.recordModify(UiLanguageSet, "de", row) logger.info("i18n de-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(cur)) return {"added": added, "removed": removed, "keysCount": len(cur)} # --- Public ----------------------------------------------------------------- @router.get("/codes") async def list_language_codes(): db = _publicMgmtDb() rows = db.getRecordset(UiLanguageSet) out = [] for r in rows: keys = r.get("keys") or {} out.append( { "code": r["id"], "label": r.get("label"), "status": r.get("status"), "isDefault": bool(r.get("isDefault")), "keysCount": len(keys) if isinstance(keys, dict) else 0, } ) return sorted(out, key=lambda x: (not x.get("isDefault"), x["code"])) @router.get("/sets/{code}") async def get_language_set(code: str): db = _publicMgmtDb() rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) if not rows: raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") return _row_to_public(rows[0]) # --- Auth user -------------------------------------------------------------- class CreateLanguageBody(BaseModel): code: str = Field(..., min_length=2, max_length=10) label: str = Field(..., min_length=1, max_length=80) def _validate_iso2_code(code: str) -> str: c = code.strip().lower() if not re.fullmatch(r"[a-z]{2}", c): raise HTTPException( status_code=400, detail="Nur ISO-639-1 Zwei-Buchstaben-Codes erlaubt." ) return c def _run_create_language_job(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None: """Background job: translate all German master keys via AI, persist, notify user.""" loop = asyncio.new_event_loop() try: loop.run_until_complete(_run_create_language_job_async(userId, code, label, currentUser, mandateId)) finally: loop.close() async def _run_create_language_job_async(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None: try: db = _publicMgmtDb() rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) if not rows: return deKeys = _load_master_de_keys(db) if not deKeys: logger.error("i18n create job: no de master keys found") return billingCb = _makeBillingCallback(currentUser, mandateId) translated = await _translateBatch(deKeys, label, code, billingCallback=billingCb) finalKeys: Dict[str, str] = {} for k in deKeys: finalKeys[k] = translated.get(k, f"[{k}]") missingCount = sum(1 for k in deKeys if k not in translated) finalStatus = "complete" if missingCount == 0 else "incomplete" now = getUtcTimestamp() merged = dict(rows[0]) merged["keys"] = finalKeys merged["status"] = finalStatus merged["label"] = label merged["sysModifiedAt"] = now merged["sysModifiedBy"] = userId db.recordModify(UiLanguageSet, code, merged) statusHint = "" if finalStatus == "complete" else f" ({missingCount} Keys ohne Übersetzung)" _createNotification( userId, NotificationType.SYSTEM, title="Sprachset erstellt", message=f"Die Sprache «{label}» ({code}) wurde per KI übersetzt{statusHint}.", ) logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(deKeys)) except Exception as e: logger.exception("create language job failed: %s", e) _createNotification( userId, NotificationType.SYSTEM, title="Sprachset fehlgeschlagen", message=f"Fehler bei «{code}»: {e}", ) @router.post("/sets") async def create_language_set( request: Request, body: CreateLanguageBody, background: BackgroundTasks, currentUser: User = Depends(getCurrentUser), ): mandateId = _resolveMandateIdForAiI18n(request, currentUser) code = _validate_iso2_code(body.code) if code == "de": raise HTTPException(status_code=400, detail="Das Standard-Set «de» kann nicht erneut angelegt werden.") db = _publicMgmtDb() existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) if existing: raise HTTPException(status_code=409, detail="Dieses Sprachset existiert bereits.") deKeys = _load_master_de_keys(db) if not deKeys: raise HTTPException(status_code=503, detail="Deutsch-Master nicht geseedet.") now = getUtcTimestamp() uid = str(currentUser.id) rec: dict = { "id": code, "label": body.label.strip(), "keys": {}, "status": "generating", "isDefault": False, "sysCreatedAt": now, "sysCreatedBy": uid, "sysModifiedAt": now, "sysModifiedBy": uid, } db.recordCreate(UiLanguageSet, rec) background.add_task(_run_create_language_job, uid, code, body.label.strip(), currentUser, mandateId) _createNotification( uid, NotificationType.SYSTEM, title="Sprachset wird erzeugt", message=f"Die Sprache «{code}» wird im Hintergrund per KI übersetzt.", ) return {"status": "accepted", "code": code} async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict: if code == "de": raise HTTPException(status_code=400, detail="Das de-Set wird nicht per Update synchronisiert.") rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) if not rows: raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") deKeys = _load_master_de_keys(db) row = dict(rows[0]) cur: Dict[str, str] = dict(row.get("keys") or {}) masterKeys = set(deKeys.keys()) currentKeys = set(cur.keys()) removed = list(currentKeys - masterKeys) added = list(masterKeys - currentKeys) for k in removed: del cur[k] translatedCount = 0 if added: toTranslate = {k: deKeys[k] for k in added} langLabel = row.get("label") or code billingCb = None if adminUser: memberIds = _userMemberMandateIds(adminUser) if memberIds: billingCb = _makeBillingCallback(adminUser, memberIds[0]) try: translated = await _translateBatch(toTranslate, langLabel, code, billingCallback=billingCb) for k in added: cur[k] = translated.get(k, f"[{k}]") translatedCount = sum(1 for k in added if k in translated) except Exception as e: logger.error("AI translation during sync failed for %s: %s", code, e) for k in added: cur[k] = f"[{k}]" now = getUtcTimestamp() row["keys"] = cur untranslated = len(added) - translatedCount row["status"] = "complete" if untranslated == 0 else "incomplete" row["sysModifiedAt"] = now row["sysModifiedBy"] = userId db.recordModify(UiLanguageSet, code, row) return {"code": code, "added": added, "removed": removed, "translated": translatedCount, "keysCount": len(cur)} @router.put("/sets/sync-de") async def sync_de_master_from_codebase( adminUser: User = Depends(requireSysAdminRole), ): """Scan frontend codebase for t() keys and synchronise the de master set. Adds new keys (key=value=German plaintext), removes orphaned keys. """ db = getMgmtInterface(adminUser, mandateId=None).db return _syncDeMasterFromCodebase(db, str(adminUser.id)) @router.put("/sets/update-all") async def update_all_language_sets( adminUser: User = Depends(requireSysAdminRole), ): """Sync de-master from codebase, then update all non-de sets via AI.""" db = getMgmtInterface(adminUser, mandateId=None).db deSync = _syncDeMasterFromCodebase(db, str(adminUser.id)) rows = db.getRecordset(UiLanguageSet) results = [] for r in rows: cid = r["id"] if cid == "de": continue res = await _sync_non_de_set_with_de(db, cid, str(adminUser.id), adminUser=adminUser) results.append(res) return {"deSync": deSync, "updated": results} @router.put("/sets/{code}") async def update_language_set( code: str, adminUser: User = Depends(requireSysAdminRole), ): c = code.strip().lower() if c in ("update-all", "sync-de"): raise HTTPException(status_code=400, detail="Ungültiger Sprachcode.") db = getMgmtInterface(adminUser, mandateId=None).db deSync = _syncDeMasterFromCodebase(db, str(adminUser.id)) langResult = await _sync_non_de_set_with_de(db, c, str(adminUser.id), adminUser=adminUser) langResult["deSync"] = deSync return langResult @router.delete("/sets/{code}") async def delete_language_set( code: str, adminUser: User = Depends(requireSysAdminRole), ): c = code.strip().lower() if c == "de": raise HTTPException(status_code=400, detail="Das Standard-Set «de» darf nicht gelöscht werden.") db = getMgmtInterface(adminUser, mandateId=None).db ok = db.recordDelete(UiLanguageSet, c) if not ok: raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") return {"deleted": c} @router.get("/sets/{code}/download", dependencies=[Depends(getCurrentUser)]) async def download_language_set( code: str, currentUser: User = Depends(getCurrentUser), ): db = _publicMgmtDb() rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code.strip().lower()}) if not rows: raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") payload = _row_to_public(rows[0]) raw = json.dumps(payload.get("keys", {}), ensure_ascii=False, indent=2) return Response( content=raw, media_type="application/json", headers={ "Content-Disposition": f'attachment; filename="ui-language-{code}.json"' }, ) # --- Export / Import (full DB) ----------------------------------------------- @router.get("/export") async def export_all_language_sets( adminUser: User = Depends(requireSysAdminRole), ): """Export the complete language database as a JSON array (all sets with full metadata).""" db = getMgmtInterface(adminUser, mandateId=None).db rows = db.getRecordset(UiLanguageSet) payload = [] for r in rows: payload.append({ "id": r["id"], "label": r.get("label", ""), "keys": dict(r.get("keys") or {}), "status": r.get("status", "complete"), "isDefault": bool(r.get("isDefault", False)), }) payload.sort(key=lambda x: (not x.get("isDefault"), x["id"])) raw = json.dumps(payload, ensure_ascii=False, indent=2) return Response( content=raw, media_type="application/json", headers={ "Content-Disposition": 'attachment; filename="ui-languages-export.json"' }, ) @router.post("/import") async def import_language_sets( file: UploadFile = File(...), adminUser: User = Depends(requireSysAdminRole), ): """Import a previously exported language database JSON. Behaviour per set in the uploaded array: - If the set already exists in DB → overwrite keys, label, status, isDefault - If the set does not exist → create it Existing sets NOT present in the upload are left untouched (no deletion). """ if not file.filename or not file.filename.endswith(".json"): raise HTTPException(status_code=400, detail="Nur .json-Dateien erlaubt.") try: raw = await file.read() data = json.loads(raw.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError) as e: raise HTTPException(status_code=400, detail=f"Ungültiges JSON: {e}") if not isinstance(data, list): raise HTTPException(status_code=400, detail="JSON muss ein Array von Sprachsets sein.") db = getMgmtInterface(adminUser, mandateId=None).db now = getUtcTimestamp() uid = str(adminUser.id) created = [] updated = [] for entry in data: if not isinstance(entry, dict): continue code = str(entry.get("id", "")).strip().lower() if not code or len(code) < 2: continue keys = entry.get("keys") if not isinstance(keys, dict): continue label = str(entry.get("label", code)) entryStatus = str(entry.get("status", "complete")) isDefault = bool(entry.get("isDefault", False)) existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) if existing: row = dict(existing[0]) row["keys"] = keys row["label"] = label row["status"] = entryStatus row["isDefault"] = isDefault row["sysModifiedAt"] = now row["sysModifiedBy"] = uid db.recordModify(UiLanguageSet, code, row) updated.append(code) else: rec = { "id": code, "label": label, "keys": keys, "status": entryStatus, "isDefault": isDefault, "sysCreatedAt": now, "sysCreatedBy": uid, "sysModifiedAt": now, "sysModifiedBy": uid, } db.recordCreate(UiLanguageSet, rec) created.append(code) logger.info("i18n import: created=%s, updated=%s", created, updated) return {"created": created, "updated": updated, "totalProcessed": len(created) + len(updated)}