gateway/modules/routes/routeI18n.py
2026-04-08 20:28:34 +02:00

711 lines
25 KiB
Python

# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Public and authenticated routes for UI language sets (DB-backed i18n).
AI translation pipeline:
- create_language_set → background job translates all keys via AiObjects
- update_language_set → synchronous AI pass for added keys
- update_all → iterates non-de sets
"""
from __future__ import annotations
import asyncio
import json
import logging
import math
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Set
from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, Request, UploadFile, status
from fastapi.responses import Response
from pydantic import BaseModel, Field
from modules.auth import getCurrentUser, requireSysAdminRole
from modules.connectors.connectorDbPostgre import _get_cached_connector
from modules.datamodels.datamodelAi import (
AiCallOptions,
AiCallRequest,
AiCallResponse,
OperationTypeEnum,
PriorityEnum,
)
from modules.datamodels.datamodelUiLanguage import UiLanguageSet
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelNotification import NotificationType
from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface
from modules.routes.routeNotifications import _createNotification
from modules.shared.configuration import APP_CONFIG
from modules.shared.timeUtils import getUtcTimestamp
logger = logging.getLogger(__name__)
router = APIRouter(
prefix="/api/i18n",
tags=["i18n"],
responses={404: {"description": "Not found"}},
)
_MIN_AI_BILLING_ESTIMATE_CHF = 0.01
_TRANSLATE_BATCH_SIZE = 80
def _publicMgmtDb():
return _get_cached_connector(
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
dbDatabase="poweron_management",
dbUser=APP_CONFIG.get("DB_USER"),
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"),
dbPort=int(APP_CONFIG.get("DB_PORT", 5432)),
userId="__i18n_public__",
)
def _row_to_public(row: dict) -> dict:
keys = row.get("keys") or {}
return {
"code": row["id"],
"label": row.get("label"),
"status": row.get("status"),
"keys": keys if isinstance(keys, dict) else {},
}
def _load_master_de_keys(db) -> Dict[str, str]:
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"})
if not rows:
return {}
keys = rows[0].get("keys") or {}
return dict(keys) if isinstance(keys, dict) else {}
def _userMemberMandateIds(currentUser: User) -> List[str]:
from modules.interfaces.interfaceDbApp import getRootInterface
root = getRootInterface()
memberships = root.getUserMandates(str(currentUser.id))
out = []
for um in memberships:
mid = getattr(um, "mandateId", None) or (
um.get("mandateId") if isinstance(um, dict) else None
)
if mid:
out.append(str(mid))
return list(dict.fromkeys(out))
def _mandatePassesAiPoolBilling(currentUser: User, mandateId: str, userId: str) -> bool:
from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface
bi = getBillingInterface(currentUser, mandateId)
res = bi.checkBalance(mandateId, userId, _MIN_AI_BILLING_ESTIMATE_CHF)
return bool(res.allowed)
# ---------------------------------------------------------------------------
# AI Translation helpers
# ---------------------------------------------------------------------------
_aiObjectsSingleton = None
async def _getAiObjects():
"""Lazy singleton — same pattern as routeFeatureWorkspace."""
global _aiObjectsSingleton
if _aiObjectsSingleton is None:
from modules.interfaces.interfaceAiObjects import AiObjects
_aiObjectsSingleton = await AiObjects.create()
return _aiObjectsSingleton
def _makeBillingCallback(currentUser: User, mandateId: str):
"""Return a billing callback that records each AI response cost."""
from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService
billingService = getBillingService(currentUser, mandateId)
def _cb(response: AiCallResponse) -> None:
if not response or getattr(response, "errorCount", 0) > 0:
return
basePriceCHF = getattr(response, "priceCHF", 0.0)
if not basePriceCHF or basePriceCHF <= 0:
return
provider = getattr(response, "provider", None) or "unknown"
modelName = getattr(response, "modelName", None) or "unknown"
try:
billingService.recordUsage(
priceCHF=basePriceCHF,
aicoreProvider=provider,
aicoreModel=modelName,
description=f"i18n translation ({modelName})",
processingTime=getattr(response, "processingTime", None),
bytesSent=getattr(response, "bytesSent", None),
bytesReceived=getattr(response, "bytesReceived", None),
)
except Exception as e:
logger.error("i18n billing callback failed: %s", e)
return _cb
async def _translateBatch(
keysToTranslate: Dict[str, str],
targetLanguageLabel: str,
targetCode: str,
billingCallback=None,
) -> Dict[str, str]:
"""Translate a batch of German-key → German-value pairs into *targetLanguageLabel*.
Returns dict { germanKey: translatedValue }.
Splits into sub-batches of _TRANSLATE_BATCH_SIZE to stay within token limits.
"""
if not keysToTranslate:
return {}
aiObjects = await _getAiObjects()
allKeys = list(keysToTranslate.items())
totalBatches = math.ceil(len(allKeys) / _TRANSLATE_BATCH_SIZE)
result: Dict[str, str] = {}
for batchIdx in range(totalBatches):
chunk = allKeys[batchIdx * _TRANSLATE_BATCH_SIZE : (batchIdx + 1) * _TRANSLATE_BATCH_SIZE]
payload = {k: v for k, v in chunk}
jsonPayload = json.dumps(payload, ensure_ascii=False)
systemPrompt = (
f"Du bist ein professioneller Übersetzer für Software-UI-Texte. "
f"Übersetze die folgenden deutschen UI-Labels ins {targetLanguageLabel} (ISO {targetCode}). "
f"Behalte Platzhalter wie {{variable}} exakt bei. "
f"Antworte NUR mit einem JSON-Objekt — gleiche Keys, übersetzte Values. Kein Markdown, kein Kommentar."
)
request = AiCallRequest(
prompt=f"Übersetze diese UI-Labels:\n{jsonPayload}",
context=systemPrompt,
options=AiCallOptions(
operationType=OperationTypeEnum.DATA_GENERATE,
priority=PriorityEnum.BALANCED,
compressPrompt=False,
compressContext=False,
resultFormat="json",
temperature=0.2,
),
)
if billingCallback:
aiObjects.billingCallback = billingCallback
try:
response = await aiObjects.callWithTextContext(request)
if response and response.content:
raw = response.content.strip()
if raw.startswith("```"):
raw = re.sub(r"^```[a-z]*\n?", "", raw)
raw = re.sub(r"\n?```$", "", raw)
parsed = json.loads(raw)
if isinstance(parsed, dict):
result.update(parsed)
else:
logger.warning("i18n AI batch %d/%d returned non-dict", batchIdx + 1, totalBatches)
else:
logger.warning("i18n AI batch %d/%d empty response", batchIdx + 1, totalBatches)
except json.JSONDecodeError as je:
logger.error("i18n AI batch %d/%d JSON parse error: %s", batchIdx + 1, totalBatches, je)
except Exception as e:
logger.error("i18n AI batch %d/%d failed: %s", batchIdx + 1, totalBatches, e)
finally:
aiObjects.billingCallback = None
return result
def _resolveMandateIdForAiI18n(request: Request, currentUser: User) -> str:
userId = str(currentUser.id)
memberIds = _userMemberMandateIds(currentUser)
if not memberIds:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Mindestens eine Mandats-Mitgliedschaft ist für die AI-Nutzung erforderlich.",
)
headerRaw = (
request.headers.get("X-Mandate-Id") or request.headers.get("x-mandate-id") or ""
).strip()
if headerRaw:
if headerRaw not in memberIds:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="X-Mandate-Id ist kein Mandat Ihrer Mitgliedschaft.",
)
if _mandatePassesAiPoolBilling(currentUser, headerRaw, userId):
return headerRaw
for mid in memberIds:
if _mandatePassesAiPoolBilling(currentUser, mid, userId):
return mid
raise HTTPException(
status_code=status.HTTP_402_PAYMENT_REQUIRED,
detail="Nicht genügend AI-Guthaben (Mandats-Pool) für diese Aktion.",
)
# ---------------------------------------------------------------------------
# de-Master sync from frontend codebase
# ---------------------------------------------------------------------------
_REPO_ROOT = Path(__file__).resolve().parents[3]
_FRONTEND_SRC = _REPO_ROOT / "frontend_nyla" / "src"
_T_CALL_RE = re.compile(
r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))"""
)
def _scanCodebaseKeys() -> Set[str]:
"""Scan all .tsx/.ts files under frontend_nyla/src for t('...') calls.
Returns the set of German plaintext keys found in the codebase.
"""
keys: Set[str] = set()
if not _FRONTEND_SRC.is_dir():
logger.warning("i18n codebase scan: %s not found", _FRONTEND_SRC)
return keys
for ext in ("*.tsx", "*.ts"):
for filepath in _FRONTEND_SRC.rglob(ext):
try:
content = filepath.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
for m in _T_CALL_RE.finditer(content):
raw = m.group(1)
raw = raw.replace("\\'", "'")
if raw:
keys.add(raw)
return keys
def _syncDeMasterFromCodebase(db, userId: Optional[str]) -> Dict[str, Any]:
"""Synchronise the de master set with t()-keys found in the frontend codebase.
- Keys in codebase but not in DB → add (key = value = German plaintext)
- Keys in DB but not in codebase → remove (orphaned)
Returns summary dict.
"""
codebaseKeys = _scanCodebaseKeys()
if not codebaseKeys:
logger.warning("i18n de-sync: codebase scan returned 0 keys — aborting")
return {"added": [], "removed": [], "keysCount": 0, "error": "Codebase scan returned 0 keys"}
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"})
if not rows:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht in DB vorhanden.")
row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {})
dbKeys = set(cur.keys())
added = sorted(codebaseKeys - dbKeys)
removed = sorted(dbKeys - codebaseKeys)
for k in removed:
del cur[k]
for k in added:
cur[k] = k
if not added and not removed:
return {"added": [], "removed": [], "keysCount": len(cur)}
now = getUtcTimestamp()
row["keys"] = cur
row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, "de", row)
logger.info("i18n de-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(cur))
return {"added": added, "removed": removed, "keysCount": len(cur)}
# --- Public -----------------------------------------------------------------
@router.get("/codes")
async def list_language_codes():
db = _publicMgmtDb()
rows = db.getRecordset(UiLanguageSet)
out = []
for r in rows:
keys = r.get("keys") or {}
out.append(
{
"code": r["id"],
"label": r.get("label"),
"status": r.get("status"),
"isDefault": bool(r.get("isDefault")),
"keysCount": len(keys) if isinstance(keys, dict) else 0,
}
)
return sorted(out, key=lambda x: (not x.get("isDefault"), x["code"]))
@router.get("/sets/{code}")
async def get_language_set(code: str):
db = _publicMgmtDb()
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
return _row_to_public(rows[0])
# --- Auth user --------------------------------------------------------------
class CreateLanguageBody(BaseModel):
code: str = Field(..., min_length=2, max_length=10)
label: str = Field(..., min_length=1, max_length=80)
def _validate_iso2_code(code: str) -> str:
c = code.strip().lower()
if not re.fullmatch(r"[a-z]{2}", c):
raise HTTPException(
status_code=400, detail="Nur ISO-639-1 Zwei-Buchstaben-Codes erlaubt."
)
return c
def _run_create_language_job(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None:
"""Background job: translate all German master keys via AI, persist, notify user."""
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(_run_create_language_job_async(userId, code, label, currentUser, mandateId))
finally:
loop.close()
async def _run_create_language_job_async(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None:
try:
db = _publicMgmtDb()
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows:
return
deKeys = _load_master_de_keys(db)
if not deKeys:
logger.error("i18n create job: no de master keys found")
return
billingCb = _makeBillingCallback(currentUser, mandateId)
translated = await _translateBatch(deKeys, label, code, billingCallback=billingCb)
finalKeys: Dict[str, str] = {}
for k in deKeys:
finalKeys[k] = translated.get(k, f"[{k}]")
missingCount = sum(1 for k in deKeys if k not in translated)
finalStatus = "complete" if missingCount == 0 else "incomplete"
now = getUtcTimestamp()
merged = dict(rows[0])
merged["keys"] = finalKeys
merged["status"] = finalStatus
merged["label"] = label
merged["sysModifiedAt"] = now
merged["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, code, merged)
statusHint = "" if finalStatus == "complete" else f" ({missingCount} Keys ohne Übersetzung)"
_createNotification(
userId,
NotificationType.SYSTEM,
title="Sprachset erstellt",
message=f"Die Sprache «{label}» ({code}) wurde per KI übersetzt{statusHint}.",
)
logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(deKeys))
except Exception as e:
logger.exception("create language job failed: %s", e)
_createNotification(
userId,
NotificationType.SYSTEM,
title="Sprachset fehlgeschlagen",
message=f"Fehler bei «{code}»: {e}",
)
@router.post("/sets")
async def create_language_set(
request: Request,
body: CreateLanguageBody,
background: BackgroundTasks,
currentUser: User = Depends(getCurrentUser),
):
mandateId = _resolveMandateIdForAiI18n(request, currentUser)
code = _validate_iso2_code(body.code)
if code == "de":
raise HTTPException(status_code=400, detail="Das Standard-Set «de» kann nicht erneut angelegt werden.")
db = _publicMgmtDb()
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing:
raise HTTPException(status_code=409, detail="Dieses Sprachset existiert bereits.")
deKeys = _load_master_de_keys(db)
if not deKeys:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht geseedet.")
now = getUtcTimestamp()
uid = str(currentUser.id)
rec: dict = {
"id": code,
"label": body.label.strip(),
"keys": {},
"status": "generating",
"isDefault": False,
"sysCreatedAt": now,
"sysCreatedBy": uid,
"sysModifiedAt": now,
"sysModifiedBy": uid,
}
db.recordCreate(UiLanguageSet, rec)
background.add_task(_run_create_language_job, uid, code, body.label.strip(), currentUser, mandateId)
_createNotification(
uid,
NotificationType.SYSTEM,
title="Sprachset wird erzeugt",
message=f"Die Sprache «{code}» wird im Hintergrund per KI übersetzt.",
)
return {"status": "accepted", "code": code}
async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict:
if code == "de":
raise HTTPException(status_code=400, detail="Das de-Set wird nicht per Update synchronisiert.")
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
deKeys = _load_master_de_keys(db)
row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {})
masterKeys = set(deKeys.keys())
currentKeys = set(cur.keys())
removed = list(currentKeys - masterKeys)
added = list(masterKeys - currentKeys)
for k in removed:
del cur[k]
translatedCount = 0
if added:
toTranslate = {k: deKeys[k] for k in added}
langLabel = row.get("label") or code
billingCb = None
if adminUser:
memberIds = _userMemberMandateIds(adminUser)
if memberIds:
billingCb = _makeBillingCallback(adminUser, memberIds[0])
try:
translated = await _translateBatch(toTranslate, langLabel, code, billingCallback=billingCb)
for k in added:
cur[k] = translated.get(k, f"[{k}]")
translatedCount = sum(1 for k in added if k in translated)
except Exception as e:
logger.error("AI translation during sync failed for %s: %s", code, e)
for k in added:
cur[k] = f"[{k}]"
now = getUtcTimestamp()
row["keys"] = cur
untranslated = len(added) - translatedCount
row["status"] = "complete" if untranslated == 0 else "incomplete"
row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, code, row)
return {"code": code, "added": added, "removed": removed, "translated": translatedCount, "keysCount": len(cur)}
@router.put("/sets/sync-de")
async def sync_de_master_from_codebase(
adminUser: User = Depends(requireSysAdminRole),
):
"""Scan frontend codebase for t() keys and synchronise the de master set.
Adds new keys (key=value=German plaintext), removes orphaned keys.
"""
db = getMgmtInterface(adminUser, mandateId=None).db
return _syncDeMasterFromCodebase(db, str(adminUser.id))
@router.put("/sets/update-all")
async def update_all_language_sets(
adminUser: User = Depends(requireSysAdminRole),
):
"""Sync de-master from codebase, then update all non-de sets via AI."""
db = getMgmtInterface(adminUser, mandateId=None).db
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id))
rows = db.getRecordset(UiLanguageSet)
results = []
for r in rows:
cid = r["id"]
if cid == "de":
continue
res = await _sync_non_de_set_with_de(db, cid, str(adminUser.id), adminUser=adminUser)
results.append(res)
return {"deSync": deSync, "updated": results}
@router.put("/sets/{code}")
async def update_language_set(
code: str,
adminUser: User = Depends(requireSysAdminRole),
):
c = code.strip().lower()
if c in ("update-all", "sync-de"):
raise HTTPException(status_code=400, detail="Ungültiger Sprachcode.")
db = getMgmtInterface(adminUser, mandateId=None).db
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id))
langResult = await _sync_non_de_set_with_de(db, c, str(adminUser.id), adminUser=adminUser)
langResult["deSync"] = deSync
return langResult
@router.delete("/sets/{code}")
async def delete_language_set(
code: str,
adminUser: User = Depends(requireSysAdminRole),
):
c = code.strip().lower()
if c == "de":
raise HTTPException(status_code=400, detail="Das Standard-Set «de» darf nicht gelöscht werden.")
db = getMgmtInterface(adminUser, mandateId=None).db
ok = db.recordDelete(UiLanguageSet, c)
if not ok:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
return {"deleted": c}
@router.get("/sets/{code}/download", dependencies=[Depends(getCurrentUser)])
async def download_language_set(
code: str,
currentUser: User = Depends(getCurrentUser),
):
db = _publicMgmtDb()
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code.strip().lower()})
if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
payload = _row_to_public(rows[0])
raw = json.dumps(payload.get("keys", {}), ensure_ascii=False, indent=2)
return Response(
content=raw,
media_type="application/json",
headers={
"Content-Disposition": f'attachment; filename="ui-language-{code}.json"'
},
)
# --- Export / Import (full DB) -----------------------------------------------
@router.get("/export")
async def export_all_language_sets(
adminUser: User = Depends(requireSysAdminRole),
):
"""Export the complete language database as a JSON array (all sets with full metadata)."""
db = getMgmtInterface(adminUser, mandateId=None).db
rows = db.getRecordset(UiLanguageSet)
payload = []
for r in rows:
payload.append({
"id": r["id"],
"label": r.get("label", ""),
"keys": dict(r.get("keys") or {}),
"status": r.get("status", "complete"),
"isDefault": bool(r.get("isDefault", False)),
})
payload.sort(key=lambda x: (not x.get("isDefault"), x["id"]))
raw = json.dumps(payload, ensure_ascii=False, indent=2)
return Response(
content=raw,
media_type="application/json",
headers={
"Content-Disposition": 'attachment; filename="ui-languages-export.json"'
},
)
@router.post("/import")
async def import_language_sets(
file: UploadFile = File(...),
adminUser: User = Depends(requireSysAdminRole),
):
"""Import a previously exported language database JSON.
Behaviour per set in the uploaded array:
- If the set already exists in DB → overwrite keys, label, status, isDefault
- If the set does not exist → create it
Existing sets NOT present in the upload are left untouched (no deletion).
"""
if not file.filename or not file.filename.endswith(".json"):
raise HTTPException(status_code=400, detail="Nur .json-Dateien erlaubt.")
try:
raw = await file.read()
data = json.loads(raw.decode("utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError) as e:
raise HTTPException(status_code=400, detail=f"Ungültiges JSON: {e}")
if not isinstance(data, list):
raise HTTPException(status_code=400, detail="JSON muss ein Array von Sprachsets sein.")
db = getMgmtInterface(adminUser, mandateId=None).db
now = getUtcTimestamp()
uid = str(adminUser.id)
created = []
updated = []
for entry in data:
if not isinstance(entry, dict):
continue
code = str(entry.get("id", "")).strip().lower()
if not code or len(code) < 2:
continue
keys = entry.get("keys")
if not isinstance(keys, dict):
continue
label = str(entry.get("label", code))
entryStatus = str(entry.get("status", "complete"))
isDefault = bool(entry.get("isDefault", False))
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing:
row = dict(existing[0])
row["keys"] = keys
row["label"] = label
row["status"] = entryStatus
row["isDefault"] = isDefault
row["sysModifiedAt"] = now
row["sysModifiedBy"] = uid
db.recordModify(UiLanguageSet, code, row)
updated.append(code)
else:
rec = {
"id": code,
"label": label,
"keys": keys,
"status": entryStatus,
"isDefault": isDefault,
"sysCreatedAt": now,
"sysCreatedBy": uid,
"sysModifiedAt": now,
"sysModifiedBy": uid,
}
db.recordCreate(UiLanguageSet, rec)
created.append(code)
logger.info("i18n import: created=%s, updated=%s", created, updated)
return {"created": created, "updated": updated, "totalProcessed": len(created) + len(updated)}