PowerOn/gateway
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

issues fixed

Browse source
This commit is contained in:
ValueOn AG 2026-04-08 22:29:30 +02:00
parent cc1fdb13e5
commit 8b33a86274
4 changed files with 13047 additions and 2045 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
modules/datamodels/datamodelUiLanguage.py
View file

@ -1,10 +1,10 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""UI language sets: global i18n strings (German key -> translated value)."""
"""UI language sets: structured i18n entries (context, key, value)."""
from typing import Dict, Optional, Literal
from typing import List, Literal
from pydantic import Field
from pydantic import BaseModel, Field
from modules.datamodels.datamodelBase import PowerOnModel
from modules.shared.attributeUtils import registerModelLabels
@ -13,16 +13,40 @@ from modules.shared.attributeUtils import registerModelLabels
UiLanguageStatus = Literal["complete", "incomplete", "generating"]
class UiLanguageSet(PowerOnModel):
class I18nEntry(BaseModel):
"""Single translation entry within a language set.
context: origin of the key, e.g. "ui" for frontend elements,
"db.management.files.name" for backend data objects.
key: German plaintext (the canonical identifier across all sets).
value: For xx (base set): UI context description for AI translation.
For language sets (de, en, …): the translated text.
"""
One row per ISO 639-1 UI language. id equals code (e.g. de, en).
keys: flat map German plaintext key -> translation for this language.
For language de, values equal keys.
context: str = Field(
...,
description="Origin: 'ui' for frontend, 'db.<schema>.<table>.<field>' for backend objects",
)
key: str = Field(
...,
description="German plaintext key (canonical identifier)",
)
value: str = Field(
default="",
description="Translation (language sets) or context description (xx base set)",
)
class UiLanguageSet(PowerOnModel):
"""One row per language. id = ISO 639-1 code or 'xx' (base set).
The xx set is the master: key = German plaintext, value = UI context for AI.
All other sets (incl. de) are AI-generated translations.
"""
id: str = Field(
...,
description="ISO 639-1 language code (primary key), e.g. de, en, fr",
description="ISO 639-1 language code or 'xx' for the base set",
json_schema_extra={
"frontend_type": "text",
"frontend_readonly": False,
@ -38,9 +62,9 @@ class UiLanguageSet(PowerOnModel):
"frontend_required": True,
},
)
keys: Dict[str, str] = Field(
default_factory=dict,
description="German plaintext key -> translated label",
entries: List[I18nEntry] = Field(
default_factory=list,
description="Translation entries: list of {context, key, value}",
json_schema_extra={
"frontend_type": "textarea",
"frontend_readonly": False,
@ -63,7 +87,7 @@ class UiLanguageSet(PowerOnModel):
)
isDefault: bool = Field(
default=False,
description="Exactly one set should be default (de)",
description="True only for the xx base set",
json_schema_extra={
"frontend_type": "boolean",
"frontend_readonly": False,
@ -78,7 +102,7 @@ registerModelLabels(
{
"id": {"en": "Code", "de": "Code"},
"label": {"en": "Label", "de": "Bezeichnung"},
"keys": {"en": "Keys", "de": "Schlüssel"},
"entries": {"en": "Entries", "de": "Einträge"},
"status": {"en": "Status", "de": "Status"},
"isDefault": {"en": "Default", "de": "Standard"},
},

6
modules/interfaces/interfaceDbManagement.py
View file

@ -219,10 +219,14 @@ class ComponentObjects:
payload = json.loads(seedPath.read_text(encoding="utf-8"))
now = getUtcTimestamp()
for row in payload:
entries = row.get("entries")
if not isinstance(entries, list):
keys = row.get("keys") or {}
entries = [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
rec = {
"id": row["id"],
"label": row["label"],
"keys": row.get("keys") or {},
"entries": entries,
"status": row.get("status") or "complete",
"isDefault": bool(row.get("isDefault", False)),
"sysCreatedAt": now,

14617
modules/migration/seedData/ui_language_seed.json
View file

File diff suppressed because it is too large Load diff

407
modules/routes/routeI18n.py
View file

@ -3,10 +3,10 @@
"""
Public and authenticated routes for UI language sets (DB-backed i18n).
AI translation pipeline:
- create_language_set → background job translates all keys via AiObjects
- update_language_set → synchronous AI pass for added keys
- update_all → iterates non-de sets
Architecture:
- xx = base set (meta): key = German plaintext, value = UI context for AI
- All languages (incl. de) are AI-generated translations from xx
- AI translation pipeline uses context from xx to disambiguate translations
"""
from __future__ import annotations
@ -32,7 +32,7 @@ from modules.datamodels.datamodelAi import (
OperationTypeEnum,
PriorityEnum,
)
from modules.datamodels.datamodelUiLanguage import UiLanguageSet
from modules.datamodels.datamodelUiLanguage import I18nEntry, UiLanguageSet
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelNotification import NotificationType
from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface
@ -51,6 +51,41 @@ router = APIRouter(
_MIN_AI_BILLING_ESTIMATE_CHF = 0.01
_TRANSLATE_BATCH_SIZE = 80
_PROTECTED_CODES = frozenset({"xx", "de"})
# ---------------------------------------------------------------------------
# ISO 639-1 label map (used when creating a language without explicit label)
# ---------------------------------------------------------------------------
_ISO_LABELS: Dict[str, str] = {
"de": "Deutsch", "en": "English", "fr": "Français", "it": "Italiano",
"es": "Español", "pt": "Português", "nl": "Nederlands", "pl": "Polski",
"cs": "Čeština", "sk": "Slovenčina", "sv": "Svenska", "no": "Norsk",
"da": "Dansk", "fi": "Suomi", "hu": "Magyar", "ro": "Română",
"bg": "Български", "hr": "Hrvatski", "sl": "Slovenščina", "et": "Eesti",
"lv": "Latviešu", "lt": "Lietuvių", "el": "Ελληνικά", "tr": "Türkçe",
"ru": "Русский", "uk": "Українська", "ar": "العربية", "he": "עברית",
"zh": "中文", "ja": "日本語", "ko": "한국어", "hi": "हिन्दी",
"th": "ไทย", "vi": "Tiếng Việt", "id": "Bahasa Indonesia", "ms": "Bahasa Melayu",
"tl": "Filipino", "sw": "Kiswahili", "af": "Afrikaans", "sq": "Shqip",
"am": "አማርኛ", "hy": "Հայերեն", "az": "Azərbaycan", "eu": "Euskara",
"be": "Беларуская", "bn": "বাংলা", "bs": "Bosanski", "ca": "Català",
"cy": "Cymraeg", "eo": "Esperanto", "fa": "فارسی", "ga": "Gaeilge",
"gl": "Galego", "gu": "ગુજરાતી", "ha": "Hausa", "is": "Íslenska",
"jv": "Basa Jawa", "ka": "ქართული", "kk": "Қазақ", "km": "ខ្មែរ",
"kn": "ಕನ್ನಡ", "ku": "Kurdî", "ky": "Кыргызча", "la": "Latina",
"lb": "Lëtzebuergesch", "lo": "ລາວ", "mk": "Македонски", "ml": "മലയാളം",
"mn": "Монгол", "mr": "मराठी", "mt": "Malti", "my": "မြန်မာ",
"ne": "नेपाली", "or": "ଓଡ଼ିଆ", "pa": "ਪੰਜਾਬੀ", "ps": "پښتو",
"si": "සිංහල", "so": "Soomaali", "sr": "Српски", "su": "Basa Sunda",
"ta": "தமிழ்", "te": "తెలుగు", "tg": "Тоҷикӣ", "tk": "Türkmen",
"ur": "اردو", "uz": "Oʻzbek", "yo": "Yorùbá", "zu": "isiZulu",
}
# ---------------------------------------------------------------------------
# DB helpers
# ---------------------------------------------------------------------------
def _publicMgmtDb():
return _get_cached_connector(
@ -63,22 +98,38 @@ def _publicMgmtDb():
)
def _rowEntries(row: dict) -> List[dict]:
"""Read entries from a DB row, supporting both new (entries) and legacy (keys) format."""
entries = row.get("entries")
if isinstance(entries, list) and entries:
return entries
keys = row.get("keys")
if isinstance(keys, dict) and keys:
return [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
return []
def _entriesToKeyValueMap(entries: List[dict]) -> Dict[str, str]:
"""Convert entries list to a flat key->value map (for frontend consumption)."""
return {e["key"]: e.get("value", "") for e in entries if e.get("key")}
def _row_to_public(row: dict) -> dict:
keys = row.get("keys") or {}
entries = _rowEntries(row)
return {
"code": row["id"],
"label": row.get("label"),
"status": row.get("status"),
"keys": keys if isinstance(keys, dict) else {},
"entries": entries,
}
def _load_master_de_keys(db) -> Dict[str, str]:
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"})
def _loadMasterXxEntries(db) -> List[dict]:
"""Load the xx base set entries from DB."""
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "xx"})
if not rows:
return {}
keys = rows[0].get("keys") or {}
return dict(keys) if isinstance(keys, dict) else {}
return []
return _rowEntries(rows[0])
def _userMemberMandateIds(currentUser: User) -> List[str]:
@ -112,7 +163,6 @@ _aiObjectsSingleton = None
async def _getAiObjects():
"""Lazy singleton — same pattern as routeFeatureWorkspace."""
global _aiObjectsSingleton
if _aiObjectsSingleton is None:
from modules.interfaces.interfaceAiObjects import AiObjects
@ -121,7 +171,6 @@ async def _getAiObjects():
def _makeBillingCallback(currentUser: User, mandateId: str):
"""Return a billing callback that records each AI response cost."""
from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService
billingService = getBillingService(currentUser, mandateId)
@ -156,10 +205,10 @@ async def _translateBatch(
targetCode: str,
billingCallback=None,
) -> Dict[str, str]:
"""Translate a batch of German-key → German-value pairs into *targetLanguageLabel*.
"""Translate German keys into targetLanguageLabel.
Returns dict { germanKey: translatedValue }.
Splits into sub-batches of _TRANSLATE_BATCH_SIZE to stay within token limits.
keysToTranslate: { germanKey: uiContext }
Returns: { germanKey: translatedValue }
"""
if not keysToTranslate:
return {}
@ -171,14 +220,17 @@ async def _translateBatch(
for batchIdx in range(totalBatches):
chunk = allKeys[batchIdx * _TRANSLATE_BATCH_SIZE : (batchIdx + 1) * _TRANSLATE_BATCH_SIZE]
payload = {k: v for k, v in chunk}
payload = [{"key": k, "context": v} for k, v in chunk]
jsonPayload = json.dumps(payload, ensure_ascii=False)
systemPrompt = (
f"Du bist ein professioneller Übersetzer für Software-UI-Texte. "
f"Übersetze die folgenden deutschen UI-Labels ins {targetLanguageLabel} (ISO {targetCode}). "
f"Du erhältst ein JSON-Array mit Objekten: {{\"key\": \"deutscher Text\", \"context\": \"UI-Kontext\"}}. "
f"Der Kontext beschreibt, wo der Text in der Anwendung verwendet wird (Datei, Komponente). "
f"Übersetze jeden «key» ins {targetLanguageLabel} (ISO {targetCode}). "
f"Behalte Platzhalter wie {{variable}} exakt bei. "
f"Antworte NUR mit einem JSON-Objekt — gleiche Keys, übersetzte Values. Kein Markdown, kein Kommentar."
f"Antworte NUR mit einem JSON-Objekt — Keys = deutsche Originaltexte, Values = Übersetzungen. "
f"Kein Markdown, kein Kommentar."
)
request = AiCallRequest(
@ -251,26 +303,21 @@ def _resolveMandateIdForAiI18n(request: Request, currentUser: User) -> str:
# ---------------------------------------------------------------------------
# de-Master sync from frontend codebase
# xx-Master sync from frontend codebase (local dev fallback)
# ---------------------------------------------------------------------------
_REPO_ROOT = Path(__file__).resolve().parents[3]
_FRONTEND_SRC = _REPO_ROOT / "frontend_nyla" / "src"
_T_CALL_RE = re.compile(
r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))"""
)
_T_CALL_RE = re.compile(r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))""")
def _scanCodebaseKeys() -> Set[str]:
"""Scan all .tsx/.ts files under frontend_nyla/src for t('...') calls.
Returns the set of German plaintext keys found in the codebase.
"""
def _scanCodebaseKeys() -> List[dict]:
"""Local dev fallback: scan frontend src for t() calls. Returns entries with context='ui'."""
keys: Set[str] = set()
if not _FRONTEND_SRC.is_dir():
logger.warning("i18n codebase scan: %s not found", _FRONTEND_SRC)
return keys
return []
for ext in ("*.tsx", "*.ts"):
for filepath in _FRONTEND_SRC.rglob(ext):
@ -283,48 +330,103 @@ def _scanCodebaseKeys() -> Set[str]:
raw = raw.replace("\\'", "'")
if raw:
keys.add(raw)
return keys
return [{"context": "ui", "key": k, "value": ""} for k in sorted(keys)]
def _syncDeMasterFromCodebase(db, userId: Optional[str]) -> Dict[str, Any]:
"""Synchronise the de master set with t()-keys found in the frontend codebase.
async def _readOptionalEntriesFromBody(request: Request) -> Optional[List[dict]]:
"""Read entries from request body. Accepts {entries: [{context, key, value}, ...]}."""
body = await request.body()
if not body or not body.strip():
return None
try:
data = json.loads(body.decode("utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError):
return None
if not isinstance(data, dict) or "entries" not in data:
return None
entries = data.get("entries")
if not isinstance(entries, list):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Feld «entries» muss ein JSON-Array sein.",
)
result = []
for e in entries:
if not isinstance(e, dict) or not e.get("key"):
continue
result.append({
"context": str(e.get("context", "ui")),
"key": str(e["key"]),
"value": str(e.get("value", "")),
})
return result if result else None
- Keys in codebase but not in DB → add (key = value = German plaintext)
- Keys in DB but not in codebase → remove (orphaned)
Returns summary dict.
def _syncXxMaster(db, userId: Optional[str], incomingEntries: List[dict]) -> Dict[str, Any]:
"""Synchronise the xx base set with incoming entries (from build bundle or codebase scan).
- Keys in incoming but not in DB -> add
- Keys in DB but not in incoming -> remove
- Keys in both -> update context (value)
"""
codebaseKeys = _scanCodebaseKeys()
if not codebaseKeys:
logger.warning("i18n de-sync: codebase scan returned 0 keys — aborting")
return {"added": [], "removed": [], "keysCount": 0, "error": "Codebase scan returned 0 keys"}
if not incomingEntries:
logger.warning("i18n xx-sync: no entries — aborting")
return {"added": [], "removed": [], "entriesCount": 0, "error": "No entries to sync"}
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"})
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "xx"})
if not rows:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht in DB vorhanden.")
now = getUtcTimestamp()
rec = {
"id": "xx",
"label": "Basisset (Meta)",
"entries": incomingEntries,
"status": "complete",
"isDefault": True,
"sysCreatedAt": now,
"sysCreatedBy": userId,
"sysModifiedAt": now,
"sysModifiedBy": userId,
}
db.recordCreate(UiLanguageSet, rec)
allKeys = [e["key"] for e in incomingEntries]
logger.info("i18n xx-master created: %d entries", len(incomingEntries))
return {"added": allKeys, "removed": [], "entriesCount": len(incomingEntries)}
row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {})
dbKeys = set(cur.keys())
curEntries = _rowEntries(row)
curByKey = {e["key"]: e for e in curEntries}
incomingByKey = {e["key"]: e for e in incomingEntries}
added = sorted(codebaseKeys - dbKeys)
removed = sorted(dbKeys - codebaseKeys)
incomingKeys = set(incomingByKey.keys())
dbKeys = set(curByKey.keys())
for k in removed:
del cur[k]
for k in added:
cur[k] = k
added = sorted(incomingKeys - dbKeys)
removed = sorted(dbKeys - incomingKeys)
if not added and not removed:
return {"added": [], "removed": [], "keysCount": len(cur)}
newEntries = []
for e in incomingEntries:
newEntries.append({"context": e["context"], "key": e["key"], "value": e["value"]})
for e in curEntries:
if e["key"] not in incomingKeys:
continue
if not added and not removed and all(
curByKey.get(e["key"], {}).get("value") == e["value"]
and curByKey.get(e["key"], {}).get("context") == e["context"]
for e in incomingEntries
):
return {"added": [], "removed": [], "entriesCount": len(newEntries)}
now = getUtcTimestamp()
row["keys"] = cur
row["entries"] = newEntries
if "keys" in row:
del row["keys"]
row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, "de", row)
db.recordModify(UiLanguageSet, "xx", row)
logger.info("i18n de-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(cur))
return {"added": added, "removed": removed, "keysCount": len(cur)}
logger.info("i18n xx-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(newEntries))
return {"added": added, "removed": removed, "entriesCount": len(newEntries)}
# --- Public -----------------------------------------------------------------
@ -336,14 +438,14 @@ async def list_language_codes():
rows = db.getRecordset(UiLanguageSet)
out = []
for r in rows:
keys = r.get("keys") or {}
entries = _rowEntries(r)
out.append(
{
"code": r["id"],
"label": r.get("label"),
"status": r.get("status"),
"isDefault": bool(r.get("isDefault")),
"keysCount": len(keys) if isinstance(keys, dict) else 0,
"entriesCount": len(entries),
}
)
return sorted(out, key=lambda x: (not x.get("isDefault"), x["code"]))
@ -363,7 +465,7 @@ async def get_language_set(code: str):
class CreateLanguageBody(BaseModel):
code: str = Field(..., min_length=2, max_length=10)
label: str = Field(..., min_length=1, max_length=80)
label: Optional[str] = Field(default=None, max_length=80)
def _validate_iso2_code(code: str) -> str:
@ -376,7 +478,6 @@ def _validate_iso2_code(code: str) -> str:
def _run_create_language_job(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None:
"""Background job: translate all German master keys via AI, persist, notify user."""
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(_run_create_language_job_async(userId, code, label, currentUser, mandateId))
@ -390,24 +491,32 @@ async def _run_create_language_job_async(userId: str, code: str, label: str, cur
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows:
return
deKeys = _load_master_de_keys(db)
if not deKeys:
logger.error("i18n create job: no de master keys found")
xxEntries = _loadMasterXxEntries(db)
if not xxEntries:
logger.error("i18n create job: no xx master entries found")
return
toTranslate = {e["key"]: e.get("value", "") for e in xxEntries}
billingCb = _makeBillingCallback(currentUser, mandateId)
translated = await _translateBatch(deKeys, label, code, billingCallback=billingCb)
translated = await _translateBatch(toTranslate, label, code, billingCallback=billingCb)
finalKeys: Dict[str, str] = {}
for k in deKeys:
finalKeys[k] = translated.get(k, f"[{k}]")
finalEntries = []
for e in xxEntries:
k = e["key"]
finalEntries.append({
"context": e["context"],
"key": k,
"value": translated.get(k, f"[{k}]"),
})
missingCount = sum(1 for k in deKeys if k not in translated)
missingCount = sum(1 for e in xxEntries if e["key"] not in translated)
finalStatus = "complete" if missingCount == 0 else "incomplete"
now = getUtcTimestamp()
merged = dict(rows[0])
merged["keys"] = finalKeys
merged["entries"] = finalEntries
if "keys" in merged:
del merged["keys"]
merged["status"] = finalStatus
merged["label"] = label
merged["sysModifiedAt"] = now
@ -421,7 +530,7 @@ async def _run_create_language_job_async(userId: str, code: str, label: str, cur
title="Sprachset erstellt",
message=f"Die Sprache «{label}» ({code}) wurde per KI übersetzt{statusHint}.",
)
logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(deKeys))
logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(xxEntries))
except Exception as e:
logger.exception("create language job failed: %s", e)
_createNotification(
@ -441,24 +550,28 @@ async def create_language_set(
):
mandateId = _resolveMandateIdForAiI18n(request, currentUser)
code = _validate_iso2_code(body.code)
if code == "de":
raise HTTPException(status_code=400, detail="Das Standard-Set «de» kann nicht erneut angelegt werden.")
if code == "xx":
raise HTTPException(status_code=400, detail="Das Basisset «xx» kann nicht manuell angelegt werden.")
db = _publicMgmtDb()
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing:
raise HTTPException(status_code=409, detail="Dieses Sprachset existiert bereits.")
deKeys = _load_master_de_keys(db)
if not deKeys:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht geseedet.")
xxEntries = _loadMasterXxEntries(db)
if not xxEntries:
raise HTTPException(status_code=503, detail="Basisset (xx) nicht vorhanden. Bitte zuerst UI-Keys einlesen.")
resolvedLabel = (body.label or "").strip() if body.label else ""
if not resolvedLabel:
resolvedLabel = _ISO_LABELS.get(code, code)
now = getUtcTimestamp()
uid = str(currentUser.id)
rec: dict = {
"id": code,
"label": body.label.strip(),
"keys": {},
"label": resolvedLabel,
"entries": [],
"status": "generating",
"isDefault": False,
"sysCreatedAt": now,
@ -468,7 +581,7 @@ async def create_language_set(
}
db.recordCreate(UiLanguageSet, rec)
background.add_task(_run_create_language_job, uid, code, body.label.strip(), currentUser, mandateId)
background.add_task(_run_create_language_job, uid, code, resolvedLabel, currentUser, mandateId)
_createNotification(
uid,
NotificationType.SYSTEM,
@ -478,25 +591,30 @@ async def create_language_set(
return {"status": "accepted", "code": code}
async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict:
if code == "de":
raise HTTPException(status_code=400, detail="Das de-Set wird nicht per Update synchronisiert.")
async def _syncLanguageWithXx(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict:
"""Synchronise a language set (incl. de) against the xx base set via AI."""
if code == "xx":
raise HTTPException(status_code=400, detail="Das xx-Set wird über 'UI-Keys einlesen' aktualisiert.")
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
deKeys = _load_master_de_keys(db)
xxEntries = _loadMasterXxEntries(db)
if not xxEntries:
raise HTTPException(status_code=503, detail="Basisset (xx) nicht vorhanden.")
row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {})
masterKeys = set(deKeys.keys())
currentKeys = set(cur.keys())
removed = list(currentKeys - masterKeys)
added = list(masterKeys - currentKeys)
for k in removed:
del cur[k]
curEntries = _rowEntries(row)
curByKey = {e["key"]: e for e in curEntries}
xxByKey = {e["key"]: e for e in xxEntries}
masterKeys = set(xxByKey.keys())
currentKeys = set(curByKey.keys())
removedKeys = sorted(currentKeys - masterKeys)
addedKeys = sorted(masterKeys - currentKeys)
translatedCount = 0
if added:
toTranslate = {k: deKeys[k] for k in added}
if addedKeys:
toTranslate = {k: xxByKey[k].get("value", "") for k in addedKeys}
langLabel = row.get("label") or code
billingCb = None
if adminUser:
@ -505,54 +623,84 @@ async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUs
billingCb = _makeBillingCallback(adminUser, memberIds[0])
try:
translated = await _translateBatch(toTranslate, langLabel, code, billingCallback=billingCb)
for k in added:
cur[k] = translated.get(k, f"[{k}]")
translatedCount = sum(1 for k in added if k in translated)
translatedCount = sum(1 for k in addedKeys if k in translated)
except Exception as e:
logger.error("AI translation during sync failed for %s: %s", code, e)
for k in added:
cur[k] = f"[{k}]"
translated = {}
for k in addedKeys:
curByKey[k] = {
"context": xxByKey[k]["context"],
"key": k,
"value": translated.get(k, f"[{k}]"),
}
for k in removedKeys:
del curByKey[k]
for k in masterKeys & currentKeys:
curByKey[k]["context"] = xxByKey[k]["context"]
newEntries = [curByKey[k] for k in sorted(curByKey.keys(), key=lambda x: x.lower())]
now = getUtcTimestamp()
row["keys"] = cur
untranslated = len(added) - translatedCount
untranslated = len(addedKeys) - translatedCount
row["entries"] = newEntries
if "keys" in row:
del row["keys"]
row["status"] = "complete" if untranslated == 0 else "incomplete"
row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, code, row)
return {"code": code, "added": added, "removed": removed, "translated": translatedCount, "keysCount": len(cur)}
return {
"code": code,
"added": addedKeys,
"removed": removedKeys,
"translated": translatedCount,
"entriesCount": len(newEntries),
}
@router.put("/sets/sync-de")
async def sync_de_master_from_codebase(
@router.put("/sets/sync-xx")
async def sync_xx_master(
request: Request,
adminUser: User = Depends(requireSysAdminRole),
):
"""Scan frontend codebase for t() keys and synchronise the de master set.
"""Synchronise the xx base set from the frontend build artefact.
Adds new keys (key=value=German plaintext), removes orphaned keys.
Expects JSON body: {"entries": [{"context":"ui","key":"…","value":"…"}, …]}
Falls back to local codebase scan if no body provided (dev mode).
"""
db = getMgmtInterface(adminUser, mandateId=None).db
return _syncDeMasterFromCodebase(db, str(adminUser.id))
fromBody = await _readOptionalEntriesFromBody(request)
entries = fromBody if fromBody is not None else _scanCodebaseKeys()
return _syncXxMaster(db, str(adminUser.id), entries)
@router.put("/sets/update-all")
async def update_all_language_sets(
request: Request,
adminUser: User = Depends(requireSysAdminRole),
):
"""Sync de-master from codebase, then update all non-de sets via AI."""
"""Sync xx-master (if body provided), then update ALL language sets via AI."""
db = getMgmtInterface(adminUser, mandateId=None).db
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id))
fromBody = await _readOptionalEntriesFromBody(request)
xxSync: Optional[dict] = None
if fromBody is not None:
xxSync = _syncXxMaster(db, str(adminUser.id), fromBody)
if xxSync.get("error"):
return {"xxSync": xxSync, "updated": []}
rows = db.getRecordset(UiLanguageSet)
results = []
for r in rows:
cid = r["id"]
if cid == "de":
if cid == "xx":
continue
res = await _sync_non_de_set_with_de(db, cid, str(adminUser.id), adminUser=adminUser)
res = await _syncLanguageWithXx(db, cid, str(adminUser.id), adminUser=adminUser)
results.append(res)
return {"deSync": deSync, "updated": results}
return {"xxSync": xxSync, "updated": results}
@router.put("/sets/{code}")
@ -561,15 +709,12 @@ async def update_language_set(
adminUser: User = Depends(requireSysAdminRole),
):
c = code.strip().lower()
if c in ("update-all", "sync-de"):
if c in ("update-all", "sync-xx", "sync-de"):
raise HTTPException(status_code=400, detail="Ungültiger Sprachcode.")
if c == "xx":
raise HTTPException(status_code=400, detail="Das xx-Set wird über 'UI-Keys einlesen' aktualisiert.")
db = getMgmtInterface(adminUser, mandateId=None).db
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id))
langResult = await _sync_non_de_set_with_de(db, c, str(adminUser.id), adminUser=adminUser)
langResult["deSync"] = deSync
return langResult
return await _syncLanguageWithXx(db, c, str(adminUser.id), adminUser=adminUser)
@router.delete("/sets/{code}")
@ -578,8 +723,8 @@ async def delete_language_set(
adminUser: User = Depends(requireSysAdminRole),
):
c = code.strip().lower()
if c == "de":
raise HTTPException(status_code=400, detail="Das Standard-Set «de» darf nicht gelöscht werden.")
if c in _PROTECTED_CODES:
raise HTTPException(status_code=400, detail=f"Das Set «{c}» darf nicht gelöscht werden.")
db = getMgmtInterface(adminUser, mandateId=None).db
ok = db.recordDelete(UiLanguageSet, c)
if not ok:
@ -597,7 +742,7 @@ async def download_language_set(
if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
payload = _row_to_public(rows[0])
raw = json.dumps(payload.get("keys", {}), ensure_ascii=False, indent=2)
raw = json.dumps(payload, ensure_ascii=False, indent=2)
return Response(
content=raw,
media_type="application/json",
@ -614,15 +759,15 @@ async def download_language_set(
async def export_all_language_sets(
adminUser: User = Depends(requireSysAdminRole),
):
"""Export the complete language database as a JSON array (all sets with full metadata)."""
db = getMgmtInterface(adminUser, mandateId=None).db
rows = db.getRecordset(UiLanguageSet)
payload = []
for r in rows:
entries = _rowEntries(r)
payload.append({
"id": r["id"],
"label": r.get("label", ""),
"keys": dict(r.get("keys") or {}),
"entries": entries,
"status": r.get("status", "complete"),
"isDefault": bool(r.get("isDefault", False)),
})
@ -642,13 +787,6 @@ async def import_language_sets(
file: UploadFile = File(...),
adminUser: User = Depends(requireSysAdminRole),
):
"""Import a previously exported language database JSON.
Behaviour per set in the uploaded array:
- If the set already exists in DB → overwrite keys, label, status, isDefault
- If the set does not exist → create it
Existing sets NOT present in the upload are left untouched (no deletion).
"""
if not file.filename or not file.filename.endswith(".json"):
raise HTTPException(status_code=400, detail="Nur .json-Dateien erlaubt.")
@ -673,8 +811,13 @@ async def import_language_sets(
code = str(entry.get("id", "")).strip().lower()
if not code or len(code) < 2:
continue
entries = entry.get("entries")
if not isinstance(entries, list):
keys = entry.get("keys")
if not isinstance(keys, dict):
if isinstance(keys, dict):
entries = [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
else:
continue
label = str(entry.get("label", code))
@ -684,7 +827,9 @@ async def import_language_sets(
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing:
row = dict(existing[0])
row["keys"] = keys
row["entries"] = entries
if "keys" in row:
del row["keys"]
row["label"] = label
row["status"] = entryStatus
row["isDefault"] = isDefault
@ -696,7 +841,7 @@ async def import_language_sets(
rec = {
"id": code,
"label": label,
"keys": keys,
"entries": entries,
"status": entryStatus,
"isDefault": isDefault,
"sysCreatedAt": now,