issues fixed

This commit is contained in:
ValueOn AG 2026-04-08 22:29:30 +02:00
parent cc1fdb13e5
commit 8b33a86274
4 changed files with 13047 additions and 2045 deletions

View file

@ -1,10 +1,10 @@
# Copyright (c) 2025 Patrick Motsch # Copyright (c) 2025 Patrick Motsch
# All rights reserved. # All rights reserved.
"""UI language sets: global i18n strings (German key -> translated value).""" """UI language sets: structured i18n entries (context, key, value)."""
from typing import Dict, Optional, Literal from typing import List, Literal
from pydantic import Field from pydantic import BaseModel, Field
from modules.datamodels.datamodelBase import PowerOnModel from modules.datamodels.datamodelBase import PowerOnModel
from modules.shared.attributeUtils import registerModelLabels from modules.shared.attributeUtils import registerModelLabels
@ -13,16 +13,40 @@ from modules.shared.attributeUtils import registerModelLabels
UiLanguageStatus = Literal["complete", "incomplete", "generating"] UiLanguageStatus = Literal["complete", "incomplete", "generating"]
class UiLanguageSet(PowerOnModel): class I18nEntry(BaseModel):
"""Single translation entry within a language set.
context: origin of the key, e.g. "ui" for frontend elements,
"db.management.files.name" for backend data objects.
key: German plaintext (the canonical identifier across all sets).
value: For xx (base set): UI context description for AI translation.
For language sets (de, en, ): the translated text.
""" """
One row per ISO 639-1 UI language. id equals code (e.g. de, en).
keys: flat map German plaintext key -> translation for this language. context: str = Field(
For language de, values equal keys. ...,
description="Origin: 'ui' for frontend, 'db.<schema>.<table>.<field>' for backend objects",
)
key: str = Field(
...,
description="German plaintext key (canonical identifier)",
)
value: str = Field(
default="",
description="Translation (language sets) or context description (xx base set)",
)
class UiLanguageSet(PowerOnModel):
"""One row per language. id = ISO 639-1 code or 'xx' (base set).
The xx set is the master: key = German plaintext, value = UI context for AI.
All other sets (incl. de) are AI-generated translations.
""" """
id: str = Field( id: str = Field(
..., ...,
description="ISO 639-1 language code (primary key), e.g. de, en, fr", description="ISO 639-1 language code or 'xx' for the base set",
json_schema_extra={ json_schema_extra={
"frontend_type": "text", "frontend_type": "text",
"frontend_readonly": False, "frontend_readonly": False,
@ -38,9 +62,9 @@ class UiLanguageSet(PowerOnModel):
"frontend_required": True, "frontend_required": True,
}, },
) )
keys: Dict[str, str] = Field( entries: List[I18nEntry] = Field(
default_factory=dict, default_factory=list,
description="German plaintext key -> translated label", description="Translation entries: list of {context, key, value}",
json_schema_extra={ json_schema_extra={
"frontend_type": "textarea", "frontend_type": "textarea",
"frontend_readonly": False, "frontend_readonly": False,
@ -63,7 +87,7 @@ class UiLanguageSet(PowerOnModel):
) )
isDefault: bool = Field( isDefault: bool = Field(
default=False, default=False,
description="Exactly one set should be default (de)", description="True only for the xx base set",
json_schema_extra={ json_schema_extra={
"frontend_type": "boolean", "frontend_type": "boolean",
"frontend_readonly": False, "frontend_readonly": False,
@ -78,7 +102,7 @@ registerModelLabels(
{ {
"id": {"en": "Code", "de": "Code"}, "id": {"en": "Code", "de": "Code"},
"label": {"en": "Label", "de": "Bezeichnung"}, "label": {"en": "Label", "de": "Bezeichnung"},
"keys": {"en": "Keys", "de": "Schlüssel"}, "entries": {"en": "Entries", "de": "Einträge"},
"status": {"en": "Status", "de": "Status"}, "status": {"en": "Status", "de": "Status"},
"isDefault": {"en": "Default", "de": "Standard"}, "isDefault": {"en": "Default", "de": "Standard"},
}, },

View file

@ -219,10 +219,14 @@ class ComponentObjects:
payload = json.loads(seedPath.read_text(encoding="utf-8")) payload = json.loads(seedPath.read_text(encoding="utf-8"))
now = getUtcTimestamp() now = getUtcTimestamp()
for row in payload: for row in payload:
entries = row.get("entries")
if not isinstance(entries, list):
keys = row.get("keys") or {}
entries = [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
rec = { rec = {
"id": row["id"], "id": row["id"],
"label": row["label"], "label": row["label"],
"keys": row.get("keys") or {}, "entries": entries,
"status": row.get("status") or "complete", "status": row.get("status") or "complete",
"isDefault": bool(row.get("isDefault", False)), "isDefault": bool(row.get("isDefault", False)),
"sysCreatedAt": now, "sysCreatedAt": now,

File diff suppressed because it is too large Load diff

View file

@ -3,10 +3,10 @@
""" """
Public and authenticated routes for UI language sets (DB-backed i18n). Public and authenticated routes for UI language sets (DB-backed i18n).
AI translation pipeline: Architecture:
- create_language_set background job translates all keys via AiObjects - xx = base set (meta): key = German plaintext, value = UI context for AI
- update_language_set synchronous AI pass for added keys - All languages (incl. de) are AI-generated translations from xx
- update_all iterates non-de sets - AI translation pipeline uses context from xx to disambiguate translations
""" """
from __future__ import annotations from __future__ import annotations
@ -32,7 +32,7 @@ from modules.datamodels.datamodelAi import (
OperationTypeEnum, OperationTypeEnum,
PriorityEnum, PriorityEnum,
) )
from modules.datamodels.datamodelUiLanguage import UiLanguageSet from modules.datamodels.datamodelUiLanguage import I18nEntry, UiLanguageSet
from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelNotification import NotificationType from modules.datamodels.datamodelNotification import NotificationType
from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface
@ -51,6 +51,41 @@ router = APIRouter(
_MIN_AI_BILLING_ESTIMATE_CHF = 0.01 _MIN_AI_BILLING_ESTIMATE_CHF = 0.01
_TRANSLATE_BATCH_SIZE = 80 _TRANSLATE_BATCH_SIZE = 80
_PROTECTED_CODES = frozenset({"xx", "de"})
# ---------------------------------------------------------------------------
# ISO 639-1 label map (used when creating a language without explicit label)
# ---------------------------------------------------------------------------
_ISO_LABELS: Dict[str, str] = {
"de": "Deutsch", "en": "English", "fr": "Français", "it": "Italiano",
"es": "Español", "pt": "Português", "nl": "Nederlands", "pl": "Polski",
"cs": "Čeština", "sk": "Slovenčina", "sv": "Svenska", "no": "Norsk",
"da": "Dansk", "fi": "Suomi", "hu": "Magyar", "ro": "Română",
"bg": "Български", "hr": "Hrvatski", "sl": "Slovenščina", "et": "Eesti",
"lv": "Latviešu", "lt": "Lietuvių", "el": "Ελληνικά", "tr": "Türkçe",
"ru": "Русский", "uk": "Українська", "ar": "العربية", "he": "עברית",
"zh": "中文", "ja": "日本語", "ko": "한국어", "hi": "हिन्दी",
"th": "ไทย", "vi": "Tiếng Việt", "id": "Bahasa Indonesia", "ms": "Bahasa Melayu",
"tl": "Filipino", "sw": "Kiswahili", "af": "Afrikaans", "sq": "Shqip",
"am": "አማርኛ", "hy": "Հայերեն", "az": "Azərbaycan", "eu": "Euskara",
"be": "Беларуская", "bn": "বাংলা", "bs": "Bosanski", "ca": "Català",
"cy": "Cymraeg", "eo": "Esperanto", "fa": "فارسی", "ga": "Gaeilge",
"gl": "Galego", "gu": "ગુજરાતી", "ha": "Hausa", "is": "Íslenska",
"jv": "Basa Jawa", "ka": "ქართული", "kk": "Қазақ", "km": "ខ្មែរ",
"kn": "ಕನ್ನಡ", "ku": "Kurdî", "ky": "Кыргызча", "la": "Latina",
"lb": "Lëtzebuergesch", "lo": "ລາວ", "mk": "Македонски", "ml": "മലയാളം",
"mn": "Монгол", "mr": "मराठी", "mt": "Malti", "my": "မြန်မာ",
"ne": "नेपाली", "or": "ଓଡ଼ିଆ", "pa": "ਪੰਜਾਬੀ", "ps": "پښتو",
"si": "සිංහල", "so": "Soomaali", "sr": "Српски", "su": "Basa Sunda",
"ta": "தமிழ்", "te": "తెలుగు", "tg": "Тоҷикӣ", "tk": "Türkmen",
"ur": "اردو", "uz": "Oʻzbek", "yo": "Yorùbá", "zu": "isiZulu",
}
# ---------------------------------------------------------------------------
# DB helpers
# ---------------------------------------------------------------------------
def _publicMgmtDb(): def _publicMgmtDb():
return _get_cached_connector( return _get_cached_connector(
@ -63,22 +98,38 @@ def _publicMgmtDb():
) )
def _rowEntries(row: dict) -> List[dict]:
"""Read entries from a DB row, supporting both new (entries) and legacy (keys) format."""
entries = row.get("entries")
if isinstance(entries, list) and entries:
return entries
keys = row.get("keys")
if isinstance(keys, dict) and keys:
return [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
return []
def _entriesToKeyValueMap(entries: List[dict]) -> Dict[str, str]:
"""Convert entries list to a flat key->value map (for frontend consumption)."""
return {e["key"]: e.get("value", "") for e in entries if e.get("key")}
def _row_to_public(row: dict) -> dict: def _row_to_public(row: dict) -> dict:
keys = row.get("keys") or {} entries = _rowEntries(row)
return { return {
"code": row["id"], "code": row["id"],
"label": row.get("label"), "label": row.get("label"),
"status": row.get("status"), "status": row.get("status"),
"keys": keys if isinstance(keys, dict) else {}, "entries": entries,
} }
def _load_master_de_keys(db) -> Dict[str, str]: def _loadMasterXxEntries(db) -> List[dict]:
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"}) """Load the xx base set entries from DB."""
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "xx"})
if not rows: if not rows:
return {} return []
keys = rows[0].get("keys") or {} return _rowEntries(rows[0])
return dict(keys) if isinstance(keys, dict) else {}
def _userMemberMandateIds(currentUser: User) -> List[str]: def _userMemberMandateIds(currentUser: User) -> List[str]:
@ -112,7 +163,6 @@ _aiObjectsSingleton = None
async def _getAiObjects(): async def _getAiObjects():
"""Lazy singleton — same pattern as routeFeatureWorkspace."""
global _aiObjectsSingleton global _aiObjectsSingleton
if _aiObjectsSingleton is None: if _aiObjectsSingleton is None:
from modules.interfaces.interfaceAiObjects import AiObjects from modules.interfaces.interfaceAiObjects import AiObjects
@ -121,7 +171,6 @@ async def _getAiObjects():
def _makeBillingCallback(currentUser: User, mandateId: str): def _makeBillingCallback(currentUser: User, mandateId: str):
"""Return a billing callback that records each AI response cost."""
from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService
billingService = getBillingService(currentUser, mandateId) billingService = getBillingService(currentUser, mandateId)
@ -156,10 +205,10 @@ async def _translateBatch(
targetCode: str, targetCode: str,
billingCallback=None, billingCallback=None,
) -> Dict[str, str]: ) -> Dict[str, str]:
"""Translate a batch of German-key → German-value pairs into *targetLanguageLabel*. """Translate German keys into targetLanguageLabel.
Returns dict { germanKey: translatedValue }. keysToTranslate: { germanKey: uiContext }
Splits into sub-batches of _TRANSLATE_BATCH_SIZE to stay within token limits. Returns: { germanKey: translatedValue }
""" """
if not keysToTranslate: if not keysToTranslate:
return {} return {}
@ -171,14 +220,17 @@ async def _translateBatch(
for batchIdx in range(totalBatches): for batchIdx in range(totalBatches):
chunk = allKeys[batchIdx * _TRANSLATE_BATCH_SIZE : (batchIdx + 1) * _TRANSLATE_BATCH_SIZE] chunk = allKeys[batchIdx * _TRANSLATE_BATCH_SIZE : (batchIdx + 1) * _TRANSLATE_BATCH_SIZE]
payload = {k: v for k, v in chunk} payload = [{"key": k, "context": v} for k, v in chunk]
jsonPayload = json.dumps(payload, ensure_ascii=False) jsonPayload = json.dumps(payload, ensure_ascii=False)
systemPrompt = ( systemPrompt = (
f"Du bist ein professioneller Übersetzer für Software-UI-Texte. " f"Du bist ein professioneller Übersetzer für Software-UI-Texte. "
f"Übersetze die folgenden deutschen UI-Labels ins {targetLanguageLabel} (ISO {targetCode}). " f"Du erhältst ein JSON-Array mit Objekten: {{\"key\": \"deutscher Text\", \"context\": \"UI-Kontext\"}}. "
f"Der Kontext beschreibt, wo der Text in der Anwendung verwendet wird (Datei, Komponente). "
f"Übersetze jeden «key» ins {targetLanguageLabel} (ISO {targetCode}). "
f"Behalte Platzhalter wie {{variable}} exakt bei. " f"Behalte Platzhalter wie {{variable}} exakt bei. "
f"Antworte NUR mit einem JSON-Objekt — gleiche Keys, übersetzte Values. Kein Markdown, kein Kommentar." f"Antworte NUR mit einem JSON-Objekt — Keys = deutsche Originaltexte, Values = Übersetzungen. "
f"Kein Markdown, kein Kommentar."
) )
request = AiCallRequest( request = AiCallRequest(
@ -251,26 +303,21 @@ def _resolveMandateIdForAiI18n(request: Request, currentUser: User) -> str:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# de-Master sync from frontend codebase # xx-Master sync from frontend codebase (local dev fallback)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
_REPO_ROOT = Path(__file__).resolve().parents[3] _REPO_ROOT = Path(__file__).resolve().parents[3]
_FRONTEND_SRC = _REPO_ROOT / "frontend_nyla" / "src" _FRONTEND_SRC = _REPO_ROOT / "frontend_nyla" / "src"
_T_CALL_RE = re.compile( _T_CALL_RE = re.compile(r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))""")
r"""\bt\(\s*'((?:\\.|[^'])+)'\s*(?:,|\))"""
)
def _scanCodebaseKeys() -> Set[str]: def _scanCodebaseKeys() -> List[dict]:
"""Scan all .tsx/.ts files under frontend_nyla/src for t('...') calls. """Local dev fallback: scan frontend src for t() calls. Returns entries with context='ui'."""
Returns the set of German plaintext keys found in the codebase.
"""
keys: Set[str] = set() keys: Set[str] = set()
if not _FRONTEND_SRC.is_dir(): if not _FRONTEND_SRC.is_dir():
logger.warning("i18n codebase scan: %s not found", _FRONTEND_SRC) logger.warning("i18n codebase scan: %s not found", _FRONTEND_SRC)
return keys return []
for ext in ("*.tsx", "*.ts"): for ext in ("*.tsx", "*.ts"):
for filepath in _FRONTEND_SRC.rglob(ext): for filepath in _FRONTEND_SRC.rglob(ext):
@ -283,48 +330,103 @@ def _scanCodebaseKeys() -> Set[str]:
raw = raw.replace("\\'", "'") raw = raw.replace("\\'", "'")
if raw: if raw:
keys.add(raw) keys.add(raw)
return keys return [{"context": "ui", "key": k, "value": ""} for k in sorted(keys)]
def _syncDeMasterFromCodebase(db, userId: Optional[str]) -> Dict[str, Any]: async def _readOptionalEntriesFromBody(request: Request) -> Optional[List[dict]]:
"""Synchronise the de master set with t()-keys found in the frontend codebase. """Read entries from request body. Accepts {entries: [{context, key, value}, ...]}."""
body = await request.body()
if not body or not body.strip():
return None
try:
data = json.loads(body.decode("utf-8"))
except (json.JSONDecodeError, UnicodeDecodeError):
return None
if not isinstance(data, dict) or "entries" not in data:
return None
entries = data.get("entries")
if not isinstance(entries, list):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Feld «entries» muss ein JSON-Array sein.",
)
result = []
for e in entries:
if not isinstance(e, dict) or not e.get("key"):
continue
result.append({
"context": str(e.get("context", "ui")),
"key": str(e["key"]),
"value": str(e.get("value", "")),
})
return result if result else None
- Keys in codebase but not in DB add (key = value = German plaintext)
- Keys in DB but not in codebase remove (orphaned) def _syncXxMaster(db, userId: Optional[str], incomingEntries: List[dict]) -> Dict[str, Any]:
Returns summary dict. """Synchronise the xx base set with incoming entries (from build bundle or codebase scan).
- Keys in incoming but not in DB -> add
- Keys in DB but not in incoming -> remove
- Keys in both -> update context (value)
""" """
codebaseKeys = _scanCodebaseKeys() if not incomingEntries:
if not codebaseKeys: logger.warning("i18n xx-sync: no entries — aborting")
logger.warning("i18n de-sync: codebase scan returned 0 keys — aborting") return {"added": [], "removed": [], "entriesCount": 0, "error": "No entries to sync"}
return {"added": [], "removed": [], "keysCount": 0, "error": "Codebase scan returned 0 keys"}
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "de"}) rows = db.getRecordset(UiLanguageSet, recordFilter={"id": "xx"})
if not rows: if not rows:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht in DB vorhanden.") now = getUtcTimestamp()
rec = {
"id": "xx",
"label": "Basisset (Meta)",
"entries": incomingEntries,
"status": "complete",
"isDefault": True,
"sysCreatedAt": now,
"sysCreatedBy": userId,
"sysModifiedAt": now,
"sysModifiedBy": userId,
}
db.recordCreate(UiLanguageSet, rec)
allKeys = [e["key"] for e in incomingEntries]
logger.info("i18n xx-master created: %d entries", len(incomingEntries))
return {"added": allKeys, "removed": [], "entriesCount": len(incomingEntries)}
row = dict(rows[0]) row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {}) curEntries = _rowEntries(row)
dbKeys = set(cur.keys()) curByKey = {e["key"]: e for e in curEntries}
incomingByKey = {e["key"]: e for e in incomingEntries}
added = sorted(codebaseKeys - dbKeys) incomingKeys = set(incomingByKey.keys())
removed = sorted(dbKeys - codebaseKeys) dbKeys = set(curByKey.keys())
for k in removed: added = sorted(incomingKeys - dbKeys)
del cur[k] removed = sorted(dbKeys - incomingKeys)
for k in added:
cur[k] = k
if not added and not removed: newEntries = []
return {"added": [], "removed": [], "keysCount": len(cur)} for e in incomingEntries:
newEntries.append({"context": e["context"], "key": e["key"], "value": e["value"]})
for e in curEntries:
if e["key"] not in incomingKeys:
continue
if not added and not removed and all(
curByKey.get(e["key"], {}).get("value") == e["value"]
and curByKey.get(e["key"], {}).get("context") == e["context"]
for e in incomingEntries
):
return {"added": [], "removed": [], "entriesCount": len(newEntries)}
now = getUtcTimestamp() now = getUtcTimestamp()
row["keys"] = cur row["entries"] = newEntries
if "keys" in row:
del row["keys"]
row["sysModifiedAt"] = now row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, "de", row) db.recordModify(UiLanguageSet, "xx", row)
logger.info("i18n de-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(cur)) logger.info("i18n xx-master sync: +%d added, -%d removed, total=%d", len(added), len(removed), len(newEntries))
return {"added": added, "removed": removed, "keysCount": len(cur)} return {"added": added, "removed": removed, "entriesCount": len(newEntries)}
# --- Public ----------------------------------------------------------------- # --- Public -----------------------------------------------------------------
@ -336,14 +438,14 @@ async def list_language_codes():
rows = db.getRecordset(UiLanguageSet) rows = db.getRecordset(UiLanguageSet)
out = [] out = []
for r in rows: for r in rows:
keys = r.get("keys") or {} entries = _rowEntries(r)
out.append( out.append(
{ {
"code": r["id"], "code": r["id"],
"label": r.get("label"), "label": r.get("label"),
"status": r.get("status"), "status": r.get("status"),
"isDefault": bool(r.get("isDefault")), "isDefault": bool(r.get("isDefault")),
"keysCount": len(keys) if isinstance(keys, dict) else 0, "entriesCount": len(entries),
} }
) )
return sorted(out, key=lambda x: (not x.get("isDefault"), x["code"])) return sorted(out, key=lambda x: (not x.get("isDefault"), x["code"]))
@ -363,7 +465,7 @@ async def get_language_set(code: str):
class CreateLanguageBody(BaseModel): class CreateLanguageBody(BaseModel):
code: str = Field(..., min_length=2, max_length=10) code: str = Field(..., min_length=2, max_length=10)
label: str = Field(..., min_length=1, max_length=80) label: Optional[str] = Field(default=None, max_length=80)
def _validate_iso2_code(code: str) -> str: def _validate_iso2_code(code: str) -> str:
@ -376,7 +478,6 @@ def _validate_iso2_code(code: str) -> str:
def _run_create_language_job(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None: def _run_create_language_job(userId: str, code: str, label: str, currentUser: User, mandateId: str) -> None:
"""Background job: translate all German master keys via AI, persist, notify user."""
loop = asyncio.new_event_loop() loop = asyncio.new_event_loop()
try: try:
loop.run_until_complete(_run_create_language_job_async(userId, code, label, currentUser, mandateId)) loop.run_until_complete(_run_create_language_job_async(userId, code, label, currentUser, mandateId))
@ -390,24 +491,32 @@ async def _run_create_language_job_async(userId: str, code: str, label: str, cur
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows: if not rows:
return return
deKeys = _load_master_de_keys(db) xxEntries = _loadMasterXxEntries(db)
if not deKeys: if not xxEntries:
logger.error("i18n create job: no de master keys found") logger.error("i18n create job: no xx master entries found")
return return
toTranslate = {e["key"]: e.get("value", "") for e in xxEntries}
billingCb = _makeBillingCallback(currentUser, mandateId) billingCb = _makeBillingCallback(currentUser, mandateId)
translated = await _translateBatch(deKeys, label, code, billingCallback=billingCb) translated = await _translateBatch(toTranslate, label, code, billingCallback=billingCb)
finalKeys: Dict[str, str] = {} finalEntries = []
for k in deKeys: for e in xxEntries:
finalKeys[k] = translated.get(k, f"[{k}]") k = e["key"]
finalEntries.append({
"context": e["context"],
"key": k,
"value": translated.get(k, f"[{k}]"),
})
missingCount = sum(1 for k in deKeys if k not in translated) missingCount = sum(1 for e in xxEntries if e["key"] not in translated)
finalStatus = "complete" if missingCount == 0 else "incomplete" finalStatus = "complete" if missingCount == 0 else "incomplete"
now = getUtcTimestamp() now = getUtcTimestamp()
merged = dict(rows[0]) merged = dict(rows[0])
merged["keys"] = finalKeys merged["entries"] = finalEntries
if "keys" in merged:
del merged["keys"]
merged["status"] = finalStatus merged["status"] = finalStatus
merged["label"] = label merged["label"] = label
merged["sysModifiedAt"] = now merged["sysModifiedAt"] = now
@ -421,7 +530,7 @@ async def _run_create_language_job_async(userId: str, code: str, label: str, cur
title="Sprachset erstellt", title="Sprachset erstellt",
message=f"Die Sprache «{label}» ({code}) wurde per KI übersetzt{statusHint}.", message=f"Die Sprache «{label}» ({code}) wurde per KI übersetzt{statusHint}.",
) )
logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(deKeys)) logger.info("i18n create job done: code=%s, translated=%d/%d", code, len(translated), len(xxEntries))
except Exception as e: except Exception as e:
logger.exception("create language job failed: %s", e) logger.exception("create language job failed: %s", e)
_createNotification( _createNotification(
@ -441,24 +550,28 @@ async def create_language_set(
): ):
mandateId = _resolveMandateIdForAiI18n(request, currentUser) mandateId = _resolveMandateIdForAiI18n(request, currentUser)
code = _validate_iso2_code(body.code) code = _validate_iso2_code(body.code)
if code == "de": if code == "xx":
raise HTTPException(status_code=400, detail="Das Standard-Set «de» kann nicht erneut angelegt werden.") raise HTTPException(status_code=400, detail="Das Basisset «xx» kann nicht manuell angelegt werden.")
db = _publicMgmtDb() db = _publicMgmtDb()
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing: if existing:
raise HTTPException(status_code=409, detail="Dieses Sprachset existiert bereits.") raise HTTPException(status_code=409, detail="Dieses Sprachset existiert bereits.")
deKeys = _load_master_de_keys(db) xxEntries = _loadMasterXxEntries(db)
if not deKeys: if not xxEntries:
raise HTTPException(status_code=503, detail="Deutsch-Master nicht geseedet.") raise HTTPException(status_code=503, detail="Basisset (xx) nicht vorhanden. Bitte zuerst UI-Keys einlesen.")
resolvedLabel = (body.label or "").strip() if body.label else ""
if not resolvedLabel:
resolvedLabel = _ISO_LABELS.get(code, code)
now = getUtcTimestamp() now = getUtcTimestamp()
uid = str(currentUser.id) uid = str(currentUser.id)
rec: dict = { rec: dict = {
"id": code, "id": code,
"label": body.label.strip(), "label": resolvedLabel,
"keys": {}, "entries": [],
"status": "generating", "status": "generating",
"isDefault": False, "isDefault": False,
"sysCreatedAt": now, "sysCreatedAt": now,
@ -468,7 +581,7 @@ async def create_language_set(
} }
db.recordCreate(UiLanguageSet, rec) db.recordCreate(UiLanguageSet, rec)
background.add_task(_run_create_language_job, uid, code, body.label.strip(), currentUser, mandateId) background.add_task(_run_create_language_job, uid, code, resolvedLabel, currentUser, mandateId)
_createNotification( _createNotification(
uid, uid,
NotificationType.SYSTEM, NotificationType.SYSTEM,
@ -478,25 +591,30 @@ async def create_language_set(
return {"status": "accepted", "code": code} return {"status": "accepted", "code": code}
async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict: async def _syncLanguageWithXx(db, code: str, userId: Optional[str], adminUser: Optional[User] = None) -> dict:
if code == "de": """Synchronise a language set (incl. de) against the xx base set via AI."""
raise HTTPException(status_code=400, detail="Das de-Set wird nicht per Update synchronisiert.") if code == "xx":
raise HTTPException(status_code=400, detail="Das xx-Set wird über 'UI-Keys einlesen' aktualisiert.")
rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) rows = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if not rows: if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
deKeys = _load_master_de_keys(db) xxEntries = _loadMasterXxEntries(db)
if not xxEntries:
raise HTTPException(status_code=503, detail="Basisset (xx) nicht vorhanden.")
row = dict(rows[0]) row = dict(rows[0])
cur: Dict[str, str] = dict(row.get("keys") or {}) curEntries = _rowEntries(row)
masterKeys = set(deKeys.keys()) curByKey = {e["key"]: e for e in curEntries}
currentKeys = set(cur.keys()) xxByKey = {e["key"]: e for e in xxEntries}
removed = list(currentKeys - masterKeys)
added = list(masterKeys - currentKeys) masterKeys = set(xxByKey.keys())
for k in removed: currentKeys = set(curByKey.keys())
del cur[k] removedKeys = sorted(currentKeys - masterKeys)
addedKeys = sorted(masterKeys - currentKeys)
translatedCount = 0 translatedCount = 0
if added: if addedKeys:
toTranslate = {k: deKeys[k] for k in added} toTranslate = {k: xxByKey[k].get("value", "") for k in addedKeys}
langLabel = row.get("label") or code langLabel = row.get("label") or code
billingCb = None billingCb = None
if adminUser: if adminUser:
@ -505,54 +623,84 @@ async def _sync_non_de_set_with_de(db, code: str, userId: Optional[str], adminUs
billingCb = _makeBillingCallback(adminUser, memberIds[0]) billingCb = _makeBillingCallback(adminUser, memberIds[0])
try: try:
translated = await _translateBatch(toTranslate, langLabel, code, billingCallback=billingCb) translated = await _translateBatch(toTranslate, langLabel, code, billingCallback=billingCb)
for k in added: translatedCount = sum(1 for k in addedKeys if k in translated)
cur[k] = translated.get(k, f"[{k}]")
translatedCount = sum(1 for k in added if k in translated)
except Exception as e: except Exception as e:
logger.error("AI translation during sync failed for %s: %s", code, e) logger.error("AI translation during sync failed for %s: %s", code, e)
for k in added: translated = {}
cur[k] = f"[{k}]"
for k in addedKeys:
curByKey[k] = {
"context": xxByKey[k]["context"],
"key": k,
"value": translated.get(k, f"[{k}]"),
}
for k in removedKeys:
del curByKey[k]
for k in masterKeys & currentKeys:
curByKey[k]["context"] = xxByKey[k]["context"]
newEntries = [curByKey[k] for k in sorted(curByKey.keys(), key=lambda x: x.lower())]
now = getUtcTimestamp() now = getUtcTimestamp()
row["keys"] = cur untranslated = len(addedKeys) - translatedCount
untranslated = len(added) - translatedCount row["entries"] = newEntries
if "keys" in row:
del row["keys"]
row["status"] = "complete" if untranslated == 0 else "incomplete" row["status"] = "complete" if untranslated == 0 else "incomplete"
row["sysModifiedAt"] = now row["sysModifiedAt"] = now
row["sysModifiedBy"] = userId row["sysModifiedBy"] = userId
db.recordModify(UiLanguageSet, code, row) db.recordModify(UiLanguageSet, code, row)
return {"code": code, "added": added, "removed": removed, "translated": translatedCount, "keysCount": len(cur)} return {
"code": code,
"added": addedKeys,
"removed": removedKeys,
"translated": translatedCount,
"entriesCount": len(newEntries),
}
@router.put("/sets/sync-de") @router.put("/sets/sync-xx")
async def sync_de_master_from_codebase( async def sync_xx_master(
request: Request,
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
"""Scan frontend codebase for t() keys and synchronise the de master set. """Synchronise the xx base set from the frontend build artefact.
Adds new keys (key=value=German plaintext), removes orphaned keys. Expects JSON body: {"entries": [{"context":"ui","key":"","value":""}, ]}
Falls back to local codebase scan if no body provided (dev mode).
""" """
db = getMgmtInterface(adminUser, mandateId=None).db db = getMgmtInterface(adminUser, mandateId=None).db
return _syncDeMasterFromCodebase(db, str(adminUser.id)) fromBody = await _readOptionalEntriesFromBody(request)
entries = fromBody if fromBody is not None else _scanCodebaseKeys()
return _syncXxMaster(db, str(adminUser.id), entries)
@router.put("/sets/update-all") @router.put("/sets/update-all")
async def update_all_language_sets( async def update_all_language_sets(
request: Request,
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
"""Sync de-master from codebase, then update all non-de sets via AI.""" """Sync xx-master (if body provided), then update ALL language sets via AI."""
db = getMgmtInterface(adminUser, mandateId=None).db db = getMgmtInterface(adminUser, mandateId=None).db
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id)) fromBody = await _readOptionalEntriesFromBody(request)
xxSync: Optional[dict] = None
if fromBody is not None:
xxSync = _syncXxMaster(db, str(adminUser.id), fromBody)
if xxSync.get("error"):
return {"xxSync": xxSync, "updated": []}
rows = db.getRecordset(UiLanguageSet) rows = db.getRecordset(UiLanguageSet)
results = [] results = []
for r in rows: for r in rows:
cid = r["id"] cid = r["id"]
if cid == "de": if cid == "xx":
continue continue
res = await _sync_non_de_set_with_de(db, cid, str(adminUser.id), adminUser=adminUser) res = await _syncLanguageWithXx(db, cid, str(adminUser.id), adminUser=adminUser)
results.append(res) results.append(res)
return {"deSync": deSync, "updated": results} return {"xxSync": xxSync, "updated": results}
@router.put("/sets/{code}") @router.put("/sets/{code}")
@ -561,15 +709,12 @@ async def update_language_set(
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
c = code.strip().lower() c = code.strip().lower()
if c in ("update-all", "sync-de"): if c in ("update-all", "sync-xx", "sync-de"):
raise HTTPException(status_code=400, detail="Ungültiger Sprachcode.") raise HTTPException(status_code=400, detail="Ungültiger Sprachcode.")
if c == "xx":
raise HTTPException(status_code=400, detail="Das xx-Set wird über 'UI-Keys einlesen' aktualisiert.")
db = getMgmtInterface(adminUser, mandateId=None).db db = getMgmtInterface(adminUser, mandateId=None).db
return await _syncLanguageWithXx(db, c, str(adminUser.id), adminUser=adminUser)
deSync = _syncDeMasterFromCodebase(db, str(adminUser.id))
langResult = await _sync_non_de_set_with_de(db, c, str(adminUser.id), adminUser=adminUser)
langResult["deSync"] = deSync
return langResult
@router.delete("/sets/{code}") @router.delete("/sets/{code}")
@ -578,8 +723,8 @@ async def delete_language_set(
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
c = code.strip().lower() c = code.strip().lower()
if c == "de": if c in _PROTECTED_CODES:
raise HTTPException(status_code=400, detail="Das Standard-Set «de» darf nicht gelöscht werden.") raise HTTPException(status_code=400, detail=f"Das Set «{c}» darf nicht gelöscht werden.")
db = getMgmtInterface(adminUser, mandateId=None).db db = getMgmtInterface(adminUser, mandateId=None).db
ok = db.recordDelete(UiLanguageSet, c) ok = db.recordDelete(UiLanguageSet, c)
if not ok: if not ok:
@ -597,7 +742,7 @@ async def download_language_set(
if not rows: if not rows:
raise HTTPException(status_code=404, detail="Sprachset nicht gefunden") raise HTTPException(status_code=404, detail="Sprachset nicht gefunden")
payload = _row_to_public(rows[0]) payload = _row_to_public(rows[0])
raw = json.dumps(payload.get("keys", {}), ensure_ascii=False, indent=2) raw = json.dumps(payload, ensure_ascii=False, indent=2)
return Response( return Response(
content=raw, content=raw,
media_type="application/json", media_type="application/json",
@ -614,15 +759,15 @@ async def download_language_set(
async def export_all_language_sets( async def export_all_language_sets(
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
"""Export the complete language database as a JSON array (all sets with full metadata)."""
db = getMgmtInterface(adminUser, mandateId=None).db db = getMgmtInterface(adminUser, mandateId=None).db
rows = db.getRecordset(UiLanguageSet) rows = db.getRecordset(UiLanguageSet)
payload = [] payload = []
for r in rows: for r in rows:
entries = _rowEntries(r)
payload.append({ payload.append({
"id": r["id"], "id": r["id"],
"label": r.get("label", ""), "label": r.get("label", ""),
"keys": dict(r.get("keys") or {}), "entries": entries,
"status": r.get("status", "complete"), "status": r.get("status", "complete"),
"isDefault": bool(r.get("isDefault", False)), "isDefault": bool(r.get("isDefault", False)),
}) })
@ -642,13 +787,6 @@ async def import_language_sets(
file: UploadFile = File(...), file: UploadFile = File(...),
adminUser: User = Depends(requireSysAdminRole), adminUser: User = Depends(requireSysAdminRole),
): ):
"""Import a previously exported language database JSON.
Behaviour per set in the uploaded array:
- If the set already exists in DB overwrite keys, label, status, isDefault
- If the set does not exist create it
Existing sets NOT present in the upload are left untouched (no deletion).
"""
if not file.filename or not file.filename.endswith(".json"): if not file.filename or not file.filename.endswith(".json"):
raise HTTPException(status_code=400, detail="Nur .json-Dateien erlaubt.") raise HTTPException(status_code=400, detail="Nur .json-Dateien erlaubt.")
@ -673,8 +811,13 @@ async def import_language_sets(
code = str(entry.get("id", "")).strip().lower() code = str(entry.get("id", "")).strip().lower()
if not code or len(code) < 2: if not code or len(code) < 2:
continue continue
entries = entry.get("entries")
if not isinstance(entries, list):
keys = entry.get("keys") keys = entry.get("keys")
if not isinstance(keys, dict): if isinstance(keys, dict):
entries = [{"context": "ui", "key": k, "value": v} for k, v in keys.items()]
else:
continue continue
label = str(entry.get("label", code)) label = str(entry.get("label", code))
@ -684,7 +827,9 @@ async def import_language_sets(
existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code}) existing = db.getRecordset(UiLanguageSet, recordFilter={"id": code})
if existing: if existing:
row = dict(existing[0]) row = dict(existing[0])
row["keys"] = keys row["entries"] = entries
if "keys" in row:
del row["keys"]
row["label"] = label row["label"] = label
row["status"] = entryStatus row["status"] = entryStatus
row["isDefault"] = isDefault row["isDefault"] = isDefault
@ -696,7 +841,7 @@ async def import_language_sets(
rec = { rec = {
"id": code, "id": code,
"label": label, "label": label,
"keys": keys, "entries": entries,
"status": entryStatus, "status": entryStatus,
"isDefault": isDefault, "isDefault": isDefault,
"sysCreatedAt": now, "sysCreatedAt": now,