Merge pull request #149 from valueonag/feat/demo-system-readieness
Feat/demo system readieness
This commit is contained in:
commit
7942766931
58 changed files with 2689 additions and 1139 deletions
3
app.py
3
app.py
|
|
@ -682,6 +682,9 @@ app.include_router(navigationRouter)
|
||||||
from modules.routes.routeWorkflowDashboard import router as workflowDashboardRouter
|
from modules.routes.routeWorkflowDashboard import router as workflowDashboardRouter
|
||||||
app.include_router(workflowDashboardRouter)
|
app.include_router(workflowDashboardRouter)
|
||||||
|
|
||||||
|
from modules.routes.routeAutomationWorkspace import router as automationWorkspaceRouter
|
||||||
|
app.include_router(automationWorkspaceRouter)
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# PLUG&PLAY FEATURE ROUTERS
|
# PLUG&PLAY FEATURE ROUTERS
|
||||||
# Dynamically load routers from feature containers in modules/features/
|
# Dynamically load routers from feature containers in modules/features/
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,7 @@ class AiCallOptions(BaseModel):
|
||||||
|
|
||||||
# Provider filtering (from UI multiselect or automation config)
|
# Provider filtering (from UI multiselect or automation config)
|
||||||
allowedProviders: Optional[List[str]] = Field(default=None, description="List of allowed AI providers to use (empty = all RBAC-permitted)")
|
allowedProviders: Optional[List[str]] = Field(default=None, description="List of allowed AI providers to use (empty = all RBAC-permitted)")
|
||||||
|
allowedModels: Optional[List[str]] = Field(default=None, description="Whitelist of allowed model names (AND-filter with allowedProviders). None/empty = all allowed.")
|
||||||
|
|
||||||
|
|
||||||
class AiCallRequest(BaseModel):
|
class AiCallRequest(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -110,11 +110,13 @@ class DocumentReferenceList(BaseModel):
|
||||||
# docItem:documentId
|
# docItem:documentId
|
||||||
references.append(DocumentItemReference(documentId=parts[0]))
|
references.append(DocumentItemReference(documentId=parts[0]))
|
||||||
|
|
||||||
# Unknown format - skip or log warning
|
|
||||||
else:
|
else:
|
||||||
# Try to parse as simple string (backward compatibility)
|
if not refStr:
|
||||||
# Assume it's a label if it doesn't match known patterns
|
continue
|
||||||
if refStr:
|
import re
|
||||||
|
if re.match(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', refStr, re.I):
|
||||||
|
references.append(DocumentItemReference(documentId=refStr))
|
||||||
|
else:
|
||||||
references.append(DocumentListReference(label=refStr))
|
references.append(DocumentListReference(label=refStr))
|
||||||
|
|
||||||
return cls(references=references)
|
return cls(references=references)
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ Unified JSON document schema and helpers used by both generation prompts and ren
|
||||||
This defines a single canonical template and the supported section types.
|
This defines a single canonical template and the supported section types.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import List
|
from typing import List, Literal, TypedDict
|
||||||
|
|
||||||
# Canonical list of supported section types across the system
|
# Canonical list of supported section types across the system
|
||||||
supportedSectionTypes: List[str] = [
|
supportedSectionTypes: List[str] = [
|
||||||
|
|
@ -18,6 +18,21 @@ supportedSectionTypes: List[str] = [
|
||||||
"image",
|
"image",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
class InlineRun(TypedDict, total=False):
|
||||||
|
"""Single inline content run. Every paragraph/cell/list-item is a List[InlineRun]."""
|
||||||
|
type: Literal["text", "image", "link", "bold", "italic", "code"]
|
||||||
|
value: str # text content (for text/bold/italic/code/link-label)
|
||||||
|
fileId: str # for type=image: reference to FileItem
|
||||||
|
base64Data: str # for type=image: resolved base64 (post-processing)
|
||||||
|
mimeType: str # for type=image: e.g. "image/png"
|
||||||
|
widthPt: int # for type=image: optional render width
|
||||||
|
href: str # for type=link: URL target
|
||||||
|
|
||||||
|
supportedInlineRunTypes: List[str] = [
|
||||||
|
"text", "image", "link", "bold", "italic", "code",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
# Canonical JSON template used for AI generation (documents array + sections)
|
# Canonical JSON template used for AI generation (documents array + sections)
|
||||||
# This template is used for STRUCTURE generation - sections have empty elements arrays.
|
# This template is used for STRUCTURE generation - sections have empty elements arrays.
|
||||||
# For content generation, elements arrays will be populated later.
|
# For content generation, elements arrays will be populated later.
|
||||||
|
|
|
||||||
|
|
@ -72,7 +72,7 @@ class AutoWorkflow(PowerOnModel):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
featureInstanceId: str = Field(
|
featureInstanceId: str = Field(
|
||||||
description="Feature instance ID",
|
description="Feature instance ID (GE owner instance / RBAC scope)",
|
||||||
json_schema_extra={
|
json_schema_extra={
|
||||||
"frontend_type": "text",
|
"frontend_type": "text",
|
||||||
"frontend_readonly": True,
|
"frontend_readonly": True,
|
||||||
|
|
@ -81,6 +81,17 @@ class AutoWorkflow(PowerOnModel):
|
||||||
"fk_target": {"db": "poweron_app", "table": "FeatureInstance", "labelField": "label"},
|
"fk_target": {"db": "poweron_app", "table": "FeatureInstance", "labelField": "label"},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
targetFeatureInstanceId: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Target feature instance for execution data scope. NULL for templates, mandatory for non-templates.",
|
||||||
|
json_schema_extra={
|
||||||
|
"frontend_type": "select",
|
||||||
|
"frontend_readonly": False,
|
||||||
|
"frontend_required": False,
|
||||||
|
"label": "Ziel-Instanz",
|
||||||
|
"fk_target": {"db": "poweron_app", "table": "FeatureInstance", "labelField": "label"},
|
||||||
|
},
|
||||||
|
)
|
||||||
label: str = Field(
|
label: str = Field(
|
||||||
description="User-friendly workflow name",
|
description="User-friendly workflow name",
|
||||||
json_schema_extra={"frontend_type": "text", "frontend_required": True, "label": "Bezeichnung"},
|
json_schema_extra={"frontend_type": "text", "frontend_required": True, "label": "Bezeichnung"},
|
||||||
|
|
|
||||||
|
|
@ -217,6 +217,8 @@ class GraphicalEditorObjects:
|
||||||
data["id"] = str(uuid.uuid4())
|
data["id"] = str(uuid.uuid4())
|
||||||
data["mandateId"] = self.mandateId
|
data["mandateId"] = self.mandateId
|
||||||
data["featureInstanceId"] = self.featureInstanceId
|
data["featureInstanceId"] = self.featureInstanceId
|
||||||
|
if not data.get("targetFeatureInstanceId") and not data.get("isTemplate"):
|
||||||
|
data["targetFeatureInstanceId"] = self.featureInstanceId
|
||||||
if "active" not in data or data.get("active") is None:
|
if "active" not in data or data.get("active") is None:
|
||||||
data["active"] = True
|
data["active"] = True
|
||||||
data["invocations"] = normalize_invocations_list(data.get("invocations"))
|
data["invocations"] = normalize_invocations_list(data.get("invocations"))
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,15 @@
|
||||||
|
|
||||||
from modules.shared.i18nRegistry import t
|
from modules.shared.i18nRegistry import t
|
||||||
|
|
||||||
|
_AI_COMMON_PARAMS = [
|
||||||
|
{"name": "requireNeutralization", "type": "boolean", "required": False,
|
||||||
|
"frontendType": "checkbox", "default": False,
|
||||||
|
"description": t("Eingaben fuer diesen Call neutralisieren")},
|
||||||
|
{"name": "allowedModels", "type": "array", "required": False,
|
||||||
|
"frontendType": "modelMultiSelect", "default": [],
|
||||||
|
"description": t("Erlaubte LLM-Modelle (leer = alle erlaubten)")},
|
||||||
|
]
|
||||||
|
|
||||||
AI_NODES = [
|
AI_NODES = [
|
||||||
{
|
{
|
||||||
"id": "ai.prompt",
|
"id": "ai.prompt",
|
||||||
|
|
@ -10,16 +19,21 @@ AI_NODES = [
|
||||||
"label": t("Prompt"),
|
"label": t("Prompt"),
|
||||||
"description": t("Prompt eingeben und KI führt aus"),
|
"description": t("Prompt eingeben und KI führt aus"),
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "aiPrompt", "type": "string", "required": True, "frontendType": "textarea",
|
{"name": "aiPrompt", "type": "string", "required": True, "frontendType": "templateTextarea",
|
||||||
"description": t("KI-Prompt")},
|
"description": t("KI-Prompt")},
|
||||||
{"name": "resultType", "type": "string", "required": False, "frontendType": "select",
|
{"name": "resultType", "type": "string", "required": False, "frontendType": "select",
|
||||||
"frontendOptions": {"options": ["txt", "json", "md", "csv", "xml", "html", "pdf", "docx", "xlsx", "pptx", "png", "jpg"]},
|
"frontendOptions": {"options": ["txt", "json", "md", "csv", "xml", "html", "pdf", "docx", "xlsx", "pptx", "png", "jpg"]},
|
||||||
"description": t("Ausgabeformat"), "default": "txt"},
|
"description": t("Ausgabeformat"), "default": "txt"},
|
||||||
{"name": "documentList", "type": "string", "required": False, "frontendType": "hidden",
|
{"name": "documentList", "type": "DocumentList", "required": False, "frontendType": "dataRef",
|
||||||
"description": t("Dokumentenliste (via Wire oder DataRef)"), "default": ""},
|
"description": t("Dokumentenliste (Upstream-Output binden)"), "default": ""},
|
||||||
|
{"name": "context", "type": "string", "required": False, "frontendType": "dataRef",
|
||||||
|
"description": t("Kontextdaten fuer den Prompt (Upstream-Output binden)"), "default": ""},
|
||||||
|
{"name": "documentTheme", "type": "string", "required": False, "frontendType": "select",
|
||||||
|
"frontendOptions": {"options": ["general", "finance", "legal", "technical", "hr"]},
|
||||||
|
"description": t("Dokument-Thema (Style-Hinweis fuer den Renderer)"), "default": "general"},
|
||||||
{"name": "simpleMode", "type": "boolean", "required": False, "frontendType": "checkbox",
|
{"name": "simpleMode", "type": "boolean", "required": False, "frontendType": "checkbox",
|
||||||
"description": t("Einfacher Modus"), "default": True},
|
"description": t("Einfacher Modus"), "default": True},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": [
|
"inputPorts": {0: {"accepts": [
|
||||||
|
|
@ -38,7 +52,7 @@ AI_NODES = [
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "prompt", "type": "string", "required": True, "frontendType": "textarea",
|
{"name": "prompt", "type": "string", "required": True, "frontendType": "textarea",
|
||||||
"description": t("Recherche-Anfrage")},
|
"description": t("Recherche-Anfrage")},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["Transit"]}},
|
"inputPorts": {0: {"accepts": ["Transit"]}},
|
||||||
|
|
@ -53,12 +67,12 @@ AI_NODES = [
|
||||||
"label": t("Dokument zusammenfassen"),
|
"label": t("Dokument zusammenfassen"),
|
||||||
"description": t("Dokumentinhalt zusammenfassen"),
|
"description": t("Dokumentinhalt zusammenfassen"),
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "documentList", "type": "string", "required": True, "frontendType": "hidden",
|
{"name": "documentList", "type": "DocumentList", "required": True, "frontendType": "dataRef",
|
||||||
"description": t("Dokumentenliste (via Wire oder DataRef)"), "default": ""},
|
"description": t("Dokumentenliste (Upstream-Output binden)"), "default": ""},
|
||||||
{"name": "summaryLength", "type": "string", "required": False, "frontendType": "select",
|
{"name": "summaryLength", "type": "string", "required": False, "frontendType": "select",
|
||||||
"frontendOptions": {"options": ["brief", "medium", "detailed"]},
|
"frontendOptions": {"options": ["brief", "medium", "detailed"]},
|
||||||
"description": t("Kurz, mittel oder ausführlich"), "default": "medium"},
|
"description": t("Kurz, mittel oder ausführlich"), "default": "medium"},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
||||||
|
|
@ -73,11 +87,11 @@ AI_NODES = [
|
||||||
"label": t("Dokument übersetzen"),
|
"label": t("Dokument übersetzen"),
|
||||||
"description": t("Dokument in Zielsprache übersetzen"),
|
"description": t("Dokument in Zielsprache übersetzen"),
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "documentList", "type": "string", "required": True, "frontendType": "hidden",
|
{"name": "documentList", "type": "DocumentList", "required": True, "frontendType": "dataRef",
|
||||||
"description": t("Dokumentenliste (via Wire oder DataRef)"), "default": ""},
|
"description": t("Dokumentenliste (Upstream-Output binden)"), "default": ""},
|
||||||
{"name": "targetLanguage", "type": "string", "required": True, "frontendType": "text",
|
{"name": "targetLanguage", "type": "string", "required": True, "frontendType": "text",
|
||||||
"description": t("Zielsprache (z.B. de, en, French)")},
|
"description": t("Zielsprache (z.B. de, en, French)")},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
||||||
|
|
@ -92,12 +106,12 @@ AI_NODES = [
|
||||||
"label": t("Dokument konvertieren"),
|
"label": t("Dokument konvertieren"),
|
||||||
"description": t("Dokument in anderes Format konvertieren"),
|
"description": t("Dokument in anderes Format konvertieren"),
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "documentList", "type": "string", "required": True, "frontendType": "hidden",
|
{"name": "documentList", "type": "DocumentList", "required": True, "frontendType": "dataRef",
|
||||||
"description": t("Dokumentenliste (via Wire oder DataRef)"), "default": ""},
|
"description": t("Dokumentenliste (Upstream-Output binden)"), "default": ""},
|
||||||
{"name": "targetFormat", "type": "string", "required": True, "frontendType": "select",
|
{"name": "targetFormat", "type": "string", "required": True, "frontendType": "select",
|
||||||
"frontendOptions": {"options": ["docx", "pdf", "xlsx", "csv", "txt", "html", "json", "md"]},
|
"frontendOptions": {"options": ["docx", "pdf", "xlsx", "csv", "txt", "html", "json", "md"]},
|
||||||
"description": t("Zielformat")},
|
"description": t("Zielformat")},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}},
|
||||||
|
|
@ -114,7 +128,7 @@ AI_NODES = [
|
||||||
"parameters": [
|
"parameters": [
|
||||||
{"name": "prompt", "type": "string", "required": True, "frontendType": "textarea",
|
{"name": "prompt", "type": "string", "required": True, "frontendType": "textarea",
|
||||||
"description": t("Generierungs-Prompt")},
|
"description": t("Generierungs-Prompt")},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["Transit"]}},
|
"inputPorts": {0: {"accepts": ["Transit"]}},
|
||||||
|
|
@ -134,7 +148,7 @@ AI_NODES = [
|
||||||
{"name": "resultType", "type": "string", "required": False, "frontendType": "select",
|
{"name": "resultType", "type": "string", "required": False, "frontendType": "select",
|
||||||
"frontendOptions": {"options": ["py", "js", "ts", "html", "java", "cpp", "txt", "json", "csv", "xml"]},
|
"frontendOptions": {"options": ["py", "js", "ts", "html", "java", "cpp", "txt", "json", "csv", "xml"]},
|
||||||
"description": t("Datei-Endung der erzeugten Code-Datei"), "default": "py"},
|
"description": t("Datei-Endung der erzeugten Code-Datei"), "default": "py"},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["Transit"]}},
|
"inputPorts": {0: {"accepts": ["Transit"]}},
|
||||||
|
|
@ -154,7 +168,7 @@ AI_NODES = [
|
||||||
"description": t("Konsolidierungsmodus"), "default": "summarize"},
|
"description": t("Konsolidierungsmodus"), "default": "summarize"},
|
||||||
{"name": "prompt", "type": "string", "required": False, "frontendType": "textarea",
|
{"name": "prompt", "type": "string", "required": False, "frontendType": "textarea",
|
||||||
"description": t("Optionaler Prompt für die Konsolidierung"), "default": ""},
|
"description": t("Optionaler Prompt für die Konsolidierung"), "default": ""},
|
||||||
],
|
] + _AI_COMMON_PARAMS,
|
||||||
"inputs": 1,
|
"inputs": 1,
|
||||||
"outputs": 1,
|
"outputs": 1,
|
||||||
"inputPorts": {0: {"accepts": ["AggregateResult", "Transit"]}},
|
"inputPorts": {0: {"accepts": ["AggregateResult", "Transit"]}},
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,7 @@ EMAIL_NODES = [
|
||||||
{"name": "connectionReference", "type": "string", "required": True, "frontendType": "userConnection",
|
{"name": "connectionReference", "type": "string", "required": True, "frontendType": "userConnection",
|
||||||
"frontendOptions": {"authority": "msft"},
|
"frontendOptions": {"authority": "msft"},
|
||||||
"description": t("E-Mail-Konto")},
|
"description": t("E-Mail-Konto")},
|
||||||
{"name": "context", "type": "string", "required": False, "frontendType": "textarea",
|
{"name": "context", "type": "string", "required": False, "frontendType": "templateTextarea",
|
||||||
"description": t("Kontext / Brief-Beschreibung für die KI-Komposition"), "default": ""},
|
"description": t("Kontext / Brief-Beschreibung für die KI-Komposition"), "default": ""},
|
||||||
{"name": "to", "type": "string", "required": False, "frontendType": "text",
|
{"name": "to", "type": "string", "required": False, "frontendType": "text",
|
||||||
"description": t("Empfänger (komma-separiert, optional für Entwurf)"), "default": ""},
|
"description": t("Empfänger (komma-separiert, optional für Entwurf)"), "default": ""},
|
||||||
|
|
|
||||||
|
|
@ -111,6 +111,44 @@ def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str:
|
||||||
return str(instance.mandateId) if instance.mandateId else ""
|
return str(instance.mandateId) if instance.mandateId else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _validateTargetInstance(
|
||||||
|
workflowData: Dict[str, Any],
|
||||||
|
ownerInstanceId: str,
|
||||||
|
context: RequestContext,
|
||||||
|
) -> None:
|
||||||
|
"""Enforce targetFeatureInstanceId rules for non-template workflows.
|
||||||
|
|
||||||
|
- Templates (isTemplate=True) may omit targetFeatureInstanceId.
|
||||||
|
- Non-templates MUST have a non-empty targetFeatureInstanceId.
|
||||||
|
- If the targetFeatureInstanceId differs from the GE owner instance,
|
||||||
|
the user must also have FeatureAccess on that target instance.
|
||||||
|
"""
|
||||||
|
if workflowData.get("isTemplate"):
|
||||||
|
return
|
||||||
|
|
||||||
|
targetId = workflowData.get("targetFeatureInstanceId")
|
||||||
|
if not targetId:
|
||||||
|
return
|
||||||
|
|
||||||
|
if targetId == ownerInstanceId:
|
||||||
|
return
|
||||||
|
|
||||||
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
|
rootInterface = getRootInterface()
|
||||||
|
targetInstance = rootInterface.getFeatureInstance(targetId)
|
||||||
|
if not targetInstance:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=routeApiMsg("targetFeatureInstanceId refers to a non-existent feature instance"),
|
||||||
|
)
|
||||||
|
targetAccess = rootInterface.getFeatureAccess(str(context.user.id), targetId)
|
||||||
|
if not targetAccess or not targetAccess.enabled:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=403,
|
||||||
|
detail=routeApiMsg("Access denied to target feature instance"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{instanceId}/node-types")
|
@router.get("/{instanceId}/node-types")
|
||||||
@limiter.limit("60/minute")
|
@limiter.limit("60/minute")
|
||||||
def get_node_types(
|
def get_node_types(
|
||||||
|
|
@ -318,9 +356,12 @@ async def post_execute(
|
||||||
workflowId = body.get("workflowId")
|
workflowId = body.get("workflowId")
|
||||||
req_nodes = graph.get("nodes") or []
|
req_nodes = graph.get("nodes") or []
|
||||||
workflow_for_envelope: Optional[Dict[str, Any]] = None
|
workflow_for_envelope: Optional[Dict[str, Any]] = None
|
||||||
|
targetFeatureInstanceId: Optional[str] = None
|
||||||
if workflowId and not str(workflowId).startswith("transient-"):
|
if workflowId and not str(workflowId).startswith("transient-"):
|
||||||
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
||||||
workflow_for_envelope = iface.getWorkflow(workflowId)
|
workflow_for_envelope = iface.getWorkflow(workflowId)
|
||||||
|
if workflow_for_envelope:
|
||||||
|
targetFeatureInstanceId = workflow_for_envelope.get("targetFeatureInstanceId")
|
||||||
if workflowId and len(req_nodes) == 0:
|
if workflowId and len(req_nodes) == 0:
|
||||||
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
||||||
wf = iface.getWorkflow(workflowId)
|
wf = iface.getWorkflow(workflowId)
|
||||||
|
|
@ -328,10 +369,18 @@ async def post_execute(
|
||||||
graph = wf["graph"]
|
graph = wf["graph"]
|
||||||
logger.info("graphicalEditor execute: loaded graph from workflow %s", workflowId)
|
logger.info("graphicalEditor execute: loaded graph from workflow %s", workflowId)
|
||||||
workflow_for_envelope = wf
|
workflow_for_envelope = wf
|
||||||
|
targetFeatureInstanceId = wf.get("targetFeatureInstanceId")
|
||||||
if not workflowId:
|
if not workflowId:
|
||||||
import uuid
|
import uuid
|
||||||
workflowId = f"transient-{uuid.uuid4().hex[:12]}"
|
workflowId = f"transient-{uuid.uuid4().hex[:12]}"
|
||||||
logger.info("graphicalEditor execute: using transient workflowId=%s", workflowId)
|
logger.info("graphicalEditor execute: using transient workflowId=%s", workflowId)
|
||||||
|
|
||||||
|
if targetFeatureInstanceId and targetFeatureInstanceId != instanceId:
|
||||||
|
_validateTargetInstance(
|
||||||
|
{"targetFeatureInstanceId": targetFeatureInstanceId},
|
||||||
|
instanceId,
|
||||||
|
context,
|
||||||
|
)
|
||||||
nodes_count = len(graph.get("nodes") or [])
|
nodes_count = len(graph.get("nodes") or [])
|
||||||
connections_count = len(graph.get("connections") or [])
|
connections_count = len(graph.get("connections") or [])
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
@ -363,6 +412,7 @@ async def post_execute(
|
||||||
automation2_interface=ge_interface,
|
automation2_interface=ge_interface,
|
||||||
run_envelope=run_env,
|
run_envelope=run_env,
|
||||||
label=_wfLabel,
|
label=_wfLabel,
|
||||||
|
targetFeatureInstanceId=targetFeatureInstanceId,
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
"graphicalEditor execute result: success=%s error=%s nodeOutputs_keys=%s failedNode=%s paused=%s",
|
"graphicalEditor execute result: success=%s error=%s nodeOutputs_keys=%s failedNode=%s paused=%s",
|
||||||
|
|
@ -1371,6 +1421,7 @@ def create_workflow(
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Create a new workflow."""
|
"""Create a new workflow."""
|
||||||
mandateId = _validateInstanceAccess(instanceId, context)
|
mandateId = _validateInstanceAccess(instanceId, context)
|
||||||
|
_validateTargetInstance(body, instanceId, context)
|
||||||
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
||||||
created = iface.createWorkflow(body)
|
created = iface.createWorkflow(body)
|
||||||
return created
|
return created
|
||||||
|
|
@ -1388,6 +1439,11 @@ def update_workflow(
|
||||||
"""Update a workflow."""
|
"""Update a workflow."""
|
||||||
mandateId = _validateInstanceAccess(instanceId, context)
|
mandateId = _validateInstanceAccess(instanceId, context)
|
||||||
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
iface = getGraphicalEditorInterface(context.user, mandateId, instanceId)
|
||||||
|
existing = iface.getWorkflow(workflowId)
|
||||||
|
if not existing:
|
||||||
|
raise HTTPException(status_code=404, detail=routeApiMsg("Workflow not found"))
|
||||||
|
merged = {**existing, **body}
|
||||||
|
_validateTargetInstance(merged, instanceId, context)
|
||||||
updated = iface.updateWorkflow(workflowId, body)
|
updated = iface.updateWorkflow(workflowId, body)
|
||||||
if not updated:
|
if not updated:
|
||||||
raise HTTPException(status_code=404, detail=routeApiMsg("Workflow not found"))
|
raise HTTPException(status_code=404, detail=routeApiMsg("Workflow not found"))
|
||||||
|
|
|
||||||
|
|
@ -361,6 +361,17 @@ QUICK_ACTIONS = [
|
||||||
# The placeholder {{featureInstanceId}} is replaced by _copyTemplateWorkflows.
|
# The placeholder {{featureInstanceId}} is replaced by _copyTemplateWorkflows.
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_FINANCE_STYLE_HINT = (
|
||||||
|
"\n\nWenn du ein Dokument erstellst, verwende einen professionellen Finanz-Stil:\n"
|
||||||
|
"- Schriftart: Calibri\n"
|
||||||
|
"- Primaerfarbe: #1F3864 (Dunkelblau)\n"
|
||||||
|
"- Akzentfarbe: #2980B9\n"
|
||||||
|
"- Tabellen mit dunklem Header (#1F3864, weisse Schrift)\n"
|
||||||
|
"- Konservatives, seriöses Layout\n"
|
||||||
|
"Nutze den style-Parameter von renderDocument um diese Vorgaben umzusetzen."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _buildAnalysisWorkflowGraph(prompt: str) -> Dict[str, Any]:
|
def _buildAnalysisWorkflowGraph(prompt: str) -> Dict[str, Any]:
|
||||||
"""Build a standard analysis graph: trigger -> refreshAccountingData -> ai.prompt."""
|
"""Build a standard analysis graph: trigger -> refreshAccountingData -> ai.prompt."""
|
||||||
return {
|
return {
|
||||||
|
|
@ -370,8 +381,9 @@ def _buildAnalysisWorkflowGraph(prompt: str) -> Dict[str, Any]:
|
||||||
"parameters": {"featureInstanceId": "{{featureInstanceId}}", "forceRefresh": False}, "position": {"x": 250, "y": 0}},
|
"parameters": {"featureInstanceId": "{{featureInstanceId}}", "forceRefresh": False}, "position": {"x": 250, "y": 0}},
|
||||||
{"id": "analyse", "type": "ai.prompt", "label": "Analyse", "_method": "ai", "_action": "process",
|
{"id": "analyse", "type": "ai.prompt", "label": "Analyse", "_method": "ai", "_action": "process",
|
||||||
"parameters": {
|
"parameters": {
|
||||||
"aiPrompt": prompt,
|
"aiPrompt": prompt + _FINANCE_STYLE_HINT,
|
||||||
"context": {"type": "ref", "nodeId": "refresh", "path": ["data", "accountingData"]},
|
"context": {"type": "ref", "nodeId": "refresh", "path": ["data", "accountingData"]},
|
||||||
|
"requireNeutralization": False,
|
||||||
"simpleMode": False,
|
"simpleMode": False,
|
||||||
}, "position": {"x": 500, "y": 0}},
|
}, "position": {"x": 500, "y": 0}},
|
||||||
],
|
],
|
||||||
|
|
@ -440,15 +452,33 @@ TEMPLATE_WORKFLOWS = [
|
||||||
{"id": "analyse", "type": "ai.prompt", "label": "Budget-Analyse", "_method": "ai", "_action": "process",
|
{"id": "analyse", "type": "ai.prompt", "label": "Budget-Analyse", "_method": "ai", "_action": "process",
|
||||||
"parameters": {
|
"parameters": {
|
||||||
"aiPrompt": (
|
"aiPrompt": (
|
||||||
"Fuehre einen Budget-Soll/Ist-Vergleich durch.\n"
|
"Fuehre einen Budget-Soll/Ist-Vergleich durch und liefere EIN Excel-Dokument "
|
||||||
"Die Budget-Datei (Excel) wurde als Dokument uebergeben. "
|
"mit folgender Struktur:\n\n"
|
||||||
"Die aktuellen Buchhaltungsdaten sind im Kontext verfuegbar.\n"
|
"1. Tabelle \"Konten-Vergleich\" -- EINE Tabelle, EINE Zeile pro Konto:\n"
|
||||||
"1. Lies die Soll-Werte aus dem uebergebenen Budget-Dokument\n"
|
" Spalten: Konto-Nr | Konto-Name | Soll | Ist | Abweichung absolut | "
|
||||||
"2. Vergleiche sie mit den Ist-Werten aus der Buchhaltung pro Konto\n"
|
"Abweichung % | Status (OK / Warnung / Kritisch).\n"
|
||||||
"3. Berechne die Abweichung (absolut und prozentual)\n"
|
"2. EINE Visualisierung \"Soll vs. Ist gesamt\" -- ein einziges "
|
||||||
"4. Erstelle ein Abweichungs-Chart (Balkendiagramm: Soll vs. Ist pro Konto)\n"
|
"Balkendiagramm UNTER der Tabelle, das ALLE Konten in einer Grafik "
|
||||||
"5. Markiere kritische Abweichungen (>10%) und gib eine kurze Einschaetzung"
|
"gegenueberstellt (gruppierte Balken: Soll und Ist je Konto).\n"
|
||||||
|
"3. Kurzer Management-Summary-Absatz (3-5 Saetze) UNTER dem Chart "
|
||||||
|
"mit den 3 groessten Abweichungen (>10%) und einer fachlichen "
|
||||||
|
"Einschaetzung.\n\n"
|
||||||
|
"Verwende die uebergebene Budget-Datei als Soll-Quelle und die im "
|
||||||
|
"Kontext bereitgestellten Buchhaltungsdaten als Ist-Quelle.\n"
|
||||||
|
"WICHTIG: Erstelle KEINEN separaten Chart pro Konto. Nur EIN "
|
||||||
|
"Uebersichts-Chart ueber alle Konten ist gewuenscht.\n\n"
|
||||||
|
"Hinweis: Das documentTheme ist 'finance'. Wenn du ein Dokument erstellst, "
|
||||||
|
"verwende einen professionellen Finanz-Stil:\n"
|
||||||
|
"- Schriftart: Calibri\n"
|
||||||
|
"- Primaerfarbe: #1F3864 (Dunkelblau)\n"
|
||||||
|
"- Akzentfarbe: #2980B9\n"
|
||||||
|
"- Tabellen mit dunklem Header (#1F3864, weisse Schrift)\n"
|
||||||
|
"- Konservatives, seriöses Layout\n"
|
||||||
|
"Nutze den style-Parameter von renderDocument um diese Vorgaben umzusetzen."
|
||||||
),
|
),
|
||||||
|
"resultType": "xlsx",
|
||||||
|
"documentTheme": "finance",
|
||||||
|
"requireNeutralization": False,
|
||||||
"documentList": {"type": "ref", "nodeId": "trigger", "path": ["payload", "documentList"]},
|
"documentList": {"type": "ref", "nodeId": "trigger", "path": ["payload", "documentList"]},
|
||||||
"context": {"type": "ref", "nodeId": "refresh", "path": ["data", "accountingData"]},
|
"context": {"type": "ref", "nodeId": "refresh", "path": ["data", "accountingData"]},
|
||||||
"simpleMode": False,
|
"simpleMode": False,
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@
|
||||||
# All rights reserved.
|
# All rights reserved.
|
||||||
"""Workspace feature data models — WorkspaceUserSettings."""
|
"""Workspace feature data models — WorkspaceUserSettings."""
|
||||||
|
|
||||||
from typing import Optional
|
from typing import List, Optional
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import Field
|
||||||
from modules.datamodels.datamodelBase import PowerOnModel
|
from modules.datamodels.datamodelBase import PowerOnModel
|
||||||
from modules.shared.i18nRegistry import i18nModel
|
from modules.shared.i18nRegistry import i18nModel
|
||||||
import uuid
|
import uuid
|
||||||
|
|
@ -52,3 +52,18 @@ class WorkspaceUserSettings(PowerOnModel):
|
||||||
description="Max agent rounds override (None = instance default)",
|
description="Max agent rounds override (None = instance default)",
|
||||||
json_schema_extra={"label": "Max. Agenten-Runden", "frontend_type": "number", "frontend_readonly": False, "frontend_required": False},
|
json_schema_extra={"label": "Max. Agenten-Runden", "frontend_type": "number", "frontend_readonly": False, "frontend_required": False},
|
||||||
)
|
)
|
||||||
|
requireNeutralization: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description="Default neutralization setting for this user",
|
||||||
|
json_schema_extra={"label": "Neutralisierung", "frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False},
|
||||||
|
)
|
||||||
|
allowedProviders: List[str] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
description="Allowed AI providers (empty = all permitted by RBAC)",
|
||||||
|
json_schema_extra={"label": "Erlaubte Provider", "frontend_type": "multiselect", "frontend_readonly": False, "frontend_required": False},
|
||||||
|
)
|
||||||
|
allowedModels: List[str] = Field(
|
||||||
|
default_factory=list,
|
||||||
|
description="Allowed AI models (empty = all permitted)",
|
||||||
|
json_schema_extra={"label": "Erlaubte Modelle", "frontend_type": "modelMultiSelect", "frontend_readonly": False, "frontend_required": False},
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -110,6 +110,7 @@ class WorkspaceInputRequest(BaseModel):
|
||||||
workflowId: Optional[str] = Field(default=None, description="Continue existing workflow")
|
workflowId: Optional[str] = Field(default=None, description="Continue existing workflow")
|
||||||
userLanguage: str = Field(default="en", description="User language code")
|
userLanguage: str = Field(default="en", description="User language code")
|
||||||
allowedProviders: List[str] = Field(default_factory=list, description="Restrict AI to these providers")
|
allowedProviders: List[str] = Field(default_factory=list, description="Restrict AI to these providers")
|
||||||
|
allowedModels: List[str] = Field(default_factory=list, description="Restrict AI to these models")
|
||||||
requireNeutralization: Optional[bool] = Field(default=None, description="Per-request neutralization override")
|
requireNeutralization: Optional[bool] = Field(default=None, description="Per-request neutralization override")
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -635,6 +636,7 @@ async def streamWorkspaceStart(
|
||||||
userLanguage=userInput.userLanguage,
|
userLanguage=userInput.userLanguage,
|
||||||
instanceConfig=instanceConfig,
|
instanceConfig=instanceConfig,
|
||||||
allowedProviders=userInput.allowedProviders,
|
allowedProviders=userInput.allowedProviders,
|
||||||
|
allowedModels=userInput.allowedModels,
|
||||||
requireNeutralization=userInput.requireNeutralization,
|
requireNeutralization=userInput.requireNeutralization,
|
||||||
billingFeatureCode=wsBillingFeatureCode,
|
billingFeatureCode=wsBillingFeatureCode,
|
||||||
)
|
)
|
||||||
|
|
@ -692,6 +694,7 @@ async def _runWorkspaceAgent(
|
||||||
userLanguage: str = "en",
|
userLanguage: str = "en",
|
||||||
instanceConfig: Dict[str, Any] = None,
|
instanceConfig: Dict[str, Any] = None,
|
||||||
allowedProviders: List[str] = None,
|
allowedProviders: List[str] = None,
|
||||||
|
allowedModels: List[str] = None,
|
||||||
requireNeutralization: Optional[bool] = None,
|
requireNeutralization: Optional[bool] = None,
|
||||||
billingFeatureCode: Optional[str] = None,
|
billingFeatureCode: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
|
@ -715,6 +718,9 @@ async def _runWorkspaceAgent(
|
||||||
logger.info(f"Workspace agent: allowedProviders={allowedProviders}")
|
logger.info(f"Workspace agent: allowedProviders={allowedProviders}")
|
||||||
else:
|
else:
|
||||||
logger.debug("Workspace agent: no allowedProviders in request")
|
logger.debug("Workspace agent: no allowedProviders in request")
|
||||||
|
if allowedModels:
|
||||||
|
aiService.services.allowedModels = allowedModels
|
||||||
|
logger.info(f"Workspace agent: allowedModels={allowedModels}")
|
||||||
if requireNeutralization is not None:
|
if requireNeutralization is not None:
|
||||||
ctx.requireNeutralization = requireNeutralization
|
ctx.requireNeutralization = requireNeutralization
|
||||||
|
|
||||||
|
|
@ -2114,6 +2120,76 @@ async def updateGeneralSettings(
|
||||||
return await getGeneralSettings(request, instanceId, context)
|
return await getGeneralSettings(request, instanceId, context)
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# User-level AI settings (neutralisation, providers, models)
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
@router.get("/{instanceId}/user-settings")
|
||||||
|
@limiter.limit("120/minute")
|
||||||
|
async def getWorkspaceUserSettings(
|
||||||
|
request: Request,
|
||||||
|
instanceId: str = Path(...),
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
):
|
||||||
|
"""Get the current user's workspace AI settings (auto-creates with defaults if not exists)."""
|
||||||
|
_mandateId, _ = _validateInstanceAccess(instanceId, context)
|
||||||
|
wsInterface = _getWorkspaceInterface(context, instanceId)
|
||||||
|
userId = str(context.user.id)
|
||||||
|
|
||||||
|
settings = wsInterface.getWorkspaceUserSettings(userId)
|
||||||
|
if settings:
|
||||||
|
return JSONResponse({
|
||||||
|
"requireNeutralization": settings.requireNeutralization,
|
||||||
|
"allowedProviders": settings.allowedProviders,
|
||||||
|
"allowedModels": settings.allowedModels,
|
||||||
|
})
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"userId": userId,
|
||||||
|
"mandateId": str(context.mandateId) if context.mandateId else "",
|
||||||
|
"featureInstanceId": instanceId,
|
||||||
|
}
|
||||||
|
created = wsInterface.saveWorkspaceUserSettings(data)
|
||||||
|
return JSONResponse({
|
||||||
|
"requireNeutralization": created.requireNeutralization,
|
||||||
|
"allowedProviders": created.allowedProviders,
|
||||||
|
"allowedModels": created.allowedModels,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{instanceId}/user-settings")
|
||||||
|
@limiter.limit("120/minute")
|
||||||
|
async def putWorkspaceUserSettings(
|
||||||
|
request: Request,
|
||||||
|
instanceId: str = Path(...),
|
||||||
|
body: dict = Body(...),
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
):
|
||||||
|
"""Save the current user's workspace AI settings."""
|
||||||
|
_mandateId, _ = _validateInstanceAccess(instanceId, context)
|
||||||
|
wsInterface = _getWorkspaceInterface(context, instanceId)
|
||||||
|
userId = str(context.user.id)
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"userId": userId,
|
||||||
|
"mandateId": str(context.mandateId) if context.mandateId else "",
|
||||||
|
"featureInstanceId": instanceId,
|
||||||
|
}
|
||||||
|
if "requireNeutralization" in body:
|
||||||
|
data["requireNeutralization"] = bool(body["requireNeutralization"])
|
||||||
|
if "allowedProviders" in body:
|
||||||
|
data["allowedProviders"] = body["allowedProviders"]
|
||||||
|
if "allowedModels" in body:
|
||||||
|
data["allowedModels"] = body["allowedModels"]
|
||||||
|
|
||||||
|
saved = wsInterface.saveWorkspaceUserSettings(data)
|
||||||
|
return JSONResponse({
|
||||||
|
"requireNeutralization": saved.requireNeutralization,
|
||||||
|
"allowedProviders": saved.allowedProviders,
|
||||||
|
"allowedModels": saved.allowedModels,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# RAG / Knowledge — anonymised instance statistics (presentation / KPIs)
|
# RAG / Knowledge — anonymised instance statistics (presentation / KPIs)
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
|
||||||
198
modules/interfaces/_legacyMigrationTelemetry.py
Normal file
198
modules/interfaces/_legacyMigrationTelemetry.py
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""Lightweight Bootstrap-Telemetrie fuer entfernte Migrationsroutinen.
|
||||||
|
|
||||||
|
Wenn eine idempotente Bootstrap-Migration (z.B. ``_migrateAndDropSysAdminRole``)
|
||||||
|
aus dem Boot-Pfad entfernt wird, koennte ein theoretischer Edge-Case (alte
|
||||||
|
DB-Restore, manueller INSERT) wieder Legacy-Daten ins System bringen. Damit das
|
||||||
|
nicht still bleibt, ruft ``initBootstrap`` nach Abschluss aller Init-Schritte
|
||||||
|
einmalig ``runLegacyDataChecks`` auf -- das logged WARN bei Restbestand.
|
||||||
|
|
||||||
|
Designprinzipien:
|
||||||
|
- KEINE Schreibzugriffe (rein lesend).
|
||||||
|
- Process-lokal gecached (``_cache``), damit identische Boots/Reloads den Check
|
||||||
|
nur einmal laufen lassen.
|
||||||
|
- Pro Check eine Recordset-Abfrage; Ausnahmen werden als WARN geloggt, nicht
|
||||||
|
re-raised, damit Telemetrie den Boot nie crasht.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
|
from modules.datamodels.datamodelRbac import Role
|
||||||
|
from modules.datamodels.datamodelUam import Mandate
|
||||||
|
from modules.shared.mandateNameUtils import isValidMandateName
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_alreadyRan: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def runLegacyDataChecks(db: DatabaseConnector) -> None:
|
||||||
|
"""Logged WARN, falls noch Legacy-Daten existieren, die durch entfernte
|
||||||
|
Migrationsroutinen behandelt wurden. Prozessweit nur einmal aktiv.
|
||||||
|
|
||||||
|
Aufruf: am Ende von ``initBootstrap``.
|
||||||
|
"""
|
||||||
|
global _alreadyRan
|
||||||
|
if _alreadyRan:
|
||||||
|
return
|
||||||
|
_alreadyRan = True
|
||||||
|
|
||||||
|
_checkMandateDescription(db)
|
||||||
|
_checkMandateSlugRules(db)
|
||||||
|
_checkLegacyRootMandate(db)
|
||||||
|
_checkSysadminRole(db)
|
||||||
|
_backfillTargetFeatureInstanceId()
|
||||||
|
|
||||||
|
|
||||||
|
def _safe(checkName: str, fn) -> Any:
|
||||||
|
try:
|
||||||
|
return fn()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Legacy-data telemetry check '%s' failed: %s: %s",
|
||||||
|
checkName, type(exc).__name__, exc,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _checkMandateDescription(db: DatabaseConnector) -> None:
|
||||||
|
def _do() -> None:
|
||||||
|
rows = db.getRecordset(Mandate)
|
||||||
|
bad = [
|
||||||
|
r.get("id") for r in rows
|
||||||
|
if r.get("description") and not r.get("label")
|
||||||
|
]
|
||||||
|
if bad:
|
||||||
|
logger.warning(
|
||||||
|
"Legacy-data check: %d Mandate row(s) still have description "
|
||||||
|
"but empty label (removed migration: _migrateMandateDescriptionToLabel). "
|
||||||
|
"Run scripts/script_db_audit_legacy_state.py for details. IDs: %s",
|
||||||
|
len(bad), bad[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
_safe("mandate-description", _do)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkMandateSlugRules(db: DatabaseConnector) -> None:
|
||||||
|
def _do() -> None:
|
||||||
|
rows = db.getRecordset(Mandate)
|
||||||
|
seen: set[str] = set()
|
||||||
|
bad: list[str] = []
|
||||||
|
for r in sorted(rows, key=lambda x: str(x.get("id", ""))):
|
||||||
|
mid = r.get("id")
|
||||||
|
if not mid:
|
||||||
|
continue
|
||||||
|
name = (r.get("name") or "").strip()
|
||||||
|
labelRaw = r.get("label")
|
||||||
|
labelEmpty = not (labelRaw or "").strip() if labelRaw is not None else True
|
||||||
|
invalid = not isValidMandateName(name)
|
||||||
|
collides = name in seen
|
||||||
|
if not invalid and not collides:
|
||||||
|
seen.add(name)
|
||||||
|
if labelEmpty or invalid or collides:
|
||||||
|
bad.append(str(mid))
|
||||||
|
if bad:
|
||||||
|
logger.warning(
|
||||||
|
"Legacy-data check: %d Mandate row(s) violate slug/label rules "
|
||||||
|
"(removed migration: _migrateMandateNameLabelSlugRules). "
|
||||||
|
"Run scripts/script_db_audit_legacy_state.py for details. IDs: %s",
|
||||||
|
len(bad), bad[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
_safe("mandate-slug-rules", _do)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkLegacyRootMandate(db: DatabaseConnector) -> None:
|
||||||
|
def _do() -> None:
|
||||||
|
legacy = db.getRecordset(Mandate, recordFilter={"name": "Root"})
|
||||||
|
rootRows = db.getRecordset(Mandate, recordFilter={"name": "root"})
|
||||||
|
legacyByFlag = [r for r in rootRows if not r.get("isSystem")]
|
||||||
|
all_ = list(legacy) + legacyByFlag
|
||||||
|
if all_:
|
||||||
|
logger.warning(
|
||||||
|
"Legacy-data check: %d Root-Mandate row(s) still in legacy form "
|
||||||
|
"(removed migration: initRootMandate-legacy-branch). IDs: %s",
|
||||||
|
len(all_), [r.get("id") for r in all_][:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
_safe("root-mandate-legacy", _do)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkSysadminRole(db: DatabaseConnector) -> None:
|
||||||
|
def _do() -> None:
|
||||||
|
rootMandates = db.getRecordset(
|
||||||
|
Mandate, recordFilter={"name": "root", "isSystem": True}
|
||||||
|
)
|
||||||
|
if not rootMandates:
|
||||||
|
return
|
||||||
|
rootId = str(rootMandates[0].get("id"))
|
||||||
|
rows = db.getRecordset(
|
||||||
|
Role,
|
||||||
|
recordFilter={
|
||||||
|
"roleLabel": "sysadmin",
|
||||||
|
"mandateId": rootId,
|
||||||
|
"featureInstanceId": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if rows:
|
||||||
|
logger.warning(
|
||||||
|
"Legacy-data check: %d 'sysadmin' role(s) still present in root mandate "
|
||||||
|
"(removed migration: _migrateAndDropSysAdminRole). "
|
||||||
|
"Authority is now User.isPlatformAdmin -- migrate manually. IDs: %s",
|
||||||
|
len(rows), [r.get("id") for r in rows],
|
||||||
|
)
|
||||||
|
|
||||||
|
_safe("sysadmin-role", _do)
|
||||||
|
|
||||||
|
|
||||||
|
def _backfillTargetFeatureInstanceId() -> None:
|
||||||
|
"""Idempotent backfill: set targetFeatureInstanceId = featureInstanceId
|
||||||
|
for all non-template AutoWorkflow rows where it is still NULL.
|
||||||
|
|
||||||
|
Connects to ``poweron_graphicaleditor`` independently.
|
||||||
|
"""
|
||||||
|
def _do() -> None:
|
||||||
|
from modules.shared.configuration import APP_CONFIG
|
||||||
|
from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoWorkflow
|
||||||
|
|
||||||
|
dbHost = APP_CONFIG.get("DB_HOST", "localhost")
|
||||||
|
dbUser = APP_CONFIG.get("DB_USER")
|
||||||
|
dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD")
|
||||||
|
dbPort = int(APP_CONFIG.get("DB_PORT", 5432))
|
||||||
|
geDb = DatabaseConnector(
|
||||||
|
dbHost=dbHost,
|
||||||
|
dbDatabase="poweron_graphicaleditor",
|
||||||
|
dbUser=dbUser,
|
||||||
|
dbPassword=dbPassword,
|
||||||
|
dbPort=dbPort,
|
||||||
|
userId=None,
|
||||||
|
)
|
||||||
|
if not geDb._ensureTableExists(AutoWorkflow):
|
||||||
|
return
|
||||||
|
|
||||||
|
rows = geDb.getRecordset(AutoWorkflow) or []
|
||||||
|
backfilled = 0
|
||||||
|
for r in rows:
|
||||||
|
if r.get("isTemplate"):
|
||||||
|
continue
|
||||||
|
if r.get("targetFeatureInstanceId"):
|
||||||
|
continue
|
||||||
|
srcId = r.get("featureInstanceId")
|
||||||
|
if not srcId:
|
||||||
|
continue
|
||||||
|
geDb.recordModify(AutoWorkflow, r["id"], {"targetFeatureInstanceId": srcId})
|
||||||
|
backfilled += 1
|
||||||
|
|
||||||
|
if backfilled:
|
||||||
|
logger.info(
|
||||||
|
"targetFeatureInstanceId backfill: set %d non-template AutoWorkflow row(s) "
|
||||||
|
"to their featureInstanceId",
|
||||||
|
backfilled,
|
||||||
|
)
|
||||||
|
|
||||||
|
_safe("backfill-targetFeatureInstanceId", _do)
|
||||||
|
|
@ -111,6 +111,19 @@ class AiObjects:
|
||||||
processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=1,
|
processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
allowedModels = getattr(options, 'allowedModels', None) if options else None
|
||||||
|
if allowedModels:
|
||||||
|
filteredModels = [m for m in availableModels if m.name in allowedModels]
|
||||||
|
if filteredModels:
|
||||||
|
availableModels = filteredModels
|
||||||
|
else:
|
||||||
|
errorMsg = f"No models match allowedModels {allowedModels} (providers={allowedProviders}) for operation {options.operationType}"
|
||||||
|
logger.error(errorMsg)
|
||||||
|
return AiCallResponse(
|
||||||
|
content=errorMsg, modelName="error", priceCHF=0.0,
|
||||||
|
processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=1,
|
||||||
|
)
|
||||||
|
|
||||||
failoverModelList = modelSelector.getFailoverModelList(prompt, context, options, availableModels)
|
failoverModelList = modelSelector.getFailoverModelList(prompt, context, options, availableModels)
|
||||||
|
|
||||||
if not failoverModelList:
|
if not failoverModelList:
|
||||||
|
|
@ -364,6 +377,19 @@ class AiObjects:
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
allowedModels = getattr(options, 'allowedModels', None) if options else None
|
||||||
|
if allowedModels:
|
||||||
|
filtered = [m for m in availableModels if m.name in allowedModels]
|
||||||
|
if filtered:
|
||||||
|
availableModels = filtered
|
||||||
|
else:
|
||||||
|
yield AiCallResponse(
|
||||||
|
content=f"No models match allowedModels {allowedModels} (providers={allowedProviders}) for operation {options.operationType}",
|
||||||
|
modelName="error", priceCHF=0.0, processingTime=0.0,
|
||||||
|
bytesSent=0, bytesReceived=0, errorCount=1,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
failoverModelList = modelSelector.getFailoverModelList(
|
failoverModelList = modelSelector.getFailoverModelList(
|
||||||
request.prompt, request.context or "", options, availableModels
|
request.prompt, request.context or "", options, availableModels
|
||||||
)
|
)
|
||||||
|
|
@ -516,6 +542,14 @@ class AiObjects:
|
||||||
else:
|
else:
|
||||||
logger.warning(f"No embedding models match allowedProviders {allowedProviders}")
|
logger.warning(f"No embedding models match allowedProviders {allowedProviders}")
|
||||||
|
|
||||||
|
allowedModels = getattr(options, 'allowedModels', None) if options else None
|
||||||
|
if allowedModels:
|
||||||
|
filtered = [m for m in availableModels if m.name in allowedModels]
|
||||||
|
if filtered:
|
||||||
|
availableModels = filtered
|
||||||
|
else:
|
||||||
|
logger.warning(f"No embedding models match allowedModels {allowedModels}")
|
||||||
|
|
||||||
failoverModelList = modelSelector.getFailoverModelList(
|
failoverModelList = modelSelector.getFailoverModelList(
|
||||||
combinedText, "", options, availableModels
|
combinedText, "", options, availableModels
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -56,14 +56,8 @@ def initBootstrap(db: DatabaseConnector) -> None:
|
||||||
|
|
||||||
logger.info("Starting system bootstrap")
|
logger.info("Starting system bootstrap")
|
||||||
|
|
||||||
# Initialize root mandate
|
|
||||||
mandateId = initRootMandate(db)
|
mandateId = initRootMandate(db)
|
||||||
|
|
||||||
# Migrate existing mandate records: description -> label
|
|
||||||
_migrateMandateDescriptionToLabel(db)
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
|
|
||||||
# Clean up duplicate roles and fix corrupted templates FIRST
|
|
||||||
_deduplicateRoles(db)
|
_deduplicateRoles(db)
|
||||||
|
|
||||||
# Initialize system role TEMPLATES (mandateId=None, isSystemRole=True)
|
# Initialize system role TEMPLATES (mandateId=None, isSystemRole=True)
|
||||||
|
|
@ -76,14 +70,6 @@ def initBootstrap(db: DatabaseConnector) -> None:
|
||||||
# This also serves as migration for existing mandates that don't have instance roles yet
|
# This also serves as migration for existing mandates that don't have instance roles yet
|
||||||
_ensureAllMandatesHaveSystemRoles(db)
|
_ensureAllMandatesHaveSystemRoles(db)
|
||||||
|
|
||||||
# Migration: eliminate the legacy ``sysadmin`` role in root mandate
|
|
||||||
# (replaced by ``User.isPlatformAdmin`` flag — see
|
|
||||||
# wiki/c-work/4-done/2026-04-sysadmin-authority-split.md).
|
|
||||||
# Idempotent: noop after first successful run.
|
|
||||||
if mandateId:
|
|
||||||
_migrateAndDropSysAdminRole(db, mandateId)
|
|
||||||
|
|
||||||
# Ensure UI rules for navigation items (admin/user/viewer roles)
|
|
||||||
_ensureUiContextRules(db)
|
_ensureUiContextRules(db)
|
||||||
|
|
||||||
# Initialize admin user
|
# Initialize admin user
|
||||||
|
|
@ -129,9 +115,22 @@ def initBootstrap(db: DatabaseConnector) -> None:
|
||||||
# Bootstrap system workflow templates for graphical editor
|
# Bootstrap system workflow templates for graphical editor
|
||||||
_bootstrapSystemTemplates(db)
|
_bootstrapSystemTemplates(db)
|
||||||
|
|
||||||
|
# Sync feature template workflows (update graph of existing instance workflows
|
||||||
|
# whose templateSourceId matches a current code-defined template)
|
||||||
|
_syncFeatureTemplateWorkflows()
|
||||||
|
|
||||||
# Ensure billing settings and accounts exist for all mandates
|
# Ensure billing settings and accounts exist for all mandates
|
||||||
_bootstrapBilling()
|
_bootstrapBilling()
|
||||||
|
|
||||||
|
# Telemetrie: warne falls Restbestaende der entfernten idempotenten
|
||||||
|
# Migrationen wieder auftauchen (Edge-Case: alter DB-Restore o.ae.).
|
||||||
|
# Schreibt nicht, scheitert nicht den Boot.
|
||||||
|
try:
|
||||||
|
from modules.interfaces._legacyMigrationTelemetry import runLegacyDataChecks
|
||||||
|
runLegacyDataChecks(db)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Legacy-data telemetry skipped: {e}")
|
||||||
|
|
||||||
|
|
||||||
def _bootstrapBilling() -> None:
|
def _bootstrapBilling() -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -195,6 +194,97 @@ def _bootstrapSystemTemplates(db: DatabaseConnector) -> None:
|
||||||
logger.warning(f"System workflow template bootstrap failed: {e}")
|
logger.warning(f"System workflow template bootstrap failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def _syncFeatureTemplateWorkflows() -> None:
|
||||||
|
"""Sync existing instance-scoped workflows with current code-defined templates.
|
||||||
|
|
||||||
|
For each feature that exposes getTemplateWorkflows(), find all AutoWorkflow
|
||||||
|
rows whose templateSourceId matches a template ID and update their graph
|
||||||
|
if the code-defined version has changed. Preserves instance-specific
|
||||||
|
fields (label, tags, targetFeatureInstanceId, invocations, active).
|
||||||
|
Idempotent, runs on every boot.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
|
||||||
|
try:
|
||||||
|
from modules.system.registry import loadFeatureMainModules
|
||||||
|
from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import AutoWorkflow
|
||||||
|
from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import graphicalEditorDatabase
|
||||||
|
|
||||||
|
mainModules = loadFeatureMainModules()
|
||||||
|
|
||||||
|
templatesBySourceId: dict = {}
|
||||||
|
for featureCode, mod in mainModules.items():
|
||||||
|
getTemplateWorkflows = getattr(mod, "getTemplateWorkflows", None)
|
||||||
|
if not getTemplateWorkflows:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
templates = getTemplateWorkflows() or []
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
for tpl in templates:
|
||||||
|
tplId = tpl.get("id")
|
||||||
|
if tplId:
|
||||||
|
templatesBySourceId[tplId] = tpl
|
||||||
|
|
||||||
|
if not templatesBySourceId:
|
||||||
|
logger.info("_syncFeatureTemplateWorkflows: no templates found, skipping")
|
||||||
|
return
|
||||||
|
logger.info(f"_syncFeatureTemplateWorkflows: found {len(templatesBySourceId)} template(s): {list(templatesBySourceId.keys())}")
|
||||||
|
|
||||||
|
greenfieldDb = DatabaseConnector(
|
||||||
|
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
|
||||||
|
dbDatabase=graphicalEditorDatabase,
|
||||||
|
dbUser=APP_CONFIG.get("DB_USER"),
|
||||||
|
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"),
|
||||||
|
)
|
||||||
|
|
||||||
|
updated = 0
|
||||||
|
for sourceId, tpl in templatesBySourceId.items():
|
||||||
|
instances = greenfieldDb.getRecordset(AutoWorkflow, recordFilter={
|
||||||
|
"templateSourceId": sourceId,
|
||||||
|
"isTemplate": False,
|
||||||
|
})
|
||||||
|
if not instances:
|
||||||
|
continue
|
||||||
|
|
||||||
|
canonicalGraph = tpl.get("graph", {})
|
||||||
|
|
||||||
|
for inst in instances:
|
||||||
|
instId = inst.get("id") if isinstance(inst, dict) else getattr(inst, "id", None)
|
||||||
|
targetInstanceId = (
|
||||||
|
inst.get("targetFeatureInstanceId") if isinstance(inst, dict)
|
||||||
|
else getattr(inst, "targetFeatureInstanceId", None)
|
||||||
|
) or ""
|
||||||
|
|
||||||
|
graphJson = json.dumps(canonicalGraph)
|
||||||
|
graphJson = graphJson.replace("{{featureInstanceId}}", targetInstanceId)
|
||||||
|
newGraph = json.loads(graphJson)
|
||||||
|
|
||||||
|
existingGraph = inst.get("graph") if isinstance(inst, dict) else getattr(inst, "graph", None)
|
||||||
|
if isinstance(existingGraph, str):
|
||||||
|
try:
|
||||||
|
existingGraph = json.loads(existingGraph)
|
||||||
|
except Exception:
|
||||||
|
existingGraph = None
|
||||||
|
|
||||||
|
if existingGraph == newGraph:
|
||||||
|
logger.debug(f"_syncFeatureTemplateWorkflows: graph unchanged for workflow {instId} (template={sourceId})")
|
||||||
|
continue
|
||||||
|
logger.debug(f"_syncFeatureTemplateWorkflows: graph DIFFERS for workflow {instId} (template={sourceId}), updating")
|
||||||
|
|
||||||
|
greenfieldDb.recordModify(AutoWorkflow, instId, {"graph": newGraph})
|
||||||
|
updated += 1
|
||||||
|
logger.info(f"_syncFeatureTemplateWorkflows: updated graph for workflow {instId} (template={sourceId})")
|
||||||
|
|
||||||
|
if updated:
|
||||||
|
logger.info(f"_syncFeatureTemplateWorkflows: synced {updated} workflow(s) with current templates")
|
||||||
|
else:
|
||||||
|
logger.info("_syncFeatureTemplateWorkflows: all instance graphs already match current templates")
|
||||||
|
greenfieldDb.close()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Feature template workflow sync failed: {e}")
|
||||||
|
|
||||||
|
|
||||||
def _buildSystemTemplates():
|
def _buildSystemTemplates():
|
||||||
"""Build the graph definitions for platform system templates."""
|
"""Build the graph definitions for platform system templates."""
|
||||||
return [
|
return [
|
||||||
|
|
@ -396,21 +486,12 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]:
|
||||||
Returns:
|
Returns:
|
||||||
Mandate ID if created or found, None otherwise
|
Mandate ID if created or found, None otherwise
|
||||||
"""
|
"""
|
||||||
# Find existing root mandate by name AND isSystem flag
|
|
||||||
existingMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
|
existingMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
|
||||||
if existingMandates:
|
if existingMandates:
|
||||||
mandateId = existingMandates[0].get("id")
|
mandateId = existingMandates[0].get("id")
|
||||||
logger.info(f"Root mandate already exists with ID {mandateId}")
|
logger.info(f"Root mandate already exists with ID {mandateId}")
|
||||||
return mandateId
|
return mandateId
|
||||||
|
|
||||||
# Check for legacy root mandates (name="Root" without isSystem flag) and migrate
|
|
||||||
legacyMandates = db.getRecordset(Mandate, recordFilter={"name": "Root"})
|
|
||||||
if legacyMandates:
|
|
||||||
mandateId = legacyMandates[0].get("id")
|
|
||||||
logger.info(f"Migrating legacy Root mandate {mandateId}: setting name='root', isSystem=True")
|
|
||||||
db.recordModify(Mandate, mandateId, {"name": "root", "isSystem": True})
|
|
||||||
return mandateId
|
|
||||||
|
|
||||||
logger.info("Creating Root mandate")
|
logger.info("Creating Root mandate")
|
||||||
rootMandate = Mandate(name="root", label="Root", isSystem=True, enabled=True)
|
rootMandate = Mandate(name="root", label="Root", isSystem=True, enabled=True)
|
||||||
createdMandate = db.recordCreate(Mandate, rootMandate)
|
createdMandate = db.recordCreate(Mandate, rootMandate)
|
||||||
|
|
@ -419,98 +500,6 @@ def initRootMandate(db: DatabaseConnector) -> Optional[str]:
|
||||||
return mandateId
|
return mandateId
|
||||||
|
|
||||||
|
|
||||||
def _migrateMandateDescriptionToLabel(db: DatabaseConnector) -> None:
|
|
||||||
"""
|
|
||||||
Migration: Rename 'description' field to 'label' in all Mandate records.
|
|
||||||
Copies existing 'description' values to 'label' and removes the old field.
|
|
||||||
Safe to run multiple times (idempotent).
|
|
||||||
"""
|
|
||||||
allMandates = db.getRecordset(Mandate)
|
|
||||||
migratedCount = 0
|
|
||||||
for mandateRecord in allMandates:
|
|
||||||
mandateId = mandateRecord.get("id")
|
|
||||||
hasDescription = "description" in mandateRecord and mandateRecord.get("description") is not None
|
|
||||||
hasLabel = "label" in mandateRecord and mandateRecord.get("label") is not None
|
|
||||||
|
|
||||||
if hasDescription and not hasLabel:
|
|
||||||
# Copy description to label
|
|
||||||
updateData = {"label": mandateRecord["description"]}
|
|
||||||
db.recordModify(Mandate, mandateId, updateData)
|
|
||||||
migratedCount += 1
|
|
||||||
logger.info(f"Migrated mandate {mandateId}: description -> label")
|
|
||||||
|
|
||||||
if migratedCount > 0:
|
|
||||||
logger.info(f"Migrated {migratedCount} mandate(s) from description to label")
|
|
||||||
else:
|
|
||||||
logger.debug("No mandate description->label migration needed")
|
|
||||||
|
|
||||||
|
|
||||||
def _migrateMandateNameLabelSlugRules(db: DatabaseConnector) -> None:
|
|
||||||
"""
|
|
||||||
Migration: normalize Mandate.name to the slug rules ([a-z0-9-], length 2..32, single
|
|
||||||
hyphen segments) and ensure Mandate.label is non-empty.
|
|
||||||
|
|
||||||
Rules (see wiki/c-work/1-plan/2026-04-mandate-name-label-logic.md):
|
|
||||||
1. If ``label`` is empty/None → set ``label := name`` (or "Mandate" when both empty).
|
|
||||||
2. If ``name`` is not a valid slug, or collides with an earlier mandate in stable id
|
|
||||||
order, allocate a unique slug from the (now non-empty) ``label`` using
|
|
||||||
``slugifyMandateName`` + ``allocateUniqueMandateSlug``.
|
|
||||||
|
|
||||||
Idempotent: a second run is a no-op because all valid names stay valid and stay unique.
|
|
||||||
Each rename and label fill-in is logged for audit.
|
|
||||||
"""
|
|
||||||
from modules.shared.mandateNameUtils import (
|
|
||||||
allocateUniqueMandateSlug,
|
|
||||||
isValidMandateName,
|
|
||||||
slugifyMandateName,
|
|
||||||
)
|
|
||||||
|
|
||||||
allRows = db.getRecordset(Mandate)
|
|
||||||
if not allRows:
|
|
||||||
return
|
|
||||||
sortedRows = sorted(allRows, key=lambda r: str(r.get("id", "")))
|
|
||||||
|
|
||||||
used: set[str] = set()
|
|
||||||
labelFills = 0
|
|
||||||
nameRenames: list[tuple[str, str, str]] = []
|
|
||||||
|
|
||||||
for rec in sortedRows:
|
|
||||||
mid = rec.get("id")
|
|
||||||
if not mid:
|
|
||||||
continue
|
|
||||||
name = (rec.get("name") or "").strip()
|
|
||||||
labelRaw = rec.get("label")
|
|
||||||
label = (labelRaw or "").strip() if labelRaw is not None else ""
|
|
||||||
|
|
||||||
if not label:
|
|
||||||
label = name if name else "Mandate"
|
|
||||||
db.recordModify(Mandate, mid, {"label": label})
|
|
||||||
labelFills += 1
|
|
||||||
logger.info(f"Mandate {mid}: filled empty label with '{label}'")
|
|
||||||
|
|
||||||
nameFits = isValidMandateName(name)
|
|
||||||
nameCollides = name in used
|
|
||||||
if nameFits and not nameCollides:
|
|
||||||
used.add(name)
|
|
||||||
continue
|
|
||||||
|
|
||||||
base = slugifyMandateName(label) or "mn"
|
|
||||||
newName = allocateUniqueMandateSlug(base, used)
|
|
||||||
used.add(newName)
|
|
||||||
if newName != name:
|
|
||||||
db.recordModify(Mandate, mid, {"name": newName})
|
|
||||||
nameRenames.append((str(mid), name, newName))
|
|
||||||
logger.info(f"Mandate {mid}: renamed name '{name}' -> '{newName}'")
|
|
||||||
|
|
||||||
if labelFills or nameRenames:
|
|
||||||
logger.info(
|
|
||||||
"Mandate name/label slug migration: %d label fill-in(s), %d name rename(s)",
|
|
||||||
labelFills, len(nameRenames),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.debug("No mandate name/label slug migration needed")
|
|
||||||
|
|
||||||
|
|
||||||
def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[str]:
|
def initAdminUser(db: DatabaseConnector, mandateId: Optional[str]) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Creates the Admin user if it doesn't exist.
|
Creates the Admin user if it doesn't exist.
|
||||||
|
|
@ -837,101 +826,6 @@ def copySystemRolesToMandate(db: DatabaseConnector, mandateId: str) -> int:
|
||||||
return copiedCount
|
return copiedCount
|
||||||
|
|
||||||
|
|
||||||
def _migrateAndDropSysAdminRole(db: DatabaseConnector, mandateId: str) -> None:
|
|
||||||
"""
|
|
||||||
One-shot migration: eliminate the legacy ``sysadmin`` role in the root mandate.
|
|
||||||
|
|
||||||
Authority semantics moved to two orthogonal flags on User:
|
|
||||||
- ``isSysAdmin`` → Infrastructure-Operator (RBAC bypass)
|
|
||||||
- ``isPlatformAdmin`` → Cross-Mandate-Governance (no bypass)
|
|
||||||
|
|
||||||
Migration steps (idempotent):
|
|
||||||
1. Find sysadmin role(s) in root mandate. If none exist → done.
|
|
||||||
2. For every UserMandateRole row referencing such a role: set
|
|
||||||
``user.isPlatformAdmin = True`` (preserves cross-mandate authority).
|
|
||||||
3. Delete those UserMandateRole rows.
|
|
||||||
4. Delete AccessRules attached to the sysadmin role.
|
|
||||||
5. Delete the sysadmin Role record.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
db: Database connector instance
|
|
||||||
mandateId: Root mandate ID
|
|
||||||
"""
|
|
||||||
sysadminRoles = db.getRecordset(
|
|
||||||
Role,
|
|
||||||
recordFilter={"roleLabel": "sysadmin", "mandateId": mandateId, "featureInstanceId": None},
|
|
||||||
)
|
|
||||||
if not sysadminRoles:
|
|
||||||
logger.debug("Sysadmin role migration: no legacy sysadmin role present, nothing to do")
|
|
||||||
return
|
|
||||||
|
|
||||||
sysadminRoleIds = [str(r.get("id")) for r in sysadminRoles if r.get("id")]
|
|
||||||
logger.warning(
|
|
||||||
f"Sysadmin role migration: found {len(sysadminRoleIds)} legacy sysadmin role(s) "
|
|
||||||
f"in root mandate, migrating to isPlatformAdmin flag"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 1) Promote every holder to isPlatformAdmin=True
|
|
||||||
promoted = 0
|
|
||||||
for sysadminRoleId in sysadminRoleIds:
|
|
||||||
umRoleRows = db.getRecordset(
|
|
||||||
UserMandateRole, recordFilter={"roleId": sysadminRoleId}
|
|
||||||
)
|
|
||||||
userMandateIds = [str(r.get("userMandateId")) for r in umRoleRows if r.get("userMandateId")]
|
|
||||||
if not userMandateIds:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Resolve userIds via UserMandate
|
|
||||||
userIds = set()
|
|
||||||
for umId in userMandateIds:
|
|
||||||
ums = db.getRecordset(UserMandate, recordFilter={"id": umId})
|
|
||||||
for um in ums:
|
|
||||||
uid = um.get("userId") if isinstance(um, dict) else getattr(um, "userId", None)
|
|
||||||
if uid:
|
|
||||||
userIds.add(str(uid))
|
|
||||||
|
|
||||||
for userId in userIds:
|
|
||||||
users = db.getRecordset(UserInDB, recordFilter={"id": userId})
|
|
||||||
if not users:
|
|
||||||
continue
|
|
||||||
current = users[0].get("isPlatformAdmin", False)
|
|
||||||
if not current:
|
|
||||||
db.recordModify(UserInDB, userId, {"isPlatformAdmin": True})
|
|
||||||
promoted += 1
|
|
||||||
logger.warning(
|
|
||||||
f"Sysadmin role migration: granted isPlatformAdmin=True to user {userId}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 2) Delete UserMandateRole rows
|
|
||||||
for umRow in umRoleRows:
|
|
||||||
rowId = umRow.get("id") if isinstance(umRow, dict) else getattr(umRow, "id", None)
|
|
||||||
if rowId:
|
|
||||||
try:
|
|
||||||
db.recordDelete(UserMandateRole, str(rowId))
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Sysadmin role migration: failed to drop UserMandateRole {rowId}: {e}")
|
|
||||||
|
|
||||||
# 3) Delete AccessRules
|
|
||||||
accessRules = db.getRecordset(AccessRule, recordFilter={"roleId": sysadminRoleId})
|
|
||||||
for ar in accessRules:
|
|
||||||
arId = ar.get("id") if isinstance(ar, dict) else getattr(ar, "id", None)
|
|
||||||
if arId:
|
|
||||||
try:
|
|
||||||
db.recordDelete(AccessRule, str(arId))
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Sysadmin role migration: failed to drop AccessRule {arId}: {e}")
|
|
||||||
|
|
||||||
# 4) Delete the Role
|
|
||||||
try:
|
|
||||||
db.recordDelete(Role, sysadminRoleId)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Sysadmin role migration: failed to drop Role {sysadminRoleId}: {e}")
|
|
||||||
|
|
||||||
logger.warning(
|
|
||||||
f"Sysadmin role migration: completed; promoted {promoted} user(s) to isPlatformAdmin"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]:
|
def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Get role ID by label, using cache or database lookup.
|
Get role ID by label, using cache or database lookup.
|
||||||
|
|
|
||||||
|
|
@ -643,41 +643,10 @@ def aggregateMandateRagTotalBytes(mandateId: str) -> int:
|
||||||
if rid and str(rid) not in byId:
|
if rid and str(rid) not in byId:
|
||||||
byId[str(rid)] = row
|
byId[str(rid)] = row
|
||||||
|
|
||||||
# DEPRECATED: file-ID-correlation fallback from poweron_management.
|
|
||||||
# Only needed for pre-migration data where mandateId/featureInstanceId on the
|
|
||||||
# FileContentIndex are empty. Safe to remove once all environments are migrated.
|
|
||||||
_fallbackCount = 0
|
|
||||||
try:
|
|
||||||
from modules.datamodels.datamodelFiles import FileItem
|
|
||||||
from modules.interfaces.interfaceDbManagement import ComponentObjects
|
|
||||||
mgmtDb = ComponentObjects().db
|
|
||||||
knowledgeIf = getInterface(None)
|
|
||||||
|
|
||||||
fileIds: set = set()
|
|
||||||
for f in mgmtDb.getRecordset(FileItem, recordFilter={"mandateId": mandateId}):
|
|
||||||
fid = f.get("id") if isinstance(f, dict) else getattr(f, "id", None)
|
|
||||||
if fid:
|
|
||||||
fileIds.add(str(fid))
|
|
||||||
for instId in instIds:
|
|
||||||
for f in mgmtDb.getRecordset(FileItem, recordFilter={"featureInstanceId": instId}):
|
|
||||||
fid = f.get("id") if isinstance(f, dict) else getattr(f, "id", None)
|
|
||||||
if fid:
|
|
||||||
fileIds.add(str(fid))
|
|
||||||
|
|
||||||
for fid in fileIds:
|
|
||||||
if fid in byId:
|
|
||||||
continue
|
|
||||||
row = knowledgeIf.getFileContentIndex(fid)
|
|
||||||
if row:
|
|
||||||
byId[fid] = row
|
|
||||||
_fallbackCount += 1
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("aggregateMandateRagTotalBytes fallback failed: %s", e)
|
|
||||||
|
|
||||||
total = sum(int(r.get("totalSize") or 0) for r in byId.values())
|
total = sum(int(r.get("totalSize") or 0) for r in byId.values())
|
||||||
logger.info(
|
logger.info(
|
||||||
"aggregateMandateRagTotalBytes(%s): %d indexes, %d bytes (fallback: %d)",
|
"aggregateMandateRagTotalBytes(%s): %d indexes, %d bytes",
|
||||||
mandateId, len(byId), total, _fallbackCount,
|
mandateId, len(byId), total,
|
||||||
)
|
)
|
||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -347,6 +347,7 @@ class FeatureInterface:
|
||||||
"templateSourceId": templateId,
|
"templateSourceId": templateId,
|
||||||
"templateScope": "instance",
|
"templateScope": "instance",
|
||||||
"active": True,
|
"active": True,
|
||||||
|
"targetFeatureInstanceId": instanceId,
|
||||||
})
|
})
|
||||||
copied += 1
|
copied += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
305
modules/routes/routeAutomationWorkspace.py
Normal file
305
modules/routes/routeAutomationWorkspace.py
Normal file
|
|
@ -0,0 +1,305 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""
|
||||||
|
User-facing Automation Workspace API.
|
||||||
|
|
||||||
|
Lists workflow runs the user can access (via FeatureAccess on
|
||||||
|
targetFeatureInstanceId) and provides detail views with step logs
|
||||||
|
and linked files. Designed for the "Workspace" tab under
|
||||||
|
Nutzung > Automation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import math
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Request, Query, Path, HTTPException
|
||||||
|
from slowapi import Limiter
|
||||||
|
from slowapi.util import get_remote_address
|
||||||
|
|
||||||
|
from modules.auth.authentication import getRequestContext, RequestContext
|
||||||
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
|
from modules.shared.configuration import APP_CONFIG
|
||||||
|
from modules.features.graphicalEditor.datamodelFeatureGraphicalEditor import (
|
||||||
|
AutoRun,
|
||||||
|
AutoStepLog,
|
||||||
|
AutoWorkflow,
|
||||||
|
)
|
||||||
|
from modules.features.graphicalEditor.interfaceFeatureGraphicalEditor import graphicalEditorDatabase
|
||||||
|
from modules.shared.i18nRegistry import apiRouteContext
|
||||||
|
|
||||||
|
routeApiMsg = apiRouteContext("routeAutomationWorkspace")
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
limiter = Limiter(key_func=get_remote_address)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/automations/runs", tags=["AutomationWorkspace"])
|
||||||
|
|
||||||
|
|
||||||
|
def _getDb() -> DatabaseConnector:
|
||||||
|
return DatabaseConnector(
|
||||||
|
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
|
||||||
|
dbDatabase=graphicalEditorDatabase,
|
||||||
|
dbUser=APP_CONFIG.get("DB_USER"),
|
||||||
|
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET") or APP_CONFIG.get("DB_PASSWORD"),
|
||||||
|
dbPort=int(APP_CONFIG.get("DB_PORT", 5432)),
|
||||||
|
userId=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _getUserAccessibleInstanceIds(userId: str) -> list[str]:
|
||||||
|
"""Return all featureInstanceIds the user has enabled FeatureAccess for."""
|
||||||
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
|
rootIface = getRootInterface()
|
||||||
|
allAccess = rootIface.getFeatureAccessesForUser(userId) or []
|
||||||
|
return [
|
||||||
|
a.featureInstanceId
|
||||||
|
for a in allAccess
|
||||||
|
if a.featureInstanceId and a.enabled
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
_FILE_REF_KEYS = ("fileId", "documentId", "fileIds", "documents")
|
||||||
|
|
||||||
|
|
||||||
|
def _extractFileIdsFromValue(value, accumulator: set[str]) -> None:
|
||||||
|
"""Recursively scan a value (dict/list/str) for file id references."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
for key, sub in value.items():
|
||||||
|
if key in _FILE_REF_KEYS:
|
||||||
|
_collectFileIdsFromRef(sub, accumulator)
|
||||||
|
else:
|
||||||
|
_extractFileIdsFromValue(sub, accumulator)
|
||||||
|
elif isinstance(value, list):
|
||||||
|
for item in value:
|
||||||
|
_extractFileIdsFromValue(item, accumulator)
|
||||||
|
|
||||||
|
|
||||||
|
def _collectFileIdsFromRef(val, accumulator: set[str]) -> None:
|
||||||
|
"""Add file ids from a value located under a known file-reference key."""
|
||||||
|
if isinstance(val, str) and val:
|
||||||
|
accumulator.add(val)
|
||||||
|
elif isinstance(val, list):
|
||||||
|
for v in val:
|
||||||
|
if isinstance(v, str) and v:
|
||||||
|
accumulator.add(v)
|
||||||
|
elif isinstance(v, dict) and v.get("id"):
|
||||||
|
accumulator.add(v["id"])
|
||||||
|
elif isinstance(val, dict) and val.get("id"):
|
||||||
|
accumulator.add(val["id"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
def listWorkspaceRuns(
|
||||||
|
request: Request,
|
||||||
|
scope: str = Query("mine", description="mine = own runs, mandate = all accessible"),
|
||||||
|
status: Optional[str] = Query(None, description="Filter by run status"),
|
||||||
|
targetInstanceId: Optional[str] = Query(None, description="Filter by targetFeatureInstanceId"),
|
||||||
|
workflowId: Optional[str] = Query(None, description="Filter by workflow"),
|
||||||
|
limit: int = Query(50, ge=1, le=200),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
) -> dict:
|
||||||
|
"""List workflow runs visible to the user.
|
||||||
|
|
||||||
|
scope=mine: only runs owned by the user.
|
||||||
|
scope=mandate: all runs where the user has FeatureAccess on the
|
||||||
|
workflow's targetFeatureInstanceId.
|
||||||
|
"""
|
||||||
|
db = _getDb()
|
||||||
|
if not db._ensureTableExists(AutoRun):
|
||||||
|
return {"runs": [], "total": 0, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
userId = str(context.user.id) if context.user else None
|
||||||
|
if not userId:
|
||||||
|
raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required"))
|
||||||
|
|
||||||
|
accessibleInstanceIds = _getUserAccessibleInstanceIds(userId)
|
||||||
|
if not accessibleInstanceIds:
|
||||||
|
return {"runs": [], "total": 0, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
if not db._ensureTableExists(AutoWorkflow):
|
||||||
|
return {"runs": [], "total": 0, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
wfFilter: dict = {}
|
||||||
|
if targetInstanceId:
|
||||||
|
if targetInstanceId not in accessibleInstanceIds:
|
||||||
|
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied to target instance"))
|
||||||
|
wfFilter["targetFeatureInstanceId"] = targetInstanceId
|
||||||
|
workflows = db.getRecordset(AutoWorkflow, recordFilter=wfFilter or None) or []
|
||||||
|
|
||||||
|
visibleWfIds: set[str] = set()
|
||||||
|
wfMap: dict = {}
|
||||||
|
for wf in workflows:
|
||||||
|
wfDict = dict(wf)
|
||||||
|
tid = wfDict.get("targetFeatureInstanceId") or wfDict.get("featureInstanceId")
|
||||||
|
if tid and tid in accessibleInstanceIds:
|
||||||
|
wfId = wfDict.get("id")
|
||||||
|
if wfId:
|
||||||
|
visibleWfIds.add(wfId)
|
||||||
|
wfMap[wfId] = wfDict
|
||||||
|
|
||||||
|
if workflowId:
|
||||||
|
if workflowId not in visibleWfIds:
|
||||||
|
return {"runs": [], "total": 0, "limit": limit, "offset": offset}
|
||||||
|
visibleWfIds = {workflowId}
|
||||||
|
|
||||||
|
if not visibleWfIds:
|
||||||
|
return {"runs": [], "total": 0, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
allRuns = db.getRecordset(AutoRun, recordFilter={}) or []
|
||||||
|
filtered = []
|
||||||
|
for r in allRuns:
|
||||||
|
row = dict(r)
|
||||||
|
if row.get("workflowId") not in visibleWfIds:
|
||||||
|
continue
|
||||||
|
if scope == "mine" and row.get("ownerId") != userId:
|
||||||
|
continue
|
||||||
|
if status and row.get("status") != status:
|
||||||
|
continue
|
||||||
|
filtered.append(row)
|
||||||
|
|
||||||
|
filtered.sort(
|
||||||
|
key=lambda x: x.get("startedAt") or x.get("sysCreatedAt") or 0,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
total = len(filtered)
|
||||||
|
page = filtered[offset: offset + limit]
|
||||||
|
|
||||||
|
from modules.routes.routeHelpers import enrichRowsWithFkLabels, resolveMandateLabels, resolveInstanceLabels
|
||||||
|
|
||||||
|
for row in page:
|
||||||
|
wf = wfMap.get(row.get("workflowId"), {})
|
||||||
|
row["workflowLabel"] = row.get("label") or wf.get("label") or row.get("workflowId", "")
|
||||||
|
row["targetFeatureInstanceId"] = wf.get("targetFeatureInstanceId") or wf.get("featureInstanceId")
|
||||||
|
|
||||||
|
enrichRowsWithFkLabels(
|
||||||
|
page,
|
||||||
|
labelResolvers={
|
||||||
|
"mandateId": resolveMandateLabels,
|
||||||
|
"targetFeatureInstanceId": resolveInstanceLabels,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for row in page:
|
||||||
|
row["targetInstanceLabel"] = row.pop("targetFeatureInstanceIdLabel", None)
|
||||||
|
row["mandateLabel"] = row.pop("mandateIdLabel", None)
|
||||||
|
|
||||||
|
return {"runs": page, "total": total, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{runId}/detail")
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
def getWorkspaceRunDetail(
|
||||||
|
request: Request,
|
||||||
|
runId: str = Path(..., description="Run ID"),
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
) -> dict:
|
||||||
|
"""Get full detail for a single run: metadata, step logs, linked files."""
|
||||||
|
db = _getDb()
|
||||||
|
userId = str(context.user.id) if context.user else None
|
||||||
|
if not userId:
|
||||||
|
raise HTTPException(status_code=401, detail=routeApiMsg("Authentication required"))
|
||||||
|
|
||||||
|
if not db._ensureTableExists(AutoRun):
|
||||||
|
raise HTTPException(status_code=404, detail=routeApiMsg("Run not found"))
|
||||||
|
|
||||||
|
runs = db.getRecordset(AutoRun, recordFilter={"id": runId})
|
||||||
|
if not runs:
|
||||||
|
raise HTTPException(status_code=404, detail=routeApiMsg("Run not found"))
|
||||||
|
run = dict(runs[0])
|
||||||
|
|
||||||
|
wfId = run.get("workflowId")
|
||||||
|
workflow: dict = {}
|
||||||
|
if wfId and db._ensureTableExists(AutoWorkflow):
|
||||||
|
wfs = db.getRecordset(AutoWorkflow, recordFilter={"id": wfId})
|
||||||
|
if wfs:
|
||||||
|
workflow = dict(wfs[0])
|
||||||
|
|
||||||
|
tid = workflow.get("targetFeatureInstanceId") or workflow.get("featureInstanceId")
|
||||||
|
accessibleIds = _getUserAccessibleInstanceIds(userId)
|
||||||
|
isOwner = run.get("ownerId") == userId
|
||||||
|
|
||||||
|
if not isOwner and (not tid or tid not in accessibleIds) and not context.isPlatformAdmin:
|
||||||
|
raise HTTPException(status_code=403, detail=routeApiMsg("Access denied"))
|
||||||
|
|
||||||
|
steps: list = []
|
||||||
|
if db._ensureTableExists(AutoStepLog):
|
||||||
|
stepRecords = db.getRecordset(AutoStepLog, recordFilter={"runId": runId}) or []
|
||||||
|
steps = [dict(s) for s in stepRecords]
|
||||||
|
steps.sort(key=lambda s: s.get("startedAt") or 0)
|
||||||
|
|
||||||
|
allFileIds: set[str] = set()
|
||||||
|
perStepFileIds: list[tuple[set[str], set[str]]] = []
|
||||||
|
for step in steps:
|
||||||
|
inputIds: set[str] = set()
|
||||||
|
outputIds: set[str] = set()
|
||||||
|
_extractFileIdsFromValue(step.get("inputSnapshot") or {}, inputIds)
|
||||||
|
_extractFileIdsFromValue(step.get("output") or {}, outputIds)
|
||||||
|
perStepFileIds.append((inputIds, outputIds))
|
||||||
|
allFileIds.update(inputIds)
|
||||||
|
allFileIds.update(outputIds)
|
||||||
|
|
||||||
|
nodeOutputs = run.get("nodeOutputs") or {}
|
||||||
|
runLevelIds: set[str] = set()
|
||||||
|
_extractFileIdsFromValue(nodeOutputs, runLevelIds)
|
||||||
|
allFileIds.update(runLevelIds)
|
||||||
|
|
||||||
|
fileMetaById: dict[str, dict] = {}
|
||||||
|
try:
|
||||||
|
from modules.datamodels.datamodelFiles import FileItem
|
||||||
|
from modules.interfaces.interfaceDbManagement import ComponentObjects
|
||||||
|
mgmtDb = ComponentObjects().db
|
||||||
|
if mgmtDb._ensureTableExists(FileItem):
|
||||||
|
for fid in allFileIds:
|
||||||
|
try:
|
||||||
|
rec = mgmtDb.getRecord(FileItem, fid)
|
||||||
|
if rec:
|
||||||
|
recDict = dict(rec)
|
||||||
|
fileMetaById[fid] = {
|
||||||
|
"id": fid,
|
||||||
|
"fileName": recDict.get("fileName") or recDict.get("name"),
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("getWorkspaceRunDetail: file lookup failed: %s", e)
|
||||||
|
|
||||||
|
def _resolveFileList(ids: set[str]) -> list[dict]:
|
||||||
|
return [fileMetaById[fid] for fid in ids if fid in fileMetaById]
|
||||||
|
|
||||||
|
assignedFileIds: set[str] = set()
|
||||||
|
for step, (inputIds, outputIds) in zip(steps, perStepFileIds):
|
||||||
|
step["inputFiles"] = _resolveFileList(inputIds)
|
||||||
|
step["outputFiles"] = _resolveFileList(outputIds)
|
||||||
|
assignedFileIds.update(inputIds)
|
||||||
|
assignedFileIds.update(outputIds)
|
||||||
|
|
||||||
|
unassignedFiles = _resolveFileList(allFileIds - assignedFileIds)
|
||||||
|
allFiles = _resolveFileList(allFileIds)
|
||||||
|
|
||||||
|
run["workflowLabel"] = run.get("label") or workflow.get("label") or wfId
|
||||||
|
run["targetFeatureInstanceId"] = tid
|
||||||
|
|
||||||
|
targetInstanceLabel = None
|
||||||
|
if tid:
|
||||||
|
try:
|
||||||
|
from modules.routes.routeHelpers import resolveInstanceLabels
|
||||||
|
labelMap = resolveInstanceLabels([tid])
|
||||||
|
targetInstanceLabel = labelMap.get(tid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
run["targetInstanceLabel"] = targetInstanceLabel
|
||||||
|
|
||||||
|
return {
|
||||||
|
"run": run,
|
||||||
|
"workflow": {
|
||||||
|
"id": workflow.get("id"),
|
||||||
|
"label": workflow.get("label"),
|
||||||
|
"targetFeatureInstanceId": tid,
|
||||||
|
"featureInstanceId": workflow.get("featureInstanceId"),
|
||||||
|
"tags": workflow.get("tags", []),
|
||||||
|
} if workflow else None,
|
||||||
|
"steps": steps,
|
||||||
|
"files": allFiles,
|
||||||
|
"unassignedFiles": unassignedFiles,
|
||||||
|
}
|
||||||
|
|
@ -25,142 +25,11 @@ def _registerMediaTools(registry: ToolRegistry, services):
|
||||||
# ---- Document rendering tool ----
|
# ---- Document rendering tool ----
|
||||||
|
|
||||||
def _markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]:
|
def _markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]:
|
||||||
"""Convert markdown content to the standard document JSON format expected by renderers."""
|
"""Delegate to the consolidated parser in subDocumentUtility."""
|
||||||
import re as _re
|
from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson
|
||||||
|
result = markdownToDocumentJson(markdown, title, language)
|
||||||
sections = []
|
result["metadata"]["extraction_method"] = "agent_rendering"
|
||||||
order = 0
|
return result
|
||||||
lines = markdown.split("\n")
|
|
||||||
i = 0
|
|
||||||
|
|
||||||
def _nextId():
|
|
||||||
nonlocal order
|
|
||||||
order += 1
|
|
||||||
return f"s_{order}"
|
|
||||||
|
|
||||||
while i < len(lines):
|
|
||||||
line = lines[i]
|
|
||||||
|
|
||||||
# --- Headings ---
|
|
||||||
headingMatch = _re.match(r'^(#{1,6})\s+(.+)', line)
|
|
||||||
if headingMatch:
|
|
||||||
level = len(headingMatch.group(1))
|
|
||||||
text = headingMatch.group(2).strip()
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "heading", "order": order,
|
|
||||||
"elements": [{"content": {"text": text, "level": level}}],
|
|
||||||
})
|
|
||||||
i += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Fenced code blocks ---
|
|
||||||
codeMatch = _re.match(r'^```(\w*)', line)
|
|
||||||
if codeMatch:
|
|
||||||
lang = codeMatch.group(1) or "text"
|
|
||||||
codeLines = []
|
|
||||||
i += 1
|
|
||||||
while i < len(lines) and not lines[i].startswith("```"):
|
|
||||||
codeLines.append(lines[i])
|
|
||||||
i += 1
|
|
||||||
i += 1
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "code_block", "order": order,
|
|
||||||
"elements": [{"content": {"code": "\n".join(codeLines), "language": lang}}],
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Tables ---
|
|
||||||
tableMatch = _re.match(r'^\|(.+)\|$', line)
|
|
||||||
if tableMatch and (i + 1) < len(lines) and _re.match(r'^\|[\s\-:|]+\|$', lines[i + 1]):
|
|
||||||
headerCells = [c.strip() for c in tableMatch.group(1).split("|")]
|
|
||||||
i += 2
|
|
||||||
rows = []
|
|
||||||
while i < len(lines) and _re.match(r'^\|(.+)\|$', lines[i]):
|
|
||||||
rowCells = [c.strip() for c in lines[i][1:-1].split("|")]
|
|
||||||
rows.append(rowCells)
|
|
||||||
i += 1
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "table", "order": order,
|
|
||||||
"elements": [{"content": {"headers": headerCells, "rows": rows}}],
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Bullet / numbered lists ---
|
|
||||||
listMatch = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', line)
|
|
||||||
if listMatch:
|
|
||||||
isNumbered = bool(_re.match(r'\d+[.)]', listMatch.group(2)))
|
|
||||||
items = []
|
|
||||||
while i < len(lines) and _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]):
|
|
||||||
m = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i])
|
|
||||||
items.append({"text": m.group(3).strip()})
|
|
||||||
i += 1
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "bullet_list", "order": order,
|
|
||||||
"elements": [{"content": {"items": items, "list_type": "numbered" if isNumbered else "bullet"}}],
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Empty lines (skip) ---
|
|
||||||
if not line.strip():
|
|
||||||
i += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Images:  or  ---
|
|
||||||
imgMatch = _re.match(r'^!\[([^\]]*)\]\(([^)]+)\)', line)
|
|
||||||
if imgMatch:
|
|
||||||
altText = imgMatch.group(1).strip() or "Image"
|
|
||||||
src = imgMatch.group(2).strip()
|
|
||||||
fileId = ""
|
|
||||||
if src.startswith("file:"):
|
|
||||||
fileId = src[5:]
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "image", "order": order,
|
|
||||||
"elements": [{
|
|
||||||
"content": {
|
|
||||||
"altText": altText,
|
|
||||||
"base64Data": "",
|
|
||||||
"_fileRef": fileId,
|
|
||||||
"_srcUrl": src if not fileId else "",
|
|
||||||
}
|
|
||||||
}],
|
|
||||||
})
|
|
||||||
i += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# --- Paragraph (collect consecutive non-empty lines) ---
|
|
||||||
paraLines = []
|
|
||||||
while i < len(lines) and lines[i].strip() and not _re.match(r'^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )', lines[i]):
|
|
||||||
paraLines.append(lines[i])
|
|
||||||
i += 1
|
|
||||||
if paraLines:
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "paragraph", "order": order,
|
|
||||||
"elements": [{"content": {"text": " ".join(paraLines)}}],
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
if not sections:
|
|
||||||
sections.append({
|
|
||||||
"id": _nextId(), "content_type": "paragraph", "order": order,
|
|
||||||
"elements": [{"content": {"text": markdown.strip() or "(empty)"}}],
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"metadata": {
|
|
||||||
"split_strategy": "single_document",
|
|
||||||
"source_documents": [],
|
|
||||||
"extraction_method": "agent_rendering",
|
|
||||||
"title": title,
|
|
||||||
"language": language,
|
|
||||||
},
|
|
||||||
"documents": [{
|
|
||||||
"id": "doc_1",
|
|
||||||
"title": title,
|
|
||||||
"sections": sections,
|
|
||||||
}],
|
|
||||||
}
|
|
||||||
|
|
||||||
async def _renderDocument(args: Dict[str, Any], context: Dict[str, Any]):
|
async def _renderDocument(args: Dict[str, Any], context: Dict[str, Any]):
|
||||||
"""Render agent-produced markdown content into any document format via the RendererRegistry."""
|
"""Render agent-produced markdown content into any document format via the RendererRegistry."""
|
||||||
|
|
@ -245,35 +114,75 @@ def _registerMediaTools(registry: ToolRegistry, services):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"renderDocument: knowledge service unavailable: {e}")
|
logger.warning(f"renderDocument: knowledge service unavailable: {e}")
|
||||||
resolvedImages = 0
|
resolvedImages = 0
|
||||||
|
|
||||||
|
def _resolveImageRef(targetObj, fileRefKey="_fileRef", fileIdKey="fileId"):
|
||||||
|
"""Resolve a single image reference dict to base64Data in-place."""
|
||||||
|
nonlocal resolvedImages
|
||||||
|
fileRef = targetObj.get(fileRefKey, "") or targetObj.get(fileIdKey, "")
|
||||||
|
if not fileRef or targetObj.get("base64Data"):
|
||||||
|
return
|
||||||
|
if knowledgeService:
|
||||||
|
chunks = knowledgeService._knowledgeDb.getContentChunks(fileRef)
|
||||||
|
imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"]
|
||||||
|
if imageChunks:
|
||||||
|
targetObj["base64Data"] = imageChunks[0].get("data", "")
|
||||||
|
chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType", "image/png")
|
||||||
|
targetObj["mimeType"] = chunkMime
|
||||||
|
resolvedImages += 1
|
||||||
|
if not targetObj.get("base64Data"):
|
||||||
|
try:
|
||||||
|
rawBytes = services.chat.getFileData(fileRef)
|
||||||
|
if rawBytes:
|
||||||
|
import base64 as _b64
|
||||||
|
targetObj["base64Data"] = _b64.b64encode(rawBytes).decode("ascii")
|
||||||
|
targetObj["mimeType"] = "image/png"
|
||||||
|
resolvedImages += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"renderDocument: image resolve failed for fileRef={fileRef}: {e}")
|
||||||
|
targetObj.pop("_fileRef", None)
|
||||||
|
targetObj.pop("_srcUrl", None)
|
||||||
|
|
||||||
|
def _resolveInlineRuns(runsList):
|
||||||
|
"""Scan a list of inline runs and resolve any image runs with fileId."""
|
||||||
|
for run in runsList:
|
||||||
|
if run.get("type") == "image" and run.get("fileId") and not run.get("base64Data"):
|
||||||
|
_resolveImageRef(run, fileRefKey="fileId", fileIdKey="fileId")
|
||||||
|
|
||||||
for doc in structuredContent.get("documents", []):
|
for doc in structuredContent.get("documents", []):
|
||||||
for section in doc.get("sections", []):
|
for section in doc.get("sections", []):
|
||||||
if section.get("content_type") != "image":
|
cType = section.get("content_type")
|
||||||
|
# Block-level image sections
|
||||||
|
if cType == "image":
|
||||||
|
for element in section.get("elements", []):
|
||||||
|
contentObj = element.get("content", {})
|
||||||
|
_resolveImageRef(contentObj)
|
||||||
continue
|
continue
|
||||||
for element in section.get("elements", []):
|
# Paragraphs with inlineRuns
|
||||||
contentObj = element.get("content", {})
|
if cType == "paragraph":
|
||||||
fileRef = contentObj.get("_fileRef", "")
|
for element in section.get("elements", []):
|
||||||
if not fileRef or contentObj.get("base64Data"):
|
runs = element.get("content", {}).get("inlineRuns")
|
||||||
continue
|
if runs:
|
||||||
if knowledgeService:
|
_resolveInlineRuns(runs)
|
||||||
chunks = knowledgeService._knowledgeDb.getContentChunks(fileRef)
|
continue
|
||||||
imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"]
|
# Bullet lists - items are List[List[InlineRun]]
|
||||||
if imageChunks:
|
if cType == "bullet_list":
|
||||||
contentObj["base64Data"] = imageChunks[0].get("data", "")
|
for element in section.get("elements", []):
|
||||||
chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType", "image/png")
|
items = element.get("content", {}).get("items", [])
|
||||||
contentObj["mimeType"] = chunkMime
|
for item in items:
|
||||||
resolvedImages += 1
|
if isinstance(item, list):
|
||||||
if not contentObj.get("base64Data"):
|
_resolveInlineRuns(item)
|
||||||
try:
|
continue
|
||||||
rawBytes = services.chat.getFileData(fileRef)
|
# Tables - headers and row cells are List[InlineRun]
|
||||||
if rawBytes:
|
if cType == "table":
|
||||||
import base64 as _b64
|
for element in section.get("elements", []):
|
||||||
contentObj["base64Data"] = _b64.b64encode(rawBytes).decode("ascii")
|
contentObj = element.get("content", {})
|
||||||
contentObj["mimeType"] = "image/png"
|
for cell in contentObj.get("headers", []):
|
||||||
resolvedImages += 1
|
if isinstance(cell, list):
|
||||||
except Exception as e:
|
_resolveInlineRuns(cell)
|
||||||
logger.warning(f"renderDocument: image resolve failed for fileRef={fileRef}: {e}")
|
for row in contentObj.get("rows", []):
|
||||||
contentObj.pop("_fileRef", None)
|
for cell in row:
|
||||||
contentObj.pop("_srcUrl", None)
|
if isinstance(cell, list):
|
||||||
|
_resolveInlineRuns(cell)
|
||||||
|
|
||||||
sectionCount = len(structuredContent.get("documents", [{}])[0].get("sections", []))
|
sectionCount = len(structuredContent.get("documents", [{}])[0].get("sections", []))
|
||||||
logger.info(f"renderDocument: parsed {sectionCount} sections from markdown ({len(content)} chars), resolved {resolvedImages} image(s), format={outputFormat}")
|
logger.info(f"renderDocument: parsed {sectionCount} sections from markdown ({len(content)} chars), resolved {resolvedImages} image(s), format={outputFormat}")
|
||||||
|
|
@ -285,6 +194,7 @@ def _registerMediaTools(registry: ToolRegistry, services):
|
||||||
language=language,
|
language=language,
|
||||||
title=title,
|
title=title,
|
||||||
userPrompt=content,
|
userPrompt=content,
|
||||||
|
style=args.get("style"),
|
||||||
)
|
)
|
||||||
|
|
||||||
if not documents:
|
if not documents:
|
||||||
|
|
@ -367,6 +277,20 @@ def _registerMediaTools(registry: ToolRegistry, services):
|
||||||
"outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"},
|
"outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"},
|
||||||
"title": {"type": "string", "description": "Document title", "default": "Document"},
|
"title": {"type": "string", "description": "Document title", "default": "Document"},
|
||||||
"language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"},
|
"language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"},
|
||||||
|
"style": {
|
||||||
|
"type": "object",
|
||||||
|
"description": (
|
||||||
|
"Optional style overrides for the rendered document. Supports nested keys: "
|
||||||
|
"fonts (primary, monospace), colors (primary, secondary, accent, background), "
|
||||||
|
"headings (h1-h4 with sizePt, weight, color, spaceBeforePt, spaceAfterPt), "
|
||||||
|
"paragraph (sizePt, lineSpacing, color), table (headerBg, headerFg, headerSizePt, "
|
||||||
|
"bodySizePt, rowBandingEven, rowBandingOdd, borderColor, borderWidthPt), "
|
||||||
|
"list (bulletChar, indentPt, sizePt), image (defaultWidthPt, maxWidthPt, alignment), "
|
||||||
|
"codeBlock (fontSizePt, background, borderColor), "
|
||||||
|
"page (format, marginsPt, showPageNumbers, headerHeight, footerHeight, headerLogo, headerText, footerText). "
|
||||||
|
"Only provided keys override defaults; omitted keys keep their default values."
|
||||||
|
),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
readOnly=False,
|
readOnly=False,
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,10 @@ class _ServicesAdapter:
|
||||||
def workflow(self):
|
def workflow(self):
|
||||||
return self._context.workflow
|
return self._context.workflow
|
||||||
|
|
||||||
|
@workflow.setter
|
||||||
|
def workflow(self, value):
|
||||||
|
self._context.workflow = value
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def chat(self):
|
def chat(self):
|
||||||
return self._get_service("chat")
|
return self._get_service("chat")
|
||||||
|
|
@ -86,7 +90,7 @@ class _ServicesAdapter:
|
||||||
return getattr(w, "featureCode", None) if w else None
|
return getattr(w, "featureCode", None) if w else None
|
||||||
|
|
||||||
def __getattr__(self, name: str):
|
def __getattr__(self, name: str):
|
||||||
if name in ("allowedProviders", "preferredProviders", "currentUserLanguage"):
|
if name in ("allowedProviders", "allowedModels", "preferredProviders", "currentUserLanguage"):
|
||||||
return getattr(self.workflow, name, None) if self.workflow else None
|
return getattr(self.workflow, name, None) if self.workflow else None
|
||||||
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
|
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
|
||||||
|
|
||||||
|
|
@ -194,6 +198,11 @@ class AiService:
|
||||||
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
|
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
|
||||||
logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}")
|
logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}")
|
||||||
|
|
||||||
|
# Calculate effective allowedModels: Workflow ∩ Request (node-level)
|
||||||
|
effectiveModels = self._calculateEffectiveModels(request)
|
||||||
|
if effectiveModels and request.options:
|
||||||
|
request.options = request.options.model_copy(update={'allowedModels': effectiveModels})
|
||||||
|
|
||||||
# Neutralize prompt if enabled (before AI call)
|
# Neutralize prompt if enabled (before AI call)
|
||||||
_wasNeutralized = False
|
_wasNeutralized = False
|
||||||
_excludedDocs: List[str] = []
|
_excludedDocs: List[str] = []
|
||||||
|
|
@ -249,6 +258,11 @@ class AiService:
|
||||||
if effectiveProviders and request.options:
|
if effectiveProviders and request.options:
|
||||||
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
|
request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
|
||||||
|
|
||||||
|
# Calculate effective allowedModels: Workflow ∩ Request (node-level)
|
||||||
|
effectiveModels = self._calculateEffectiveModels(request)
|
||||||
|
if effectiveModels and request.options:
|
||||||
|
request.options = request.options.model_copy(update={'allowedModels': effectiveModels})
|
||||||
|
|
||||||
# Neutralize prompt if enabled (before streaming)
|
# Neutralize prompt if enabled (before streaming)
|
||||||
_wasNeutralized = False
|
_wasNeutralized = False
|
||||||
_excludedDocs: List[str] = []
|
_excludedDocs: List[str] = []
|
||||||
|
|
@ -1264,6 +1278,43 @@ detectedIntent-Werte:
|
||||||
logger.warning(f"Error calculating effective providers: {e}")
|
logger.warning(f"Error calculating effective providers: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _calculateEffectiveModels(self, request: AiCallRequest = None) -> Optional[List[str]]:
|
||||||
|
"""
|
||||||
|
Calculate effective allowed models: Workflow.allowedModels ∩ request.options.allowedModels.
|
||||||
|
|
||||||
|
AND-logic intersection:
|
||||||
|
- If workflow specifies allowedModels, start with those.
|
||||||
|
- If request (node-level) also specifies allowedModels, intersect.
|
||||||
|
- Returns None if no model filtering is needed.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
effectiveModels = None
|
||||||
|
|
||||||
|
# Workflow-level allowedModels (from automation config)
|
||||||
|
workflowModels = getattr(self.services, 'allowedModels', None)
|
||||||
|
if workflowModels:
|
||||||
|
effectiveModels = list(workflowModels)
|
||||||
|
|
||||||
|
# Request-level (node-level) allowedModels
|
||||||
|
requestModels = None
|
||||||
|
if request and request.options and request.options.allowedModels:
|
||||||
|
requestModels = request.options.allowedModels
|
||||||
|
|
||||||
|
if requestModels:
|
||||||
|
if effectiveModels:
|
||||||
|
effectiveModels = [m for m in effectiveModels if m in requestModels]
|
||||||
|
else:
|
||||||
|
effectiveModels = list(requestModels)
|
||||||
|
|
||||||
|
if effectiveModels:
|
||||||
|
logger.debug(f"Model filter: Workflow={workflowModels}, Request={requestModels}, Effective={effectiveModels}")
|
||||||
|
|
||||||
|
return effectiveModels if effectiveModels else None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Error calculating effective models: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
async def ensureAiObjectsInitialized(self):
|
async def ensureAiObjectsInitialized(self):
|
||||||
"""Ensure aiObjects is initialized and submodules are ready."""
|
"""Ensure aiObjects is initialized and submodules are ready."""
|
||||||
if self.aiObjects is None:
|
if self.aiObjects is None:
|
||||||
|
|
|
||||||
|
|
@ -199,13 +199,8 @@ class ChatService:
|
||||||
label = parts[1]
|
label = parts[1]
|
||||||
messageFound = None
|
messageFound = None
|
||||||
for message in workflow.messages:
|
for message in workflow.messages:
|
||||||
# Validate message belongs to this workflow
|
|
||||||
msgWorkflowId = getattr(message, 'workflowId', None)
|
msgWorkflowId = getattr(message, 'workflowId', None)
|
||||||
if not msgWorkflowId or msgWorkflowId != workflowId:
|
if not msgWorkflowId or msgWorkflowId != workflowId:
|
||||||
if msgWorkflowId:
|
|
||||||
logger.warning(f"Message {message.id} has workflowId {msgWorkflowId} but belongs to workflow {workflowId}. Skipping.")
|
|
||||||
else:
|
|
||||||
logger.warning(f"Message {message.id} has no workflowId. Skipping.")
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
msgLabel = getattr(message, 'documentsLabel', None)
|
msgLabel = getattr(message, 'documentsLabel', None)
|
||||||
|
|
@ -213,7 +208,6 @@ class ChatService:
|
||||||
messageFound = message
|
messageFound = message
|
||||||
break
|
break
|
||||||
|
|
||||||
# If found, add documents
|
|
||||||
if messageFound and messageFound.documents:
|
if messageFound and messageFound.documents:
|
||||||
allDocuments.extend(messageFound.documents)
|
allDocuments.extend(messageFound.documents)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ from .subDocumentUtility import (
|
||||||
detectMimeTypeFromData,
|
detectMimeTypeFromData,
|
||||||
convertDocumentDataToString
|
convertDocumentDataToString
|
||||||
)
|
)
|
||||||
|
from .styleDefaults import resolveStyle
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -382,7 +383,7 @@ class GenerationService:
|
||||||
'workflowId': 'unknown'
|
'workflowId': 'unknown'
|
||||||
}
|
}
|
||||||
|
|
||||||
async def renderReport(self, extractedContent: Dict[str, Any], outputFormat: str, language: str, title: str, userPrompt: str = None, aiService=None, parentOperationId: Optional[str] = None) -> List[RenderedDocument]:
|
async def renderReport(self, extractedContent: Dict[str, Any], outputFormat: str, language: str, title: str, userPrompt: str = None, aiService=None, parentOperationId: Optional[str] = None, style: Optional[Dict[str, Any]] = None) -> List[RenderedDocument]:
|
||||||
"""
|
"""
|
||||||
Render extracted JSON content to the specified output format.
|
Render extracted JSON content to the specified output format.
|
||||||
Processes EACH document separately and calls renderer for each.
|
Processes EACH document separately and calls renderer for each.
|
||||||
|
|
@ -399,12 +400,14 @@ class GenerationService:
|
||||||
userPrompt: User's original prompt for report generation
|
userPrompt: User's original prompt for report generation
|
||||||
aiService: AI service instance for generation prompt creation
|
aiService: AI service instance for generation prompt creation
|
||||||
parentOperationId: Optional parent operation ID for hierarchical logging
|
parentOperationId: Optional parent operation ID for hierarchical logging
|
||||||
|
style: Optional style overrides (deep-merged with DEFAULT_STYLE)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of RenderedDocument objects.
|
List of RenderedDocument objects.
|
||||||
Each RenderedDocument represents one rendered file (main document or supporting file)
|
Each RenderedDocument represents one rendered file (main document or supporting file)
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
|
resolvedStyle = resolveStyle(style)
|
||||||
# Validate JSON input
|
# Validate JSON input
|
||||||
if not isinstance(extractedContent, dict):
|
if not isinstance(extractedContent, dict):
|
||||||
raise ValueError("extractedContent must be a JSON dictionary")
|
raise ValueError("extractedContent must be a JSON dictionary")
|
||||||
|
|
@ -469,7 +472,7 @@ class GenerationService:
|
||||||
docTitle = doc.get("title", title)
|
docTitle = doc.get("title", title)
|
||||||
|
|
||||||
# Render this document (can return multiple files, e.g., HTML + images)
|
# Render this document (can return multiple files, e.g., HTML + images)
|
||||||
renderedDocs = await renderer.render(singleDocContent, docTitle, userPrompt, aiService)
|
renderedDocs = await renderer.render(singleDocContent, docTitle, userPrompt, aiService, style=resolvedStyle)
|
||||||
allRenderedDocuments.extend(renderedDocs)
|
allRenderedDocuments.extend(renderedDocs)
|
||||||
|
|
||||||
logger.info(f"Rendered {len(documents)} document(s) into {len(allRenderedDocuments)} file(s)")
|
logger.info(f"Rendered {len(documents)} document(s) into {len(allRenderedDocuments)} file(s)")
|
||||||
|
|
|
||||||
|
|
@ -84,7 +84,7 @@ class BaseRenderer(ABC):
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""
|
"""
|
||||||
Render extracted JSON content to multiple documents.
|
Render extracted JSON content to multiple documents.
|
||||||
Each renderer must implement this method.
|
Each renderer must implement this method.
|
||||||
|
|
@ -95,6 +95,9 @@ class BaseRenderer(ABC):
|
||||||
title: Report title
|
title: Report title
|
||||||
userPrompt: Original user prompt for context
|
userPrompt: Original user prompt for context
|
||||||
aiService: AI service instance for additional processing
|
aiService: AI service instance for additional processing
|
||||||
|
style: Fully-resolved unified style dict from styleDefaults.resolveStyle().
|
||||||
|
When provided, renderers use these values instead of their
|
||||||
|
own defaults / AI-generated styles.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of RenderedDocument objects.
|
List of RenderedDocument objects.
|
||||||
|
|
@ -103,6 +106,112 @@ class BaseRenderer(ABC):
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def _convertUnifiedStyleToInternal(self, style: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Convert the unified resolvedStyle dict (from styleDefaults) into
|
||||||
|
the renderer-internal style-set format that all rendering methods already
|
||||||
|
consume. Override in subclasses for format-specific tweaks."""
|
||||||
|
h1 = style["headings"]["h1"]
|
||||||
|
h2 = style["headings"]["h2"]
|
||||||
|
h3 = style["headings"].get("h3", h2)
|
||||||
|
h4 = style["headings"].get("h4", h3)
|
||||||
|
tbl = style["table"]
|
||||||
|
para = style["paragraph"]
|
||||||
|
lst = style["list"]
|
||||||
|
cb = style["codeBlock"]
|
||||||
|
return {
|
||||||
|
"title": {
|
||||||
|
"font_size": h1["sizePt"], "color": h1["color"],
|
||||||
|
"bold": h1.get("weight") == "bold", "align": "left",
|
||||||
|
},
|
||||||
|
"heading1": {
|
||||||
|
"font_size": h1["sizePt"], "color": h1["color"],
|
||||||
|
"bold": h1.get("weight") == "bold", "align": "left",
|
||||||
|
},
|
||||||
|
"heading2": {
|
||||||
|
"font_size": h2["sizePt"], "color": h2["color"],
|
||||||
|
"bold": h2.get("weight") == "bold", "align": "left",
|
||||||
|
},
|
||||||
|
"heading3": {
|
||||||
|
"font_size": h3["sizePt"], "color": h3["color"],
|
||||||
|
"bold": h3.get("weight") == "bold", "align": "left",
|
||||||
|
},
|
||||||
|
"heading4": {
|
||||||
|
"font_size": h4["sizePt"], "color": h4["color"],
|
||||||
|
"bold": h4.get("weight") == "bold", "align": "left",
|
||||||
|
},
|
||||||
|
"paragraph": {
|
||||||
|
"font_size": para["sizePt"], "color": para["color"],
|
||||||
|
"bold": False, "align": "left",
|
||||||
|
},
|
||||||
|
"table_header": {
|
||||||
|
"background": tbl["headerBg"], "text_color": tbl["headerFg"],
|
||||||
|
"bold": True, "align": "center",
|
||||||
|
},
|
||||||
|
"table_cell": {
|
||||||
|
"background": tbl["rowBandingOdd"], "text_color": para["color"],
|
||||||
|
"bold": False, "align": "left",
|
||||||
|
},
|
||||||
|
"table_border": {
|
||||||
|
"style": "grid", "color": tbl["borderColor"],
|
||||||
|
},
|
||||||
|
"bullet_list": {
|
||||||
|
"font_size": lst["sizePt"], "color": para["color"],
|
||||||
|
"indent": lst["indentPt"],
|
||||||
|
},
|
||||||
|
"code_block": {
|
||||||
|
"font": style["fonts"]["monospace"],
|
||||||
|
"font_size": cb["fontSizePt"], "color": para["color"],
|
||||||
|
"background": cb["background"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _inlineRunsFromContent(content: Dict[str, Any], *, itemsKey: str = None) -> Any:
|
||||||
|
"""Extract inline runs from new-format content, falling back to old format.
|
||||||
|
|
||||||
|
For paragraphs (itemsKey=None):
|
||||||
|
new: content["inlineRuns"] -> List[InlineRun]
|
||||||
|
old: content["text"] -> wrapped in [{"type":"text","value":text}]
|
||||||
|
|
||||||
|
For list items (itemsKey="items"):
|
||||||
|
new: content["items"] is List[List[InlineRun]]
|
||||||
|
old: content["items"] is List[str] or List[{"text":…}]
|
||||||
|
Returns the items list (caller decides per-item conversion).
|
||||||
|
|
||||||
|
For table headers/cells:
|
||||||
|
new: each header/cell is List[InlineRun]
|
||||||
|
old: each header/cell is a plain str
|
||||||
|
Caller handles per-cell.
|
||||||
|
"""
|
||||||
|
if itemsKey:
|
||||||
|
return content.get(itemsKey, [])
|
||||||
|
inlineRuns = content.get("inlineRuns")
|
||||||
|
if inlineRuns:
|
||||||
|
return inlineRuns
|
||||||
|
text = content.get("text", "")
|
||||||
|
if text:
|
||||||
|
return [{"type": "text", "value": text}]
|
||||||
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _inlineRunsForCell(cell) -> list:
|
||||||
|
"""Normalize a single table header or cell value to List[InlineRun].
|
||||||
|
Accepts either a plain string or an already-correct list of run dicts."""
|
||||||
|
if isinstance(cell, list):
|
||||||
|
return cell
|
||||||
|
return [{"type": "text", "value": str(cell) if cell is not None else ""}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _inlineRunsForListItem(item) -> list:
|
||||||
|
"""Normalize a single list item to List[InlineRun].
|
||||||
|
Accepts a plain string, a dict with 'text', or an already-correct list of run dicts."""
|
||||||
|
if isinstance(item, list):
|
||||||
|
return item
|
||||||
|
if isinstance(item, dict):
|
||||||
|
text = item.get("text", "")
|
||||||
|
return [{"type": "text", "value": text}]
|
||||||
|
return [{"type": "text", "value": str(item)}]
|
||||||
|
|
||||||
def _determineFilename(self, title: str, mimeType: str) -> str:
|
def _determineFilename(self, title: str, mimeType: str) -> str:
|
||||||
"""Determine filename from title and mimeType."""
|
"""Determine filename from title and mimeType."""
|
||||||
import re
|
import re
|
||||||
|
|
|
||||||
|
|
@ -53,18 +53,17 @@ class RendererDocx(BaseRenderer):
|
||||||
from modules.datamodels.datamodelJson import supportedSectionTypes
|
from modules.datamodels.datamodelJson import supportedSectionTypes
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""Render extracted JSON content to DOCX format using AI-analyzed styling."""
|
"""Render extracted JSON content to DOCX format using AI-analyzed styling."""
|
||||||
self.services.utils.debugLogToFile(f"DOCX RENDER CALLED: title={title}, user_prompt={userPrompt[:50] if userPrompt else 'None'}...", "DOCX_RENDERER")
|
self.services.utils.debugLogToFile(f"DOCX RENDER CALLED: title={title}, user_prompt={userPrompt[:50] if userPrompt else 'None'}...", "DOCX_RENDERER")
|
||||||
try:
|
try:
|
||||||
if not DOCX_AVAILABLE:
|
if not DOCX_AVAILABLE:
|
||||||
# Fallback to HTML if python-docx not available
|
|
||||||
from .rendererHtml import RendererHtml
|
from .rendererHtml import RendererHtml
|
||||||
htmlRenderer = RendererHtml()
|
htmlRenderer = RendererHtml()
|
||||||
return await htmlRenderer.render(extractedContent, title, userPrompt, aiService)
|
return await htmlRenderer.render(extractedContent, title, userPrompt, aiService, style=style)
|
||||||
|
|
||||||
# Generate DOCX using AI-analyzed styling
|
# Generate DOCX using AI-analyzed styling
|
||||||
docx_content = await self._generateDocxFromJson(extractedContent, title, userPrompt, aiService)
|
docx_content = await self._generateDocxFromJson(extractedContent, title, userPrompt, aiService, unifiedStyle=style)
|
||||||
|
|
||||||
# Extract metadata for document type and other info
|
# Extract metadata for document type and other info
|
||||||
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
||||||
|
|
@ -114,23 +113,27 @@ class RendererDocx(BaseRenderer):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
async def _generateDocxFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str:
|
async def _generateDocxFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, unifiedStyle: Dict[str, Any] = None) -> str:
|
||||||
"""Generate DOCX content from structured JSON document."""
|
"""Generate DOCX content from structured JSON document."""
|
||||||
import time
|
import time
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
try:
|
try:
|
||||||
self.logger.debug("_generateDocxFromJson: Starting document generation")
|
self.logger.debug("_generateDocxFromJson: Starting document generation")
|
||||||
# Create new document
|
|
||||||
doc = Document()
|
doc = Document()
|
||||||
self.logger.debug(f"_generateDocxFromJson: Document created in {time.time() - start_time:.2f}s")
|
self.logger.debug(f"_generateDocxFromJson: Document created in {time.time() - start_time:.2f}s")
|
||||||
|
|
||||||
# Get style set: use styles from metadata if available, otherwise enhance with AI
|
# Phase 3: prefer unified style when provided
|
||||||
template_from_metadata = None
|
|
||||||
if json_content and isinstance(json_content.get("metadata"), dict):
|
|
||||||
template_from_metadata = json_content["metadata"].get("templateName")
|
|
||||||
style_start = time.time()
|
style_start = time.time()
|
||||||
self.logger.debug("_generateDocxFromJson: About to get style set")
|
self.logger.debug("_generateDocxFromJson: About to get style set")
|
||||||
styleSet = await self._getStyleSet(json_content, userPrompt, aiService, templateName=template_from_metadata)
|
if unifiedStyle:
|
||||||
|
styleSet = self._convertUnifiedStyleToInternal(unifiedStyle)
|
||||||
|
self._unifiedStyle = unifiedStyle
|
||||||
|
else:
|
||||||
|
template_from_metadata = None
|
||||||
|
if json_content and isinstance(json_content.get("metadata"), dict):
|
||||||
|
template_from_metadata = json_content["metadata"].get("templateName")
|
||||||
|
styleSet = await self._getStyleSet(json_content, userPrompt, aiService, templateName=template_from_metadata)
|
||||||
|
self._unifiedStyle = None
|
||||||
self.logger.debug(f"_generateDocxFromJson: Style set retrieved in {time.time() - style_start:.2f}s")
|
self.logger.debug(f"_generateDocxFromJson: Style set retrieved in {time.time() - style_start:.2f}s")
|
||||||
|
|
||||||
# Setup basic document styles and create all styles from style set
|
# Setup basic document styles and create all styles from style set
|
||||||
|
|
@ -298,11 +301,11 @@ class RendererDocx(BaseRenderer):
|
||||||
def _setupBasicDocumentStyles(self, doc: Document) -> None:
|
def _setupBasicDocumentStyles(self, doc: Document) -> None:
|
||||||
"""Set up basic document styles."""
|
"""Set up basic document styles."""
|
||||||
try:
|
try:
|
||||||
# Set default font
|
|
||||||
style = doc.styles['Normal']
|
style = doc.styles['Normal']
|
||||||
font = style.font
|
font = style.font
|
||||||
font.name = 'Calibri'
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
font.size = Pt(11)
|
font.name = us["fonts"]["primary"] if us else 'Calibri'
|
||||||
|
font.size = Pt(us["paragraph"]["sizePt"] if us else 11)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Could not set up basic document styles: {str(e)}")
|
self.logger.warning(f"Could not set up basic document styles: {str(e)}")
|
||||||
|
|
||||||
|
|
@ -421,6 +424,8 @@ class RendererDocx(BaseRenderer):
|
||||||
def _addMarkdownInlineRuns(self, paragraph, text: str) -> None:
|
def _addMarkdownInlineRuns(self, paragraph, text: str) -> None:
|
||||||
"""Parse markdown inline formatting and add corresponding Runs to a python-docx paragraph."""
|
"""Parse markdown inline formatting and add corresponding Runs to a python-docx paragraph."""
|
||||||
pos = 0
|
pos = 0
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
monoFont = us["fonts"]["monospace"] if us else "Courier New"
|
||||||
for m in self._MD_INLINE_RE.finditer(text):
|
for m in self._MD_INLINE_RE.finditer(text):
|
||||||
if m.start() > pos:
|
if m.start() > pos:
|
||||||
paragraph.add_run(text[pos:m.start()])
|
paragraph.add_run(text[pos:m.start()])
|
||||||
|
|
@ -434,12 +439,45 @@ class RendererDocx(BaseRenderer):
|
||||||
paragraph.add_run(m.group(6)).italic = True
|
paragraph.add_run(m.group(6)).italic = True
|
||||||
elif m.group(7):
|
elif m.group(7):
|
||||||
run = paragraph.add_run(m.group(7))
|
run = paragraph.add_run(m.group(7))
|
||||||
run.font.name = "Courier New"
|
run.font.name = monoFont
|
||||||
run.font.size = Pt(9)
|
run.font.size = Pt(9)
|
||||||
pos = m.end()
|
pos = m.end()
|
||||||
if pos < len(text):
|
if pos < len(text):
|
||||||
paragraph.add_run(text[pos:])
|
paragraph.add_run(text[pos:])
|
||||||
|
|
||||||
|
def _renderInlineRuns(self, runs: list, paragraph, styleSet: Dict[str, Any]) -> None:
|
||||||
|
"""Process a list of InlineRun dicts into python-docx Runs on a paragraph."""
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
monoFont = us["fonts"]["monospace"] if us else "Courier New"
|
||||||
|
for run in runs:
|
||||||
|
runType = run.get("type", "text")
|
||||||
|
value = run.get("value", "")
|
||||||
|
if runType == "text":
|
||||||
|
paragraph.add_run(value)
|
||||||
|
elif runType == "bold":
|
||||||
|
paragraph.add_run(value).bold = True
|
||||||
|
elif runType == "italic":
|
||||||
|
paragraph.add_run(value).italic = True
|
||||||
|
elif runType == "code":
|
||||||
|
r = paragraph.add_run(value)
|
||||||
|
r.font.name = monoFont
|
||||||
|
r.font.size = Pt(9)
|
||||||
|
elif runType == "link":
|
||||||
|
r = paragraph.add_run(value)
|
||||||
|
r.font.underline = True
|
||||||
|
r.font.color.rgb = RGBColor(0x29, 0x80, 0xB9)
|
||||||
|
elif runType == "image":
|
||||||
|
b64 = run.get("base64Data", "")
|
||||||
|
if b64:
|
||||||
|
try:
|
||||||
|
imgBytes = base64.b64decode(b64)
|
||||||
|
imgStream = io.BytesIO(imgBytes)
|
||||||
|
paragraph.add_run().add_picture(imgStream, width=Inches(2))
|
||||||
|
except Exception:
|
||||||
|
paragraph.add_run(f"[Image: {run.get('altText', '')}]")
|
||||||
|
else:
|
||||||
|
paragraph.add_run(value)
|
||||||
|
|
||||||
def _renderJsonTable(self, doc: Document, table_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
def _renderJsonTable(self, doc: Document, table_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
||||||
"""
|
"""
|
||||||
Render a JSON table to DOCX using AI-generated styles.
|
Render a JSON table to DOCX using AI-generated styles.
|
||||||
|
|
@ -485,7 +523,7 @@ class RendererDocx(BaseRenderer):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error rendering table: {str(e)}", exc_info=True)
|
self.logger.error(f"Error rendering table: {str(e)}", exc_info=True)
|
||||||
|
|
||||||
def _renderTableFastXml(self, doc: Document, headers: List[str], rows: List[List[Any]], styles: Dict[str, Any]) -> None:
|
def _renderTableFastXml(self, doc: Document, headers: list, rows: list, styles: Dict[str, Any]) -> None:
|
||||||
"""
|
"""
|
||||||
High-performance table rendering using direct XML manipulation.
|
High-performance table rendering using direct XML manipulation.
|
||||||
|
|
||||||
|
|
@ -546,22 +584,32 @@ class RendererDocx(BaseRenderer):
|
||||||
# Build all rows using fast XML
|
# Build all rows using fast XML
|
||||||
rows_start = time.time()
|
rows_start = time.time()
|
||||||
|
|
||||||
# Header row
|
# Resolve header style colors
|
||||||
headerRow = self._createTableRowXml(headers, isHeader=True)
|
tableStyle = styles.get("table_header", {})
|
||||||
|
headerBg = tableStyle.get("background", "")
|
||||||
|
headerFg = tableStyle.get("text_color", "")
|
||||||
|
|
||||||
|
# Flatten inline-run headers to plain strings for fast XML path
|
||||||
|
flatHeaders = []
|
||||||
|
for h in headers:
|
||||||
|
runs = self._inlineRunsForCell(h)
|
||||||
|
flatHeaders.append("".join(r.get("value", "") for r in runs))
|
||||||
|
|
||||||
|
headerRow = self._createTableRowXml(flatHeaders, isHeader=True, headerBgHex=headerBg or None, headerFgHex=headerFg or None)
|
||||||
tbl.append(headerRow)
|
tbl.append(headerRow)
|
||||||
|
|
||||||
header_time = time.time() - rows_start
|
header_time = time.time() - rows_start
|
||||||
self.logger.debug(f"_renderTableFastXml: Header row created in {header_time:.3f}s")
|
self.logger.debug(f"_renderTableFastXml: Header row created in {header_time:.3f}s")
|
||||||
|
|
||||||
# Data rows - batch process for performance
|
|
||||||
data_start = time.time()
|
data_start = time.time()
|
||||||
rowCount = len(rows)
|
rowCount = len(rows)
|
||||||
|
|
||||||
for idx, rowData in enumerate(rows):
|
for idx, rowData in enumerate(rows):
|
||||||
# Convert all cells to strings
|
cellTexts = []
|
||||||
cellTexts = [str(cell) if cell is not None else '' for cell in rowData]
|
for cell in rowData:
|
||||||
# Pad if needed
|
runs = self._inlineRunsForCell(cell)
|
||||||
while len(cellTexts) < len(headers):
|
cellTexts.append("".join(r.get("value", "") for r in runs))
|
||||||
|
while len(cellTexts) < len(flatHeaders):
|
||||||
cellTexts.append('')
|
cellTexts.append('')
|
||||||
|
|
||||||
row = self._createTableRowXml(cellTexts, isHeader=False)
|
row = self._createTableRowXml(cellTexts, isHeader=False)
|
||||||
|
|
@ -641,70 +689,60 @@ class RendererDocx(BaseRenderer):
|
||||||
|
|
||||||
return tblBorders
|
return tblBorders
|
||||||
|
|
||||||
def _createTableRowXml(self, cells: List[str], isHeader: bool = False) -> Any:
|
def _createTableRowXml(self, cells: list, isHeader: bool = False, headerBgHex: str = None, headerFgHex: str = None) -> Any:
|
||||||
"""
|
"""Create a table row XML element with cells.
|
||||||
Create a table row XML element with cells.
|
Fast-path: builds row XML directly via lxml."""
|
||||||
|
|
||||||
This is the core fast-path: builds the row XML directly without
|
|
||||||
going through python-docx's slow cell.text assignment.
|
|
||||||
"""
|
|
||||||
from docx.oxml.shared import OxmlElement, qn
|
from docx.oxml.shared import OxmlElement, qn
|
||||||
|
|
||||||
tr = OxmlElement('w:tr')
|
if headerBgHex is None:
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
headerBgHex = us["table"]["headerBg"].lstrip('#') if us else '1F3864'
|
||||||
|
else:
|
||||||
|
headerBgHex = headerBgHex.lstrip('#')
|
||||||
|
if headerFgHex is None:
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
headerFgHex = us["table"]["headerFg"].lstrip('#') if us else 'FFFFFF'
|
||||||
|
else:
|
||||||
|
headerFgHex = headerFgHex.lstrip('#')
|
||||||
|
|
||||||
# Row properties for header
|
tr = OxmlElement('w:tr')
|
||||||
if isHeader:
|
if isHeader:
|
||||||
trPr = OxmlElement('w:trPr')
|
trPr = OxmlElement('w:trPr')
|
||||||
tblHeader = OxmlElement('w:tblHeader')
|
trPr.append(OxmlElement('w:tblHeader'))
|
||||||
trPr.append(tblHeader)
|
|
||||||
tr.append(trPr)
|
tr.append(trPr)
|
||||||
|
|
||||||
for cellText in cells:
|
for cellText in cells:
|
||||||
# Create cell
|
|
||||||
tc = OxmlElement('w:tc')
|
tc = OxmlElement('w:tc')
|
||||||
|
|
||||||
# Cell properties
|
|
||||||
tcPr = OxmlElement('w:tcPr')
|
tcPr = OxmlElement('w:tcPr')
|
||||||
tcW = OxmlElement('w:tcW')
|
tcW = OxmlElement('w:tcW')
|
||||||
tcW.set(qn('w:type'), 'auto')
|
tcW.set(qn('w:type'), 'auto')
|
||||||
tcW.set(qn('w:w'), '0')
|
tcW.set(qn('w:w'), '0')
|
||||||
tcPr.append(tcW)
|
tcPr.append(tcW)
|
||||||
|
|
||||||
# Header cell styling - light blue background
|
|
||||||
if isHeader:
|
if isHeader:
|
||||||
shd = OxmlElement('w:shd')
|
shd = OxmlElement('w:shd')
|
||||||
shd.set(qn('w:val'), 'clear')
|
shd.set(qn('w:val'), 'clear')
|
||||||
shd.set(qn('w:color'), 'auto')
|
shd.set(qn('w:color'), 'auto')
|
||||||
shd.set(qn('w:fill'), '4472C4') # Professional blue
|
shd.set(qn('w:fill'), headerBgHex)
|
||||||
tcPr.append(shd)
|
tcPr.append(shd)
|
||||||
|
|
||||||
tc.append(tcPr)
|
tc.append(tcPr)
|
||||||
|
|
||||||
# Paragraph with text
|
|
||||||
p = OxmlElement('w:p')
|
p = OxmlElement('w:p')
|
||||||
|
|
||||||
# Add run with text
|
|
||||||
r = OxmlElement('w:r')
|
r = OxmlElement('w:r')
|
||||||
|
|
||||||
# Header text styling - bold and white
|
|
||||||
if isHeader:
|
if isHeader:
|
||||||
rPr = OxmlElement('w:rPr')
|
rPr = OxmlElement('w:rPr')
|
||||||
b = OxmlElement('w:b')
|
rPr.append(OxmlElement('w:b'))
|
||||||
rPr.append(b)
|
|
||||||
# White text color
|
|
||||||
color = OxmlElement('w:color')
|
color = OxmlElement('w:color')
|
||||||
color.set(qn('w:val'), 'FFFFFF')
|
color.set(qn('w:val'), headerFgHex)
|
||||||
rPr.append(color)
|
rPr.append(color)
|
||||||
r.append(rPr)
|
r.append(rPr)
|
||||||
|
|
||||||
# Text element
|
|
||||||
t = OxmlElement('w:t')
|
t = OxmlElement('w:t')
|
||||||
# Preserve spaces if text starts/ends with whitespace
|
|
||||||
if cellText and (cellText[0] == ' ' or cellText[-1] == ' '):
|
if cellText and (cellText[0] == ' ' or cellText[-1] == ' '):
|
||||||
t.set('{http://www.w3.org/XML/1998/namespace}space', 'preserve')
|
t.set('{http://www.w3.org/XML/1998/namespace}space', 'preserve')
|
||||||
t.text = cellText
|
t.text = cellText
|
||||||
r.append(t)
|
r.append(t)
|
||||||
|
|
||||||
p.append(r)
|
p.append(r)
|
||||||
tc.append(p)
|
tc.append(p)
|
||||||
tr.append(tc)
|
tr.append(tc)
|
||||||
|
|
@ -836,46 +874,36 @@ class RendererDocx(BaseRenderer):
|
||||||
def _renderJsonBulletList(self, doc: Document, list_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
def _renderJsonBulletList(self, doc: Document, list_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
||||||
"""Render a JSON bullet list to DOCX using AI-generated styles - OPTIMIZED for performance."""
|
"""Render a JSON bullet list to DOCX using AI-generated styles - OPTIMIZED for performance."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = list_data.get("content", {})
|
content = list_data.get("content", {})
|
||||||
if not isinstance(content, dict):
|
if not isinstance(content, dict):
|
||||||
return
|
return
|
||||||
items = content.get("items", [])
|
items = content.get("items", [])
|
||||||
bullet_style = styles.get("bullet_list", {})
|
bullet_style = styles.get("bullet_list", {})
|
||||||
|
|
||||||
# Pre-calculate and cache style objects to avoid repeated parsing
|
font_size_pt = Pt(bullet_style["font_size"]) if bullet_style.get("font_size") else None
|
||||||
font_size_pt = None
|
|
||||||
text_color_rgb = None
|
text_color_rgb = None
|
||||||
if bullet_style:
|
if bullet_style.get("color"):
|
||||||
if "font_size" in bullet_style:
|
color_hex = bullet_style["color"].lstrip('#')
|
||||||
font_size_pt = Pt(bullet_style["font_size"])
|
text_color_rgb = RGBColor(int(color_hex[0:2], 16), int(color_hex[2:4], 16), int(color_hex[4:6], 16))
|
||||||
if "color" in bullet_style:
|
|
||||||
color_hex = bullet_style["color"].lstrip('#')
|
|
||||||
text_color_rgb = RGBColor(int(color_hex[0:2], 16), int(color_hex[2:4], 16), int(color_hex[4:6], 16))
|
|
||||||
|
|
||||||
for item in items:
|
for item in items:
|
||||||
itemText = item if isinstance(item, str) else (item.get("text", "") if isinstance(item, dict) else "")
|
itemRuns = self._inlineRunsForListItem(item)
|
||||||
if not itemText:
|
if not itemRuns or not any(r.get("value") for r in itemRuns):
|
||||||
continue
|
continue
|
||||||
para = doc.add_paragraph(style='List Bullet')
|
para = doc.add_paragraph(style='List Bullet')
|
||||||
self._addMarkdownInlineRuns(para, itemText)
|
isNewRunFormat = isinstance(item, list)
|
||||||
|
if isNewRunFormat:
|
||||||
|
self._renderInlineRuns(itemRuns, para, styles)
|
||||||
|
else:
|
||||||
|
itemText = "".join(r.get("value", "") for r in itemRuns)
|
||||||
|
self._addMarkdownInlineRuns(para, itemText)
|
||||||
|
|
||||||
# Apply bullet list styling from style set - use cached objects
|
if bullet_style and para.runs and len(para.runs) > 0:
|
||||||
if bullet_style and para.runs:
|
run = para.runs[0]
|
||||||
# Use direct access instead of iterating
|
if font_size_pt:
|
||||||
if len(para.runs) > 0:
|
run.font.size = font_size_pt
|
||||||
run = para.runs[0]
|
if text_color_rgb:
|
||||||
if font_size_pt:
|
run.font.color.rgb = text_color_rgb
|
||||||
run.font.size = font_size_pt
|
|
||||||
if text_color_rgb:
|
|
||||||
run.font.color.rgb = text_color_rgb
|
|
||||||
else:
|
|
||||||
# Create run if none exists
|
|
||||||
run = para.add_run()
|
|
||||||
if font_size_pt:
|
|
||||||
run.font.size = font_size_pt
|
|
||||||
if text_color_rgb:
|
|
||||||
run.font.color.rgb = text_color_rgb
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Error rendering bullet list: {str(e)}")
|
self.logger.warning(f"Error rendering bullet list: {str(e)}")
|
||||||
|
|
@ -905,66 +933,56 @@ class RendererDocx(BaseRenderer):
|
||||||
def _renderJsonParagraph(self, doc: Document, paragraph_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
def _renderJsonParagraph(self, doc: Document, paragraph_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
||||||
"""Render a JSON paragraph to DOCX using AI-generated styles."""
|
"""Render a JSON paragraph to DOCX using AI-generated styles."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = paragraph_data.get("content", {})
|
content = paragraph_data.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
text = content.get("text", "")
|
inlineRuns = self._inlineRunsFromContent(content)
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
inlineRuns = [{"type": "text", "value": content}]
|
||||||
else:
|
else:
|
||||||
text = ""
|
inlineRuns = []
|
||||||
|
|
||||||
# CRITICAL: Prevent rendering base64 image data as text
|
if not inlineRuns:
|
||||||
# Base64 image data typically starts with /9j/ (JPEG) or iVBORw0KGgo (PNG)
|
|
||||||
if text and (text.startswith("/9j/") or text.startswith("iVBORw0KGgo") or
|
|
||||||
(len(text) > 100 and all(c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" for c in text[:100]))):
|
|
||||||
# This looks like base64 data - don't render as text
|
|
||||||
self.logger.warning(f"Skipping rendering of what appears to be base64 data in paragraph (length: {len(text)})")
|
|
||||||
para = doc.add_paragraph("[Error: Image data found in text content - image embedding may have failed]")
|
|
||||||
if para.runs:
|
|
||||||
para.runs[0].font.color.rgb = RGBColor(255, 0, 0) # Red color for error
|
|
||||||
return
|
return
|
||||||
|
|
||||||
if text:
|
plainText = "".join(r.get("value", "") for r in inlineRuns)
|
||||||
para = doc.add_paragraph()
|
if plainText and (plainText.startswith("/9j/") or plainText.startswith("iVBORw0KGgo") or
|
||||||
self._addMarkdownInlineRuns(para, text)
|
(len(plainText) > 100 and all(c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" for c in plainText[:100]))):
|
||||||
paragraph_style = styles.get("paragraph", {})
|
self.logger.warning(f"Skipping rendering of what appears to be base64 data in paragraph (length: {len(plainText)})")
|
||||||
if paragraph_style:
|
para = doc.add_paragraph("[Error: Image data found in text content - image embedding may have failed]")
|
||||||
# Pre-calculate and cache style objects
|
if para.runs:
|
||||||
font_size_pt = None
|
para.runs[0].font.color.rgb = RGBColor(255, 0, 0)
|
||||||
text_color_rgb = None
|
return
|
||||||
if "font_size" in paragraph_style:
|
|
||||||
font_size_pt = Pt(paragraph_style["font_size"])
|
|
||||||
if "color" in paragraph_style:
|
|
||||||
color_hex = paragraph_style["color"].lstrip('#')
|
|
||||||
text_color_rgb = RGBColor(int(color_hex[0:2], 16), int(color_hex[2:4], 16), int(color_hex[4:6], 16))
|
|
||||||
bold = paragraph_style.get("bold", False)
|
|
||||||
|
|
||||||
# Use direct access instead of iterating
|
para = doc.add_paragraph()
|
||||||
if len(para.runs) > 0:
|
hasNewRuns = content.get("inlineRuns") if isinstance(content, dict) else None
|
||||||
run = para.runs[0]
|
if hasNewRuns:
|
||||||
if font_size_pt:
|
self._renderInlineRuns(inlineRuns, para, styles)
|
||||||
run.font.size = font_size_pt
|
else:
|
||||||
run.font.bold = bold
|
self._addMarkdownInlineRuns(para, plainText)
|
||||||
if text_color_rgb:
|
|
||||||
run.font.color.rgb = text_color_rgb
|
paragraph_style = styles.get("paragraph", {})
|
||||||
|
if paragraph_style:
|
||||||
|
font_size_pt = Pt(paragraph_style["font_size"]) if "font_size" in paragraph_style else None
|
||||||
|
text_color_rgb = None
|
||||||
|
if "color" in paragraph_style:
|
||||||
|
color_hex = paragraph_style["color"].lstrip('#')
|
||||||
|
text_color_rgb = RGBColor(int(color_hex[0:2], 16), int(color_hex[2:4], 16), int(color_hex[4:6], 16))
|
||||||
|
bold = paragraph_style.get("bold", False)
|
||||||
|
if len(para.runs) > 0:
|
||||||
|
run = para.runs[0]
|
||||||
|
if font_size_pt:
|
||||||
|
run.font.size = font_size_pt
|
||||||
|
run.font.bold = bold
|
||||||
|
if text_color_rgb:
|
||||||
|
run.font.color.rgb = text_color_rgb
|
||||||
|
if "align" in paragraph_style:
|
||||||
|
align = paragraph_style["align"]
|
||||||
|
if align == "center":
|
||||||
|
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||||
|
elif align == "right":
|
||||||
|
para.alignment = WD_ALIGN_PARAGRAPH.RIGHT
|
||||||
else:
|
else:
|
||||||
# Create run if none exists
|
para.alignment = WD_ALIGN_PARAGRAPH.LEFT
|
||||||
run = para.add_run()
|
|
||||||
if font_size_pt:
|
|
||||||
run.font.size = font_size_pt
|
|
||||||
run.font.bold = bold
|
|
||||||
if text_color_rgb:
|
|
||||||
run.font.color.rgb = text_color_rgb
|
|
||||||
|
|
||||||
if "align" in paragraph_style:
|
|
||||||
align = paragraph_style["align"]
|
|
||||||
if align == "center":
|
|
||||||
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
|
||||||
elif align == "right":
|
|
||||||
para.alignment = WD_ALIGN_PARAGRAPH.RIGHT
|
|
||||||
else:
|
|
||||||
para.alignment = WD_ALIGN_PARAGRAPH.LEFT
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Error rendering paragraph: {str(e)}")
|
self.logger.warning(f"Error rendering paragraph: {str(e)}")
|
||||||
|
|
@ -972,13 +990,13 @@ class RendererDocx(BaseRenderer):
|
||||||
def _renderJsonCodeBlock(self, doc: Document, code_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
def _renderJsonCodeBlock(self, doc: Document, code_data: Dict[str, Any], styles: Dict[str, Any]) -> None:
|
||||||
"""Render a JSON code block to DOCX using AI-generated styles."""
|
"""Render a JSON code block to DOCX using AI-generated styles."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = code_data.get("content", {})
|
content = code_data.get("content", {})
|
||||||
if not isinstance(content, dict):
|
if not isinstance(content, dict):
|
||||||
return
|
return
|
||||||
code = content.get("code", "")
|
code = content.get("code", "")
|
||||||
language = content.get("language", "")
|
language = content.get("language", "")
|
||||||
code_style = styles.get("code_block", {})
|
code_style = styles.get("code_block", {})
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
|
||||||
if code:
|
if code:
|
||||||
if language:
|
if language:
|
||||||
|
|
@ -986,9 +1004,8 @@ class RendererDocx(BaseRenderer):
|
||||||
if len(lang_para.runs) > 0:
|
if len(lang_para.runs) > 0:
|
||||||
lang_para.runs[0].bold = True
|
lang_para.runs[0].bold = True
|
||||||
|
|
||||||
# Pre-calculate and cache style objects
|
code_font_name = code_style.get("font", us["fonts"]["monospace"] if us else "Courier New")
|
||||||
code_font_name = code_style.get("font", "Courier New")
|
code_font_size_pt = Pt(code_style.get("font_size", us["codeBlock"]["fontSizePt"] if us else 9))
|
||||||
code_font_size_pt = Pt(code_style.get("font_size", 9))
|
|
||||||
code_text_color_rgb = None
|
code_text_color_rgb = None
|
||||||
if "color" in code_style:
|
if "color" in code_style:
|
||||||
color_hex = code_style["color"].lstrip('#')
|
color_hex = code_style["color"].lstrip('#')
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,7 @@ class RendererHtml(BaseRenderer):
|
||||||
from modules.datamodels.datamodelJson import supportedSectionTypes
|
from modules.datamodels.datamodelJson import supportedSectionTypes
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""
|
"""
|
||||||
Render HTML document with images as separate files.
|
Render HTML document with images as separate files.
|
||||||
Returns list of documents: [HTML document, image1, image2, ...]
|
Returns list of documents: [HTML document, image1, image2, ...]
|
||||||
|
|
@ -54,7 +54,7 @@ class RendererHtml(BaseRenderer):
|
||||||
self._renderedImages = images
|
self._renderedImages = images
|
||||||
|
|
||||||
# Generate HTML using AI-analyzed styling
|
# Generate HTML using AI-analyzed styling
|
||||||
htmlContent = await self._generateHtmlFromJson(extractedContent, title, userPrompt, aiService)
|
htmlContent = await self._generateHtmlFromJson(extractedContent, title, userPrompt, aiService, style=style)
|
||||||
|
|
||||||
# Replace base64 data URIs with relative file paths if images exist
|
# Replace base64 data URIs with relative file paths if images exist
|
||||||
if images:
|
if images:
|
||||||
|
|
@ -107,11 +107,16 @@ class RendererHtml(BaseRenderer):
|
||||||
|
|
||||||
return resultDocuments
|
return resultDocuments
|
||||||
|
|
||||||
async def _generateHtmlFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str:
|
async def _generateHtmlFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> str:
|
||||||
"""Generate HTML content from structured JSON document using AI-generated styling."""
|
"""Generate HTML content from structured JSON document using AI-generated styling."""
|
||||||
try:
|
try:
|
||||||
# Get style set: use styles from metadata if available, otherwise enhance with AI
|
# Use unified style when provided, otherwise fall back to existing flow
|
||||||
styles = await self._getStyleSet(jsonContent, userPrompt, aiService)
|
if style:
|
||||||
|
styles = self._convertUnifiedStyleToInternal(style)
|
||||||
|
self._unifiedStyle = style
|
||||||
|
else:
|
||||||
|
styles = await self._getStyleSet(jsonContent, userPrompt, aiService)
|
||||||
|
self._unifiedStyle = None
|
||||||
|
|
||||||
# Validate JSON structure
|
# Validate JSON structure
|
||||||
if not self._validateJsonStructure(jsonContent):
|
if not self._validateJsonStructure(jsonContent):
|
||||||
|
|
@ -272,6 +277,10 @@ class RendererHtml(BaseRenderer):
|
||||||
|
|
||||||
def _generateCssStyles(self, styles: Dict[str, Any]) -> str:
|
def _generateCssStyles(self, styles: Dict[str, Any]) -> str:
|
||||||
"""Generate CSS from style definitions."""
|
"""Generate CSS from style definitions."""
|
||||||
|
# When unified style is available, generate CSS directly from it
|
||||||
|
if getattr(self, "_unifiedStyle", None):
|
||||||
|
return self._generateCssFromUnifiedStyle(self._unifiedStyle)
|
||||||
|
|
||||||
css_parts = []
|
css_parts = []
|
||||||
|
|
||||||
# Body styles
|
# Body styles
|
||||||
|
|
@ -368,6 +377,164 @@ class RendererHtml(BaseRenderer):
|
||||||
|
|
||||||
return '\n'.join(css_parts)
|
return '\n'.join(css_parts)
|
||||||
|
|
||||||
|
def _generateCssFromUnifiedStyle(self, style: Dict[str, Any]) -> str:
|
||||||
|
"""Generate CSS directly from unified style dict."""
|
||||||
|
fonts = style.get("fonts", {})
|
||||||
|
colors = style.get("colors", {})
|
||||||
|
headings = style.get("headings", {})
|
||||||
|
para = style.get("paragraph", {})
|
||||||
|
tbl = style.get("table", {})
|
||||||
|
lst = style.get("list", {})
|
||||||
|
cb = style.get("codeBlock", {})
|
||||||
|
page = style.get("page", {})
|
||||||
|
|
||||||
|
primaryFont = fonts.get("primary", "Arial, sans-serif")
|
||||||
|
monoFont = fonts.get("monospace", "Courier New, monospace")
|
||||||
|
bgColor = colors.get("background", "#FFFFFF")
|
||||||
|
primaryColor = colors.get("primary", "#1F3864")
|
||||||
|
paraColor = para.get("color", "#333333")
|
||||||
|
paraSizePt = para.get("sizePt", 11)
|
||||||
|
lineSpacing = para.get("lineSpacing", 1.15)
|
||||||
|
|
||||||
|
css_parts = []
|
||||||
|
|
||||||
|
# Body
|
||||||
|
css_parts.append("body {")
|
||||||
|
css_parts.append(f" font-family: {primaryFont};")
|
||||||
|
css_parts.append(f" background: {bgColor};")
|
||||||
|
css_parts.append(f" color: {paraColor};")
|
||||||
|
css_parts.append(f" font-size: {paraSizePt}pt;")
|
||||||
|
css_parts.append(f" line-height: {lineSpacing};")
|
||||||
|
margins = page.get("marginsPt", {})
|
||||||
|
if margins:
|
||||||
|
css_parts.append(f" margin: {margins.get('top', 60)}pt {margins.get('right', 60)}pt {margins.get('bottom', 60)}pt {margins.get('left', 60)}pt;")
|
||||||
|
else:
|
||||||
|
css_parts.append(" margin: 0; padding: 20px;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Document title (uses h1 style)
|
||||||
|
h1 = headings.get("h1", {})
|
||||||
|
css_parts.append(".document-title {")
|
||||||
|
css_parts.append(f" font-size: {h1.get('sizePt', 24)}pt;")
|
||||||
|
css_parts.append(f" color: {h1.get('color', primaryColor)};")
|
||||||
|
css_parts.append(f" font-weight: {h1.get('weight', 'bold')};")
|
||||||
|
css_parts.append(" margin: 0 0 1em 0;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Headings h1-h4
|
||||||
|
for level in range(1, 5):
|
||||||
|
key = f"h{level}"
|
||||||
|
h = headings.get(key, h1 if level == 1 else headings.get(f"h{level-1}", {}))
|
||||||
|
css_parts.append(f"h{level} {{")
|
||||||
|
css_parts.append(f" font-size: {h.get('sizePt', max(24 - (level-1)*4, 12))}pt;")
|
||||||
|
css_parts.append(f" color: {h.get('color', primaryColor)};")
|
||||||
|
css_parts.append(f" font-weight: {h.get('weight', 'bold')};")
|
||||||
|
css_parts.append(f" margin: 1.2em 0 0.4em 0;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Paragraphs
|
||||||
|
css_parts.append("p {")
|
||||||
|
css_parts.append(f" font-size: {paraSizePt}pt;")
|
||||||
|
css_parts.append(f" color: {paraColor};")
|
||||||
|
css_parts.append(f" line-height: {lineSpacing};")
|
||||||
|
css_parts.append(" margin: 0 0 1em 0;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Tables
|
||||||
|
borderColor = tbl.get("borderColor", "#DEE2E6")
|
||||||
|
css_parts.append("table {")
|
||||||
|
css_parts.append(f" border-collapse: collapse;")
|
||||||
|
css_parts.append(f" width: 100%;")
|
||||||
|
css_parts.append(f" margin: 1em 0;")
|
||||||
|
css_parts.append(f" border: 1px solid {borderColor};")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Table headers
|
||||||
|
css_parts.append("th {")
|
||||||
|
css_parts.append(f" background: {tbl.get('headerBg', '#1F3864')};")
|
||||||
|
css_parts.append(f" color: {tbl.get('headerFg', '#FFFFFF')};")
|
||||||
|
css_parts.append(" font-weight: bold;")
|
||||||
|
css_parts.append(" text-align: center;")
|
||||||
|
css_parts.append(f" padding: 10px;")
|
||||||
|
css_parts.append(f" border: 1px solid {borderColor};")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Table cells
|
||||||
|
css_parts.append("td {")
|
||||||
|
css_parts.append(f" color: {paraColor};")
|
||||||
|
css_parts.append(" padding: 8px;")
|
||||||
|
css_parts.append(f" border: 1px solid {borderColor};")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Lists
|
||||||
|
css_parts.append("ul {")
|
||||||
|
css_parts.append(f" font-size: {lst.get('sizePt', paraSizePt)}pt;")
|
||||||
|
css_parts.append(f" color: {paraColor};")
|
||||||
|
css_parts.append(f" padding-left: {lst.get('indentPt', 18)}pt;")
|
||||||
|
css_parts.append(" margin: 0 0 1em 0;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Code blocks
|
||||||
|
css_parts.append("pre {")
|
||||||
|
css_parts.append(f" font-family: {monoFont};")
|
||||||
|
css_parts.append(f" font-size: {cb.get('fontSizePt', 9)}pt;")
|
||||||
|
css_parts.append(f" color: {paraColor};")
|
||||||
|
css_parts.append(f" background: {cb.get('background', '#F8F9FA')};")
|
||||||
|
css_parts.append(f" border: 1px solid {cb.get('borderColor', '#E2E8F0')};")
|
||||||
|
css_parts.append(" border-radius: 4px;")
|
||||||
|
css_parts.append(" padding: 1em;")
|
||||||
|
css_parts.append(" margin: 1em 0;")
|
||||||
|
css_parts.append(" overflow-x: auto;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Images
|
||||||
|
css_parts.append("img {")
|
||||||
|
css_parts.append(" max-width: 100%;")
|
||||||
|
css_parts.append(" height: auto;")
|
||||||
|
css_parts.append(" margin: 1em 0;")
|
||||||
|
css_parts.append(" border-radius: 4px;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
# Generated info
|
||||||
|
css_parts.append(".generated-info {")
|
||||||
|
css_parts.append(" font-size: 0.9em;")
|
||||||
|
css_parts.append(" color: #666;")
|
||||||
|
css_parts.append(" text-align: center;")
|
||||||
|
css_parts.append(" margin-top: 2em;")
|
||||||
|
css_parts.append(" padding-top: 1em;")
|
||||||
|
css_parts.append(" border-top: 1px solid #ddd;")
|
||||||
|
css_parts.append("}")
|
||||||
|
|
||||||
|
return '\n'.join(css_parts)
|
||||||
|
|
||||||
|
def _renderInlineRuns(self, runs: list) -> str:
|
||||||
|
"""Convert inline runs to HTML markup."""
|
||||||
|
import html as htmlLib
|
||||||
|
parts = []
|
||||||
|
for run in runs:
|
||||||
|
runType = run.get("type", "text")
|
||||||
|
value = htmlLib.escape(run.get("value", ""))
|
||||||
|
if runType == "text":
|
||||||
|
parts.append(value)
|
||||||
|
elif runType == "bold":
|
||||||
|
parts.append(f"<strong>{value}</strong>")
|
||||||
|
elif runType == "italic":
|
||||||
|
parts.append(f"<em>{value}</em>")
|
||||||
|
elif runType == "code":
|
||||||
|
parts.append(f"<code>{value}</code>")
|
||||||
|
elif runType == "link":
|
||||||
|
href = htmlLib.escape(run.get("href", ""))
|
||||||
|
parts.append(f'<a href="{href}">{value}</a>')
|
||||||
|
elif runType == "image":
|
||||||
|
b64 = run.get("base64Data", "")
|
||||||
|
mime = run.get("mimeType", "image/png")
|
||||||
|
alt = value
|
||||||
|
if b64:
|
||||||
|
parts.append(f'<img src="data:{mime};base64,{b64}" alt="{alt}" style="max-width:100%;height:auto;">')
|
||||||
|
else:
|
||||||
|
parts.append(value)
|
||||||
|
return "".join(parts)
|
||||||
|
|
||||||
def _renderJsonSection(self, section: Dict[str, Any], styles: Dict[str, Any]) -> str:
|
def _renderJsonSection(self, section: Dict[str, Any], styles: Dict[str, Any]) -> str:
|
||||||
"""Render a single JSON section to HTML using AI-generated styles.
|
"""Render a single JSON section to HTML using AI-generated styles.
|
||||||
Supports three content formats: reference, object (base64), extracted_text.
|
Supports three content formats: reference, object (base64), extracted_text.
|
||||||
|
|
@ -419,6 +586,11 @@ class RendererHtml(BaseRenderer):
|
||||||
# Regular paragraph element - extract from nested content structure (standard JSON format)
|
# Regular paragraph element - extract from nested content structure (standard JSON format)
|
||||||
content = element.get("content", {})
|
content = element.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
|
# New format: inlineRuns
|
||||||
|
inlineRuns = content.get("inlineRuns")
|
||||||
|
if inlineRuns and isinstance(inlineRuns, list):
|
||||||
|
htmlParts.append(f'<p>{self._renderInlineRuns(inlineRuns)}</p>')
|
||||||
|
continue
|
||||||
text = content.get("text", "")
|
text = content.get("text", "")
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
|
|
@ -495,7 +667,8 @@ class RendererHtml(BaseRenderer):
|
||||||
# Table header
|
# Table header
|
||||||
htmlParts.append('<thead><tr>')
|
htmlParts.append('<thead><tr>')
|
||||||
for header in headers:
|
for header in headers:
|
||||||
htmlParts.append(f'<th>{header}</th>')
|
runs = self._inlineRunsForCell(header)
|
||||||
|
htmlParts.append(f'<th>{self._renderInlineRuns(runs)}</th>')
|
||||||
htmlParts.append('</tr></thead>')
|
htmlParts.append('</tr></thead>')
|
||||||
|
|
||||||
# Table body
|
# Table body
|
||||||
|
|
@ -503,7 +676,8 @@ class RendererHtml(BaseRenderer):
|
||||||
for row in rows:
|
for row in rows:
|
||||||
htmlParts.append('<tr>')
|
htmlParts.append('<tr>')
|
||||||
for cellData in row:
|
for cellData in row:
|
||||||
htmlParts.append(f'<td>{cellData}</td>')
|
runs = self._inlineRunsForCell(cellData)
|
||||||
|
htmlParts.append(f'<td>{self._renderInlineRuns(runs)}</td>')
|
||||||
htmlParts.append('</tr>')
|
htmlParts.append('</tr>')
|
||||||
htmlParts.append('</tbody>')
|
htmlParts.append('</tbody>')
|
||||||
|
|
||||||
|
|
@ -528,10 +702,8 @@ class RendererHtml(BaseRenderer):
|
||||||
|
|
||||||
htmlParts = ['<ul>']
|
htmlParts = ['<ul>']
|
||||||
for item in items:
|
for item in items:
|
||||||
if isinstance(item, str):
|
runs = self._inlineRunsForListItem(item)
|
||||||
htmlParts.append(f'<li>{item}</li>')
|
htmlParts.append(f'<li>{self._renderInlineRuns(runs)}</li>')
|
||||||
elif isinstance(item, dict) and "text" in item:
|
|
||||||
htmlParts.append(f'<li>{item["text"]}</li>')
|
|
||||||
htmlParts.append('</ul>')
|
htmlParts.append('</ul>')
|
||||||
|
|
||||||
return '\n'.join(htmlParts)
|
return '\n'.join(htmlParts)
|
||||||
|
|
@ -571,6 +743,11 @@ class RendererHtml(BaseRenderer):
|
||||||
if isinstance(el, dict):
|
if isinstance(el, dict):
|
||||||
content = el.get("content", {})
|
content = el.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
|
# New format: inlineRuns
|
||||||
|
inlineRuns = content.get("inlineRuns")
|
||||||
|
if inlineRuns and isinstance(inlineRuns, list):
|
||||||
|
texts.append(self._renderInlineRuns(inlineRuns))
|
||||||
|
continue
|
||||||
text = content.get("text", "")
|
text = content.get("text", "")
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
|
|
@ -581,16 +758,18 @@ class RendererHtml(BaseRenderer):
|
||||||
elif isinstance(el, str):
|
elif isinstance(el, str):
|
||||||
texts.append(el)
|
texts.append(el)
|
||||||
if texts:
|
if texts:
|
||||||
# Join multiple paragraphs with <p> tags
|
|
||||||
return '\n'.join(f'<p>{text}</p>' for text in texts)
|
return '\n'.join(f'<p>{text}</p>' for text in texts)
|
||||||
return ""
|
return ""
|
||||||
elif isinstance(paragraphData, str):
|
elif isinstance(paragraphData, str):
|
||||||
return f'<p>{paragraphData}</p>'
|
return f'<p>{paragraphData}</p>'
|
||||||
elif isinstance(paragraphData, dict):
|
elif isinstance(paragraphData, dict):
|
||||||
# Handle nested content structure: element.content vs element.text
|
|
||||||
# Extract from nested content structure
|
# Extract from nested content structure
|
||||||
content = paragraphData.get("content", {})
|
content = paragraphData.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
|
# New format: inlineRuns
|
||||||
|
inlineRuns = content.get("inlineRuns")
|
||||||
|
if inlineRuns and isinstance(inlineRuns, list):
|
||||||
|
return f'<p>{self._renderInlineRuns(inlineRuns)}</p>'
|
||||||
text = content.get("text", "")
|
text = content.get("text", "")
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
|
|
|
||||||
|
|
@ -106,17 +106,17 @@ class RendererPdf(BaseRenderer):
|
||||||
from modules.datamodels.datamodelJson import supportedSectionTypes
|
from modules.datamodels.datamodelJson import supportedSectionTypes
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""Render extracted JSON content to PDF format using AI-analyzed styling."""
|
"""Render extracted JSON content to PDF format using AI-analyzed styling."""
|
||||||
try:
|
try:
|
||||||
if not REPORTLAB_AVAILABLE:
|
if not REPORTLAB_AVAILABLE:
|
||||||
# Fallback to HTML if reportlab not available
|
# Fallback to HTML if reportlab not available
|
||||||
from .rendererHtml import RendererHtml
|
from .rendererHtml import RendererHtml
|
||||||
html_renderer = RendererHtml()
|
html_renderer = RendererHtml()
|
||||||
return await html_renderer.render(extractedContent, title, userPrompt, aiService)
|
return await html_renderer.render(extractedContent, title, userPrompt, aiService, style=style)
|
||||||
|
|
||||||
# Generate PDF using AI-analyzed styling
|
# Generate PDF using AI-analyzed styling
|
||||||
pdf_content = await self._generatePdfFromJson(extractedContent, title, userPrompt, aiService)
|
pdf_content = await self._generatePdfFromJson(extractedContent, title, userPrompt, aiService, unifiedStyle=style)
|
||||||
|
|
||||||
# Extract metadata for document type and other info
|
# Extract metadata for document type and other info
|
||||||
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
||||||
|
|
@ -163,11 +163,28 @@ class RendererPdf(BaseRenderer):
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
async def _generatePdfFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str:
|
async def _generatePdfFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, unifiedStyle: Dict[str, Any] = None) -> str:
|
||||||
"""Generate PDF content from structured JSON document using AI-generated styling."""
|
"""Generate PDF content from structured JSON document using AI-generated styling."""
|
||||||
try:
|
try:
|
||||||
# Get style set: use styles from metadata if available, otherwise enhance with AI
|
# Get style set from unified style or legacy approach
|
||||||
styles = await self._getStyleSet(json_content, userPrompt, aiService)
|
if unifiedStyle:
|
||||||
|
styles = self._convertUnifiedStyleToInternal(unifiedStyle)
|
||||||
|
self._unifiedStyle = unifiedStyle
|
||||||
|
for level in range(1, 7):
|
||||||
|
hKey = f"heading{level}"
|
||||||
|
if hKey not in styles:
|
||||||
|
styles[hKey] = self._defaultHeadingStyleDef(level)
|
||||||
|
else:
|
||||||
|
styles[hKey].setdefault("space_after", 12)
|
||||||
|
styles[hKey].setdefault("space_before", 12)
|
||||||
|
styles["paragraph"].setdefault("space_after", 6)
|
||||||
|
styles["paragraph"].setdefault("line_height", unifiedStyle["paragraph"].get("lineSpacing", 1.2))
|
||||||
|
styles["bullet_list"].setdefault("space_after", 3)
|
||||||
|
styles["code_block"].setdefault("space_after", 6)
|
||||||
|
styles["code_block"].setdefault("align", "left")
|
||||||
|
else:
|
||||||
|
styles = await self._getStyleSet(json_content, userPrompt, aiService)
|
||||||
|
self._unifiedStyle = None
|
||||||
|
|
||||||
# Validate JSON structure
|
# Validate JSON structure
|
||||||
if not self._validateJsonStructure(json_content):
|
if not self._validateJsonStructure(json_content):
|
||||||
|
|
@ -179,15 +196,13 @@ class RendererPdf(BaseRenderer):
|
||||||
# Create a buffer to hold the PDF
|
# Create a buffer to hold the PDF
|
||||||
buffer = io.BytesIO()
|
buffer = io.BytesIO()
|
||||||
|
|
||||||
# Create PDF document
|
# Create PDF document with unified page margins or defaults
|
||||||
doc = SimpleDocTemplate(
|
pageCfg = unifiedStyle["page"] if unifiedStyle else None
|
||||||
buffer,
|
if pageCfg:
|
||||||
pagesize=A4,
|
m = pageCfg["marginsPt"]
|
||||||
rightMargin=72,
|
doc = SimpleDocTemplate(buffer, pagesize=A4, rightMargin=m["right"], leftMargin=m["left"], topMargin=m["top"], bottomMargin=m["bottom"])
|
||||||
leftMargin=72,
|
else:
|
||||||
topMargin=72,
|
doc = SimpleDocTemplate(buffer, pagesize=A4, rightMargin=72, leftMargin=72, topMargin=72, bottomMargin=18)
|
||||||
bottomMargin=18
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build PDF content (no cover page — body starts on page 1; filename still uses `title`)
|
# Build PDF content (no cover page — body starts on page 1; filename still uses `title`)
|
||||||
story = []
|
story = []
|
||||||
|
|
@ -232,13 +247,28 @@ class RendererPdf(BaseRenderer):
|
||||||
removed = False
|
removed = False
|
||||||
for idx, flowable in enumerate(story):
|
for idx, flowable in enumerate(story):
|
||||||
fRepr = repr(flowable)
|
fRepr = repr(flowable)
|
||||||
|
if "Image" in fRepr and hasattr(flowable, 'drawWidth') and hasattr(flowable, 'drawHeight'):
|
||||||
|
from reportlab.platypus import Image as ReportLabImage
|
||||||
|
if isinstance(flowable, ReportLabImage):
|
||||||
|
frameH = 650.0
|
||||||
|
frameW = 450.0
|
||||||
|
if flowable.drawHeight > frameH or flowable.drawWidth > frameW:
|
||||||
|
scaleW = frameW / flowable.drawWidth if flowable.drawWidth > frameW else 1.0
|
||||||
|
scaleH = frameH / flowable.drawHeight if flowable.drawHeight > frameH else 1.0
|
||||||
|
s = min(scaleW, scaleH) * 0.9
|
||||||
|
flowable.drawWidth = flowable.drawWidth * s
|
||||||
|
flowable.drawHeight = flowable.drawHeight * s
|
||||||
|
flowable._width = flowable.drawWidth
|
||||||
|
flowable._height = flowable.drawHeight
|
||||||
|
removed = True
|
||||||
|
break
|
||||||
if "Table" in fRepr and hasattr(flowable, '_cellvalues'):
|
if "Table" in fRepr and hasattr(flowable, '_cellvalues'):
|
||||||
try:
|
try:
|
||||||
nRows = len(flowable._cellvalues)
|
nRows = len(flowable._cellvalues)
|
||||||
nCols = len(flowable._cellvalues[0]) if flowable._cellvalues else 0
|
nCols = len(flowable._cellvalues[0]) if flowable._cellvalues else 0
|
||||||
if nRows == 1 and nCols == 1:
|
if nRows == 1 and nCols == 1:
|
||||||
errPara = Paragraph(
|
errPara = Paragraph(
|
||||||
"[Code block omitted — content too large for PDF page]",
|
"[Code block omitted - content too large for PDF page]",
|
||||||
self._createNormalStyle({}),
|
self._createNormalStyle({}),
|
||||||
)
|
)
|
||||||
story[idx] = errPara
|
story[idx] = errPara
|
||||||
|
|
@ -609,6 +639,31 @@ class RendererPdf(BaseRenderer):
|
||||||
.replace(">", ">")
|
.replace(">", ">")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _renderInlineRunsToPdfXml(self, runs: list) -> str:
|
||||||
|
"""Convert inline runs to ReportLab Paragraph XML."""
|
||||||
|
parts = []
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
monoFont = us["fonts"]["monospace"] if us else "Courier"
|
||||||
|
for run in runs:
|
||||||
|
runType = run.get("type", "text")
|
||||||
|
value = self._escapeReportlabXml(run.get("value", ""))
|
||||||
|
if runType == "text":
|
||||||
|
parts.append(value)
|
||||||
|
elif runType == "bold":
|
||||||
|
parts.append(f"<b>{value}</b>")
|
||||||
|
elif runType == "italic":
|
||||||
|
parts.append(f"<i>{value}</i>")
|
||||||
|
elif runType == "code":
|
||||||
|
parts.append(f'<font name="{monoFont}">{value}</font>')
|
||||||
|
elif runType == "link":
|
||||||
|
href = self._escapeReportlabXml(run.get("href", ""))
|
||||||
|
parts.append(f'<a href="{href}">{value}</a>')
|
||||||
|
elif runType == "image":
|
||||||
|
parts.append(f"[Image: {value}]")
|
||||||
|
else:
|
||||||
|
parts.append(value)
|
||||||
|
return "".join(parts)
|
||||||
|
|
||||||
def _applyInlineMarkdownToEscapedPlain(self, text: str) -> str:
|
def _applyInlineMarkdownToEscapedPlain(self, text: str) -> str:
|
||||||
"""Escape XML then apply bold/italic to a segment with no `code` spans (code is handled separately)."""
|
"""Escape XML then apply bold/italic to a segment with no `code` spans (code is handled separately)."""
|
||||||
if not text:
|
if not text:
|
||||||
|
|
@ -755,8 +810,12 @@ class RendererPdf(BaseRenderer):
|
||||||
hdrPs = self._createTableCellParagraphStyle(styles, header=True, tableStyleKey="table_header")
|
hdrPs = self._createTableCellParagraphStyle(styles, header=True, tableStyleKey="table_header")
|
||||||
cellPs = self._createTableCellParagraphStyle(styles, header=False, tableStyleKey="table_cell")
|
cellPs = self._createTableCellParagraphStyle(styles, header=False, tableStyleKey="table_cell")
|
||||||
|
|
||||||
def _cellPara(val, ps):
|
def _cellPara(cell, ps):
|
||||||
return self._paragraphFromInlineMarkdown(str(val) if val is not None else "", ps)
|
runs = self._inlineRunsForCell(cell)
|
||||||
|
if isinstance(cell, list):
|
||||||
|
xml = self._renderInlineRunsToPdfXml(runs)
|
||||||
|
return Paragraph(_wrapEmojiSpansInXml(xml), ps)
|
||||||
|
return self._paragraphFromInlineMarkdown(str(cell) if cell is not None else "", ps)
|
||||||
|
|
||||||
headerRow = [_cellPara(h, hdrPs) for h in headers]
|
headerRow = [_cellPara(h, hdrPs) for h in headers]
|
||||||
bodyRows = []
|
bodyRows = []
|
||||||
|
|
@ -794,29 +853,26 @@ class RendererPdf(BaseRenderer):
|
||||||
def _renderJsonBulletList(self, list_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]:
|
def _renderJsonBulletList(self, list_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]:
|
||||||
"""Render a JSON bullet list to PDF elements using AI-generated styles."""
|
"""Render a JSON bullet list to PDF elements using AI-generated styles."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = list_data.get("content", {})
|
content = list_data.get("content", {})
|
||||||
if not isinstance(content, dict):
|
if not isinstance(content, dict):
|
||||||
return []
|
return []
|
||||||
items = content.get("items", [])
|
items = content.get("items", [])
|
||||||
bullet_style_def = styles.get("bullet_list", {})
|
bulletStyleDef = styles.get("bullet_list", {})
|
||||||
|
normalStyle = self._createNormalStyle(styles)
|
||||||
|
|
||||||
elements = []
|
elements = []
|
||||||
for item in items:
|
for item in items:
|
||||||
if isinstance(item, str):
|
runs = self._inlineRunsForListItem(item)
|
||||||
elements.append(
|
if isinstance(item, list):
|
||||||
Paragraph(f"• {self._markdownInlineToReportlabXml(item)}", self._createNormalStyle(styles))
|
xml = self._renderInlineRunsToPdfXml(runs)
|
||||||
)
|
elements.append(Paragraph(f"\u2022 {_wrapEmojiSpansInXml(xml)}", normalStyle))
|
||||||
|
elif isinstance(item, str):
|
||||||
|
elements.append(Paragraph(f"\u2022 {self._markdownInlineToReportlabXml(item)}", normalStyle))
|
||||||
elif isinstance(item, dict) and "text" in item:
|
elif isinstance(item, dict) and "text" in item:
|
||||||
elements.append(
|
elements.append(Paragraph(f"\u2022 {self._markdownInlineToReportlabXml(item['text'])}", normalStyle))
|
||||||
Paragraph(
|
|
||||||
f"• {self._markdownInlineToReportlabXml(item['text'])}",
|
|
||||||
self._createNormalStyle(styles),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if elements:
|
if elements:
|
||||||
elements.append(Spacer(1, bullet_style_def.get("space_after", 3)))
|
elements.append(Spacer(1, bulletStyleDef.get("space_after", 3)))
|
||||||
|
|
||||||
return elements
|
return elements
|
||||||
|
|
||||||
|
|
@ -848,17 +904,24 @@ class RendererPdf(BaseRenderer):
|
||||||
def _renderJsonParagraph(self, paragraph_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]:
|
def _renderJsonParagraph(self, paragraph_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]:
|
||||||
"""Render a JSON paragraph to PDF elements using AI-generated styles."""
|
"""Render a JSON paragraph to PDF elements using AI-generated styles."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = paragraph_data.get("content", {})
|
content = paragraph_data.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, str):
|
||||||
text = content.get("text", "")
|
content = {"text": content}
|
||||||
elif isinstance(content, str):
|
if not isinstance(content, dict):
|
||||||
text = content
|
return []
|
||||||
else:
|
|
||||||
text = ""
|
|
||||||
|
|
||||||
|
normalStyle = self._createNormalStyle(styles)
|
||||||
|
|
||||||
|
if "inlineRuns" in content:
|
||||||
|
runs = self._inlineRunsFromContent(content)
|
||||||
|
xml = self._renderInlineRunsToPdfXml(runs)
|
||||||
|
if xml:
|
||||||
|
return [Paragraph(_wrapEmojiSpansInXml(xml), normalStyle)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
text = content.get("text", "")
|
||||||
if text:
|
if text:
|
||||||
return [self._paragraphFromInlineMarkdown(text, self._createNormalStyle(styles))]
|
return [self._paragraphFromInlineMarkdown(text, normalStyle)]
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
@ -1030,20 +1093,18 @@ class RendererPdf(BaseRenderer):
|
||||||
pilImage = PILImage.open(imageStream)
|
pilImage = PILImage.open(imageStream)
|
||||||
originalWidth, originalHeight = pilImage.size
|
originalWidth, originalHeight = pilImage.size
|
||||||
|
|
||||||
# Calculate available page dimensions (A4 with margins: 72pt left/right, 72pt top, 18pt bottom)
|
|
||||||
pageWidth = A4[0] # 595.27 points
|
pageWidth = A4[0] # 595.27 points
|
||||||
pageHeight = A4[1] # 841.89 points
|
pageHeight = A4[1] # 841.89 points
|
||||||
leftMargin = 72
|
# Use page dimensions minus margins with generous safety buffer
|
||||||
rightMargin = 72
|
# A4 = 595.27 x 841.89 pt; frame = page - margins - internal padding
|
||||||
topMargin = 72
|
_us = getattr(self, '_unifiedStyle', None) or {}
|
||||||
bottomMargin = 18
|
_pageMgn = (_us.get('page') or {}).get('marginsPt') or {}
|
||||||
|
marginTop = _pageMgn.get('top', 60)
|
||||||
# Use actual frame dimensions from SimpleDocTemplate
|
marginBottom = _pageMgn.get('bottom', 60)
|
||||||
# Frame is smaller than page minus margins due to internal spacing
|
marginLeft = _pageMgn.get('left', 60)
|
||||||
# From error message: frame is 439.27559055118115 x 739.8897637795277
|
marginRight = _pageMgn.get('right', 60)
|
||||||
# Use conservative values with safety margin
|
availableWidth = pageWidth - marginLeft - marginRight - 20 # 20pt safety
|
||||||
availableWidth = 430.0 # Slightly smaller than frame width for safety
|
availableHeight = pageHeight - marginTop - marginBottom - 80 # 80pt safety for header/footer
|
||||||
availableHeight = 730.0 # Slightly smaller than frame height for safety
|
|
||||||
|
|
||||||
# Convert original image size from pixels to points
|
# Convert original image size from pixels to points
|
||||||
# PIL provides size in pixels, need to convert to points
|
# PIL provides size in pixels, need to convert to points
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ class RendererPptx(BaseRenderer):
|
||||||
from modules.datamodels.datamodelJson import supportedSectionTypes
|
from modules.datamodels.datamodelJson import supportedSectionTypes
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""
|
"""
|
||||||
Render content as PowerPoint presentation from JSON data.
|
Render content as PowerPoint presentation from JSON data.
|
||||||
|
|
||||||
|
|
@ -68,7 +68,7 @@ class RendererPptx(BaseRenderer):
|
||||||
title: Title for the presentation
|
title: Title for the presentation
|
||||||
userPrompt: User prompt for AI styling
|
userPrompt: User prompt for AI styling
|
||||||
aiService: AI service for styling
|
aiService: AI service for styling
|
||||||
**kwargs: Additional rendering options
|
style: Unified style dict from pipeline (preferred over AI-generated styles)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Base64-encoded PowerPoint presentation as string
|
Base64-encoded PowerPoint presentation as string
|
||||||
|
|
@ -81,8 +81,19 @@ class RendererPptx(BaseRenderer):
|
||||||
from pptx.dml.color import RGBColor
|
from pptx.dml.color import RGBColor
|
||||||
import re
|
import re
|
||||||
|
|
||||||
# Get style set: use styles from metadata if available, otherwise enhance with AI
|
# Get style set: prefer unified style, then metadata, then AI-enhanced
|
||||||
styles = await self._getStyleSet(extractedContent, userPrompt, aiService)
|
if style:
|
||||||
|
internalStyle = self._convertUnifiedStyleToInternal(style)
|
||||||
|
defaultPptx = self._getDefaultStyleSet()
|
||||||
|
for key in ("slide_size", "content_per_slide", "design_theme", "color_scheme", "background_style", "accent_colors", "professional_grade", "executive_ready"):
|
||||||
|
internalStyle[key] = defaultPptx.get(key)
|
||||||
|
internalStyle["heading"] = internalStyle["heading1"]
|
||||||
|
internalStyle["subheading"] = internalStyle["heading2"]
|
||||||
|
styles = internalStyle
|
||||||
|
self._unifiedStyle = style
|
||||||
|
else:
|
||||||
|
styles = await self._getStyleSet(extractedContent, userPrompt, aiService)
|
||||||
|
self._unifiedStyle = None
|
||||||
|
|
||||||
# Create new presentation
|
# Create new presentation
|
||||||
prs = Presentation()
|
prs = Presentation()
|
||||||
|
|
@ -910,15 +921,17 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
# Extract from nested content structure
|
# Extract from nested content structure
|
||||||
content = paragraph_data.get("content", {})
|
content = paragraph_data.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
text = content.get("text", "")
|
if content.get("inlineRuns"):
|
||||||
|
text = "".join(r.get("value", "") for r in content["inlineRuns"])
|
||||||
|
else:
|
||||||
|
text = content.get("text", "")
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
else:
|
else:
|
||||||
text = ""
|
text = ""
|
||||||
|
|
||||||
if text:
|
if text:
|
||||||
# Limit paragraph length based on content density
|
max_length = 200
|
||||||
max_length = 200 # Default limit
|
|
||||||
if len(text) > max_length:
|
if len(text) > max_length:
|
||||||
text = text[:max_length] + "..."
|
text = text[:max_length] + "..."
|
||||||
|
|
||||||
|
|
@ -1303,6 +1316,32 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
r.text = text[pos:]
|
r.text = text[pos:]
|
||||||
_applyBase(r)
|
_applyBase(r)
|
||||||
|
|
||||||
|
def _renderInlineRunsPptx(self, runs, paragraph, fontSize=None, fontColor=None):
|
||||||
|
"""Process InlineRun dicts into pptx text runs."""
|
||||||
|
from pptx.util import Pt
|
||||||
|
paragraph.text = ""
|
||||||
|
us = getattr(self, '_unifiedStyle', None)
|
||||||
|
monoFont = us["fonts"]["monospace"] if us else "Courier New"
|
||||||
|
for run in runs:
|
||||||
|
runType = run.get("type", "text")
|
||||||
|
value = run.get("value", "")
|
||||||
|
r = paragraph.add_run()
|
||||||
|
r.text = value
|
||||||
|
if fontSize:
|
||||||
|
r.font.size = fontSize
|
||||||
|
if fontColor:
|
||||||
|
r.font.color.rgb = fontColor
|
||||||
|
if runType == "bold":
|
||||||
|
r.font.bold = True
|
||||||
|
elif runType == "italic":
|
||||||
|
r.font.italic = True
|
||||||
|
elif runType == "code":
|
||||||
|
r.font.name = monoFont
|
||||||
|
if fontSize and hasattr(fontSize, 'pt'):
|
||||||
|
r.font.size = Pt(max(8, int(fontSize.pt * 0.85)))
|
||||||
|
elif runType == "link":
|
||||||
|
r.font.underline = True
|
||||||
|
|
||||||
def _addTableToSlide(self, slide, element: Dict[str, Any], styles: Dict[str, Any], top: float = None, max_width: float = None) -> None:
|
def _addTableToSlide(self, slide, element: Dict[str, Any], styles: Dict[str, Any], top: float = None, max_width: float = None) -> None:
|
||||||
"""Add a PowerPoint table to slide."""
|
"""Add a PowerPoint table to slide."""
|
||||||
try:
|
try:
|
||||||
|
|
@ -1374,7 +1413,8 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
cell = table.cell(0, col_idx)
|
cell = table.cell(0, col_idx)
|
||||||
# Clear existing text and set new text
|
# Clear existing text and set new text
|
||||||
cell.text_frame.clear()
|
cell.text_frame.clear()
|
||||||
header_text = str(header) if header else ""
|
cellRuns = self._inlineRunsForCell(header)
|
||||||
|
header_text = "".join(r.get("value", "") for r in cellRuns)
|
||||||
cell.text = header_text
|
cell.text = header_text
|
||||||
|
|
||||||
# Ensure paragraph exists
|
# Ensure paragraph exists
|
||||||
|
|
@ -1420,7 +1460,8 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
cell = table.cell(row_idx, col_idx)
|
cell = table.cell(row_idx, col_idx)
|
||||||
# Clear existing text and set new text
|
# Clear existing text and set new text
|
||||||
cell.text_frame.clear()
|
cell.text_frame.clear()
|
||||||
cell_text = str(cell_data) if cell_data is not None else ""
|
cellRuns = self._inlineRunsForCell(cell_data)
|
||||||
|
cell_text = "".join(r.get("value", "") for r in cellRuns)
|
||||||
cell.text = cell_text
|
cell.text = cell_text
|
||||||
|
|
||||||
# Ensure paragraph exists
|
# Ensure paragraph exists
|
||||||
|
|
@ -1462,9 +1503,8 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
fontColor = RGBColor(*self._getSafeColor(listStyle.get("color", (47, 47, 47))))
|
fontColor = RGBColor(*self._getSafeColor(listStyle.get("color", (47, 47, 47))))
|
||||||
|
|
||||||
for item in items:
|
for item in items:
|
||||||
itemText = item.get("text", "") if isinstance(item, dict) else str(item)
|
runs = self._inlineRunsForListItem(item)
|
||||||
if not itemText or not itemText.strip():
|
isNewFormat = isinstance(item, list)
|
||||||
continue
|
|
||||||
|
|
||||||
p = text_frame.add_paragraph()
|
p = text_frame.add_paragraph()
|
||||||
p.level = 0
|
p.level = 0
|
||||||
|
|
@ -1472,21 +1512,33 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
p.space_before = Pt(2)
|
p.space_before = Pt(2)
|
||||||
p.space_after = Pt(2)
|
p.space_after = Pt(2)
|
||||||
|
|
||||||
# Consistent bullet prefix
|
if isNewFormat:
|
||||||
self._addMarkdownInlineRuns(p, f" • {itemText}", fontSize=fontSize, fontColor=fontColor, fontBold=False)
|
bulletRuns = [{"type": "text", "value": " \u2022 "}] + runs
|
||||||
|
self._renderInlineRunsPptx(bulletRuns, p, fontSize=fontSize, fontColor=fontColor)
|
||||||
|
else:
|
||||||
|
itemText = item.get("text", "") if isinstance(item, dict) else str(item)
|
||||||
|
if not itemText or not itemText.strip():
|
||||||
|
continue
|
||||||
|
self._addMarkdownInlineRuns(p, f" \u2022 {itemText}", fontSize=fontSize, fontColor=fontColor, fontBold=False)
|
||||||
|
|
||||||
# Subitems
|
# Subitems (only for dict-style items)
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
for sub in item.get("subitems", []):
|
for sub in item.get("subitems", []):
|
||||||
subText = sub.get("text", "") if isinstance(sub, dict) else str(sub)
|
subRuns = self._inlineRunsForListItem(sub)
|
||||||
if not subText:
|
isSubNew = isinstance(sub, list)
|
||||||
continue
|
|
||||||
sp = text_frame.add_paragraph()
|
sp = text_frame.add_paragraph()
|
||||||
sp.level = 0
|
sp.level = 0
|
||||||
sp.alignment = PP_ALIGN.LEFT
|
sp.alignment = PP_ALIGN.LEFT
|
||||||
sp.space_before = Pt(1)
|
sp.space_before = Pt(1)
|
||||||
sp.space_after = Pt(1)
|
sp.space_after = Pt(1)
|
||||||
self._addMarkdownInlineRuns(sp, f" – {subText}", fontSize=fontSize, fontColor=fontColor, fontBold=False)
|
if isSubNew:
|
||||||
|
subBulletRuns = [{"type": "text", "value": " \u2013 "}] + subRuns
|
||||||
|
self._renderInlineRunsPptx(subBulletRuns, sp, fontSize=fontSize, fontColor=fontColor)
|
||||||
|
else:
|
||||||
|
subText = sub.get("text", "") if isinstance(sub, dict) else str(sub)
|
||||||
|
if not subText:
|
||||||
|
continue
|
||||||
|
self._addMarkdownInlineRuns(sp, f" \u2013 {subText}", fontSize=fontSize, fontColor=fontColor, fontBold=False)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Error adding bullet list to slide: {str(e)}")
|
logger.warning(f"Error adding bullet list to slide: {str(e)}")
|
||||||
|
|
@ -1540,42 +1592,53 @@ JSON ONLY. NO OTHER TEXT."""
|
||||||
# Extract from nested content structure
|
# Extract from nested content structure
|
||||||
content = element.get("content", {})
|
content = element.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
|
inlineRuns = self._inlineRunsFromContent(content)
|
||||||
|
hasInlineRuns = content.get("inlineRuns") is not None
|
||||||
text = content.get("text", "")
|
text = content.get("text", "")
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
|
inlineRuns = [{"type": "text", "value": text}] if text else []
|
||||||
|
hasInlineRuns = False
|
||||||
else:
|
else:
|
||||||
text = ""
|
text = ""
|
||||||
|
inlineRuns = []
|
||||||
|
hasInlineRuns = False
|
||||||
|
|
||||||
if text:
|
if not inlineRuns and not text:
|
||||||
p = text_frame.add_paragraph()
|
return
|
||||||
p.level = 0
|
|
||||||
|
|
||||||
try:
|
p = text_frame.add_paragraph()
|
||||||
if hasattr(p, 'paragraph_format'):
|
p.level = 0
|
||||||
p.paragraph_format.bullet.type = None
|
|
||||||
except (AttributeError, TypeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
paragraph_style = styles.get("paragraph", {})
|
try:
|
||||||
base_font_size = paragraph_style.get("font_size", 14)
|
if hasattr(p, 'paragraph_format'):
|
||||||
calculated_size = max(10, int(base_font_size * font_size_multiplier))
|
p.paragraph_format.bullet.type = None
|
||||||
fSize = Pt(calculated_size)
|
except (AttributeError, TypeError):
|
||||||
fColor = RGBColor(*self._getSafeColor(paragraph_style.get("color", (47, 47, 47))))
|
pass
|
||||||
fBold = paragraph_style.get("bold", False)
|
|
||||||
|
paragraph_style = styles.get("paragraph", {})
|
||||||
|
base_font_size = paragraph_style.get("font_size", 14)
|
||||||
|
calculated_size = max(10, int(base_font_size * font_size_multiplier))
|
||||||
|
fSize = Pt(calculated_size)
|
||||||
|
fColor = RGBColor(*self._getSafeColor(paragraph_style.get("color", (47, 47, 47))))
|
||||||
|
fBold = paragraph_style.get("bold", False)
|
||||||
|
|
||||||
|
if hasInlineRuns:
|
||||||
|
self._renderInlineRunsPptx(inlineRuns, p, fontSize=fSize, fontColor=fColor)
|
||||||
|
else:
|
||||||
self._addMarkdownInlineRuns(p, text, fontSize=fSize, fontColor=fColor, fontBold=fBold)
|
self._addMarkdownInlineRuns(p, text, fontSize=fSize, fontColor=fColor, fontBold=fBold)
|
||||||
|
|
||||||
# Add proper spacing
|
p.space_before = Pt(6)
|
||||||
p.space_before = Pt(6) # Space before paragraph
|
p.space_after = Pt(6)
|
||||||
p.space_after = Pt(6) # Space after paragraph
|
p.line_spacing = 1.2
|
||||||
p.line_spacing = 1.2 # Line spacing for readability
|
|
||||||
|
|
||||||
align = paragraph_style.get("align", "left")
|
align = paragraph_style.get("align", "left")
|
||||||
if align == "center":
|
if align == "center":
|
||||||
p.alignment = PP_ALIGN.CENTER
|
p.alignment = PP_ALIGN.CENTER
|
||||||
elif align == "right":
|
elif align == "right":
|
||||||
p.alignment = PP_ALIGN.RIGHT
|
p.alignment = PP_ALIGN.RIGHT
|
||||||
else:
|
else:
|
||||||
p.alignment = PP_ALIGN.LEFT
|
p.alignment = PP_ALIGN.LEFT
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Error adding paragraph to slide: {str(e)}")
|
logger.warning(f"Error adding paragraph to slide: {str(e)}")
|
||||||
|
|
|
||||||
|
|
@ -68,17 +68,17 @@ class RendererXlsx(BaseRenderer):
|
||||||
from modules.datamodels.datamodelJson import supportedSectionTypes
|
from modules.datamodels.datamodelJson import supportedSectionTypes
|
||||||
return list(supportedSectionTypes)
|
return list(supportedSectionTypes)
|
||||||
|
|
||||||
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]:
|
||||||
"""Render extracted JSON content to Excel format using AI-analyzed styling."""
|
"""Render extracted JSON content to Excel format using AI-analyzed styling."""
|
||||||
try:
|
try:
|
||||||
if not OPENPYXL_AVAILABLE:
|
if not OPENPYXL_AVAILABLE:
|
||||||
# Fallback to CSV if openpyxl not available
|
# Fallback to CSV if openpyxl not available
|
||||||
from .rendererCsv import RendererCsv
|
from .rendererCsv import RendererCsv
|
||||||
csvRenderer = RendererCsv()
|
csvRenderer = RendererCsv()
|
||||||
return await csvRenderer.render(extractedContent, title, userPrompt, aiService)
|
return await csvRenderer.render(extractedContent, title, userPrompt, aiService, style=style)
|
||||||
|
|
||||||
# Generate Excel using AI-analyzed styling
|
# Generate Excel using AI-analyzed styling
|
||||||
excelContent = await self._generateExcelFromJson(extractedContent, title, userPrompt, aiService)
|
excelContent = await self._generateExcelFromJson(extractedContent, title, userPrompt, aiService, style=style)
|
||||||
|
|
||||||
# Extract metadata for document type and other info
|
# Extract metadata for document type and other info
|
||||||
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
metadata = extractedContent.get("metadata", {}) if extractedContent else {}
|
||||||
|
|
@ -298,15 +298,22 @@ class RendererXlsx(BaseRenderer):
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Could not populate analysis sheet: {str(e)}")
|
self.logger.warning(f"Could not populate analysis sheet: {str(e)}")
|
||||||
|
|
||||||
async def _generateExcelFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str:
|
async def _generateExcelFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> str:
|
||||||
"""Generate Excel content from structured JSON document using AI-generated styling."""
|
"""Generate Excel content from structured JSON document using AI-generated styling."""
|
||||||
try:
|
try:
|
||||||
# Debug output
|
# Debug output
|
||||||
self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT TYPE: {type(jsonContent)}", "EXCEL_RENDERER")
|
self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT TYPE: {type(jsonContent)}", "EXCEL_RENDERER")
|
||||||
self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(jsonContent.keys()) if isinstance(jsonContent, dict) else 'Not a dict'}", "EXCEL_RENDERER")
|
self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(jsonContent.keys()) if isinstance(jsonContent, dict) else 'Not a dict'}", "EXCEL_RENDERER")
|
||||||
|
|
||||||
# Get style set: use styles from metadata if available, otherwise enhance with AI
|
# Store unified style for use by inline-run helpers
|
||||||
styles = await self._getStyleSet(jsonContent, userPrompt, aiService)
|
self._unifiedStyle = style
|
||||||
|
|
||||||
|
# Get style set: prefer unified style, fall back to legacy approach
|
||||||
|
if style:
|
||||||
|
styles = self._convertUnifiedStyleToInternal(style)
|
||||||
|
styles = self._convertColorsFormat(styles)
|
||||||
|
else:
|
||||||
|
styles = await self._getStyleSet(jsonContent, userPrompt, aiService)
|
||||||
|
|
||||||
# Validate JSON structure (standardized schema: {metadata: {...}, documents: [{sections: [...]}]})
|
# Validate JSON structure (standardized schema: {metadata: {...}, documents: [{sections: [...]}]})
|
||||||
if not self._validateJsonStructure(jsonContent):
|
if not self._validateJsonStructure(jsonContent):
|
||||||
|
|
@ -511,6 +518,10 @@ class RendererXlsx(BaseRenderer):
|
||||||
"code_block": {"font": "Courier New", "font_size": 10, "color": "FF2F2F2F", "background": "FFF5F5F5"}
|
"code_block": {"font": "Courier New", "font_size": 10, "color": "FF2F2F2F", "background": "FFF5F5F5"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _renderInlineRuns(self, runs: list) -> str:
|
||||||
|
"""Flatten inline runs to plain text for Excel cells."""
|
||||||
|
return "".join(r.get("value", "") for r in runs)
|
||||||
|
|
||||||
async def _getAiStylesWithExcelColors(self, aiService, styleTemplate: str, defaultStyles: Dict[str, Any]) -> Dict[str, Any]:
|
async def _getAiStylesWithExcelColors(self, aiService, styleTemplate: str, defaultStyles: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
"""Get AI styles with proper Excel color conversion."""
|
"""Get AI styles with proper Excel color conversion."""
|
||||||
if not aiService:
|
if not aiService:
|
||||||
|
|
@ -1206,7 +1217,9 @@ class RendererXlsx(BaseRenderer):
|
||||||
|
|
||||||
# Add headers with formatting - OPTIMIZED: use cached style objects
|
# Add headers with formatting - OPTIMIZED: use cached style objects
|
||||||
for col, header in enumerate(headers, 1):
|
for col, header in enumerate(headers, 1):
|
||||||
sanitized_header = self._sanitizeCellValue(header)
|
runs = self._inlineRunsForCell(header)
|
||||||
|
headerText = self._renderInlineRuns(runs)
|
||||||
|
sanitized_header = self._sanitizeCellValue(headerText)
|
||||||
cell = sheet.cell(row=headerRow, column=col, value=sanitized_header)
|
cell = sheet.cell(row=headerRow, column=col, value=sanitized_header)
|
||||||
|
|
||||||
# Apply styling with fallbacks - use pre-calculated objects
|
# Apply styling with fallbacks - use pre-calculated objects
|
||||||
|
|
@ -1272,7 +1285,9 @@ class RendererXlsx(BaseRenderer):
|
||||||
cell_values = cell_values[:header_count]
|
cell_values = cell_values[:header_count]
|
||||||
|
|
||||||
for col, cell_value in enumerate(cell_values, 1):
|
for col, cell_value in enumerate(cell_values, 1):
|
||||||
sanitized_value = self._sanitizeCellValue(cell_value)
|
runs = self._inlineRunsForCell(cell_value)
|
||||||
|
cellText = self._renderInlineRuns(runs)
|
||||||
|
sanitized_value = self._sanitizeCellValue(cellText)
|
||||||
cell = sheet.cell(row=startRow, column=col, value=sanitized_value)
|
cell = sheet.cell(row=startRow, column=col, value=sanitized_value)
|
||||||
|
|
||||||
# Apply styling with fallbacks - use pre-calculated objects
|
# Apply styling with fallbacks - use pre-calculated objects
|
||||||
|
|
@ -1311,20 +1326,20 @@ class RendererXlsx(BaseRenderer):
|
||||||
def _addListToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int:
|
def _addListToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int:
|
||||||
"""Add a list element to Excel sheet. Expects nested content structure."""
|
"""Add a list element to Excel sheet. Expects nested content structure."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = element.get("content", {})
|
content = element.get("content", {})
|
||||||
if not isinstance(content, dict):
|
if not isinstance(content, dict):
|
||||||
return startRow
|
return startRow
|
||||||
list_items = content.get("items") or []
|
listItems = content.get("items") or []
|
||||||
# Ensure list_items is a list
|
if not isinstance(listItems, list):
|
||||||
if not isinstance(list_items, list):
|
listItems = []
|
||||||
list_items = []
|
|
||||||
|
|
||||||
list_style = styles.get("bullet_list", {})
|
listStyle = styles.get("bullet_list", {})
|
||||||
for item in list_items:
|
for item in listItems:
|
||||||
sheet.cell(row=startRow, column=1, value=f"• {item}")
|
runs = self._inlineRunsForListItem(item)
|
||||||
if list_style.get("color"):
|
text = self._renderInlineRuns(runs)
|
||||||
sheet.cell(row=startRow, column=1).font = Font(color=self._getSafeColor(list_style["color"]))
|
sheet.cell(row=startRow, column=1, value=f"\u2022 {text}")
|
||||||
|
if listStyle.get("color"):
|
||||||
|
sheet.cell(row=startRow, column=1).font = Font(color=self._getSafeColor(listStyle["color"]))
|
||||||
startRow += 1
|
startRow += 1
|
||||||
|
|
||||||
return startRow
|
return startRow
|
||||||
|
|
@ -1336,10 +1351,10 @@ class RendererXlsx(BaseRenderer):
|
||||||
def _addParagraphToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int:
|
def _addParagraphToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int:
|
||||||
"""Add a paragraph element to Excel sheet. Expects nested content structure."""
|
"""Add a paragraph element to Excel sheet. Expects nested content structure."""
|
||||||
try:
|
try:
|
||||||
# Extract from nested content structure
|
|
||||||
content = element.get("content", {})
|
content = element.get("content", {})
|
||||||
if isinstance(content, dict):
|
if isinstance(content, dict):
|
||||||
text = content.get("text", "")
|
runs = self._inlineRunsFromContent(content)
|
||||||
|
text = self._renderInlineRuns(runs)
|
||||||
elif isinstance(content, str):
|
elif isinstance(content, str):
|
||||||
text = content
|
text = content
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""Default style definitions and style resolution for document rendering."""
|
||||||
|
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_STYLE: Dict[str, Any] = {
|
||||||
|
"fonts": {
|
||||||
|
"primary": "Calibri",
|
||||||
|
"monospace": "Consolas",
|
||||||
|
},
|
||||||
|
"colors": {
|
||||||
|
"primary": "#1F3864",
|
||||||
|
"secondary": "#2C3E50",
|
||||||
|
"accent": "#2980B9",
|
||||||
|
"background": "#FFFFFF",
|
||||||
|
},
|
||||||
|
"headings": {
|
||||||
|
"h1": {"sizePt": 24, "weight": "bold", "color": "#1F3864", "spaceBeforePt": 12, "spaceAfterPt": 6},
|
||||||
|
"h2": {"sizePt": 18, "weight": "bold", "color": "#1F3864", "spaceBeforePt": 10, "spaceAfterPt": 4},
|
||||||
|
"h3": {"sizePt": 14, "weight": "bold", "color": "#2C3E50", "spaceBeforePt": 8, "spaceAfterPt": 3},
|
||||||
|
"h4": {"sizePt": 12, "weight": "bold", "color": "#2C3E50", "spaceBeforePt": 6, "spaceAfterPt": 2},
|
||||||
|
},
|
||||||
|
"paragraph": {"sizePt": 11, "lineSpacing": 1.15, "color": "#333333"},
|
||||||
|
"table": {
|
||||||
|
"headerBg": "#1F3864",
|
||||||
|
"headerFg": "#FFFFFF",
|
||||||
|
"headerSizePt": 10,
|
||||||
|
"bodySizePt": 10,
|
||||||
|
"rowBandingEven": "#F2F6FC",
|
||||||
|
"rowBandingOdd": "#FFFFFF",
|
||||||
|
"borderColor": "#CBD5E1",
|
||||||
|
"borderWidthPt": 0.5,
|
||||||
|
},
|
||||||
|
"list": {"bulletChar": "\u2022", "indentPt": 18, "sizePt": 11},
|
||||||
|
"image": {"defaultWidthPt": 480, "maxWidthPt": 800, "alignment": "center"},
|
||||||
|
"codeBlock": {"fontSizePt": 9, "background": "#F8F9FA", "borderColor": "#E2E8F0"},
|
||||||
|
"page": {
|
||||||
|
"format": "A4",
|
||||||
|
"marginsPt": {"top": 60, "bottom": 60, "left": 60, "right": 60},
|
||||||
|
"showPageNumbers": True,
|
||||||
|
"headerHeight": 30,
|
||||||
|
"footerHeight": 30,
|
||||||
|
"headerLogo": None,
|
||||||
|
"headerText": "",
|
||||||
|
"footerText": "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _deepMerge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Recursively merge override into base. Both dicts left unchanged; returns new dict."""
|
||||||
|
result = {}
|
||||||
|
for key in base:
|
||||||
|
if key in override:
|
||||||
|
baseVal = base[key]
|
||||||
|
overVal = override[key]
|
||||||
|
if isinstance(baseVal, dict) and isinstance(overVal, dict):
|
||||||
|
result[key] = _deepMerge(baseVal, overVal)
|
||||||
|
else:
|
||||||
|
result[key] = overVal
|
||||||
|
else:
|
||||||
|
result[key] = base[key]
|
||||||
|
for key in override:
|
||||||
|
if key not in base:
|
||||||
|
result[key] = override[key]
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def resolveStyle(agentStyle: dict | None) -> Dict[str, Any]:
|
||||||
|
"""Deep-merge DEFAULT_STYLE <- agentStyle. Returns fully resolved style dict."""
|
||||||
|
if not agentStyle:
|
||||||
|
return dict(DEFAULT_STYLE)
|
||||||
|
return _deepMerge(DEFAULT_STYLE, agentStyle)
|
||||||
|
|
@ -9,11 +9,70 @@ from typing import Any, Dict
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _parseInlineRuns(text: str) -> list:
|
||||||
|
"""
|
||||||
|
Parse inline markdown formatting into a list of InlineRun dicts.
|
||||||
|
Handles: images, links, bold, italic, inline code, plain text.
|
||||||
|
Uses a regex-based tokenizer that processes tokens left-to-right.
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return [{"type": "text", "value": ""}]
|
||||||
|
|
||||||
|
# Pattern order matters: images before links, bold before italic
|
||||||
|
_TOKEN_RE = re.compile(
|
||||||
|
r'!\[(?P<imgAlt>[^\]]*)\]\((?P<imgSrc>[^)"]+)(?:\s+"(?P<imgWidth>\d+)pt")?\)' # image
|
||||||
|
r'|\[(?P<linkText>[^\]]+)\]\((?P<linkHref>[^)]+)\)' # link
|
||||||
|
r'|`(?P<code>[^`]+)`' # inline code
|
||||||
|
r'|\*\*(?P<bold>.+?)\*\*' # bold
|
||||||
|
r'|(?<!\w)\*(?P<italic1>.+?)\*(?!\w)' # italic *x*
|
||||||
|
r'|(?<!\w)_(?P<italic2>.+?)_(?!\w)' # italic _x_
|
||||||
|
)
|
||||||
|
|
||||||
|
runs = []
|
||||||
|
lastEnd = 0
|
||||||
|
|
||||||
|
for m in _TOKEN_RE.finditer(text):
|
||||||
|
# Plain text before this match
|
||||||
|
if m.start() > lastEnd:
|
||||||
|
runs.append({"type": "text", "value": text[lastEnd:m.start()]})
|
||||||
|
|
||||||
|
if m.group("imgAlt") is not None or m.group("imgSrc") is not None:
|
||||||
|
alt = (m.group("imgAlt") or "").strip() or "Image"
|
||||||
|
src = (m.group("imgSrc") or "").strip()
|
||||||
|
widthStr = m.group("imgWidth")
|
||||||
|
run = {"type": "image", "value": alt}
|
||||||
|
if src.startswith("file:"):
|
||||||
|
run["fileId"] = src[5:]
|
||||||
|
else:
|
||||||
|
run["href"] = src
|
||||||
|
if widthStr:
|
||||||
|
run["widthPt"] = int(widthStr)
|
||||||
|
runs.append(run)
|
||||||
|
elif m.group("linkText") is not None:
|
||||||
|
runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")})
|
||||||
|
elif m.group("code") is not None:
|
||||||
|
runs.append({"type": "code", "value": m.group("code")})
|
||||||
|
elif m.group("bold") is not None:
|
||||||
|
runs.append({"type": "bold", "value": m.group("bold")})
|
||||||
|
elif m.group("italic1") is not None:
|
||||||
|
runs.append({"type": "italic", "value": m.group("italic1")})
|
||||||
|
elif m.group("italic2") is not None:
|
||||||
|
runs.append({"type": "italic", "value": m.group("italic2")})
|
||||||
|
|
||||||
|
lastEnd = m.end()
|
||||||
|
|
||||||
|
# Trailing plain text
|
||||||
|
if lastEnd < len(text):
|
||||||
|
runs.append({"type": "text", "value": text[lastEnd:]})
|
||||||
|
|
||||||
|
return runs if runs else [{"type": "text", "value": text}]
|
||||||
|
|
||||||
|
|
||||||
def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]:
|
def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Convert markdown content to the standard document JSON format expected by renderReport.
|
Convert markdown content to the standard document JSON format with Inline-Run model.
|
||||||
Supports headings, code blocks, tables, lists, images (file: refs), paragraphs.
|
Sections use inlineRuns (list of run dicts) instead of plain text strings.
|
||||||
For plain text: wraps entire content in a single paragraph section.
|
Supports headings, code blocks, tables, lists, images, paragraphs.
|
||||||
"""
|
"""
|
||||||
if not isinstance(markdown, str):
|
if not isinstance(markdown, str):
|
||||||
markdown = str(markdown) if markdown else ""
|
markdown = str(markdown) if markdown else ""
|
||||||
|
|
@ -31,7 +90,7 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
|
||||||
while i < len(lines):
|
while i < len(lines):
|
||||||
line = lines[i]
|
line = lines[i]
|
||||||
|
|
||||||
# Headings
|
# Headings (plain text, no inline formatting)
|
||||||
headingMatch = re.match(r"^(#{1,6})\s+(.+)", line)
|
headingMatch = re.match(r"^(#{1,6})\s+(.+)", line)
|
||||||
if headingMatch:
|
if headingMatch:
|
||||||
level = len(headingMatch.group(1))
|
level = len(headingMatch.group(1))
|
||||||
|
|
@ -43,7 +102,7 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Fenced code blocks
|
# Fenced code blocks (no inline formatting)
|
||||||
codeMatch = re.match(r"^```(\w*)", line)
|
codeMatch = re.match(r"^```(\w*)", line)
|
||||||
if codeMatch:
|
if codeMatch:
|
||||||
lang = codeMatch.group(1) or "text"
|
lang = codeMatch.group(1) or "text"
|
||||||
|
|
@ -59,14 +118,14 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Tables
|
# Tables - cells are List[InlineRun]
|
||||||
tableMatch = re.match(r"^\|(.+)\|$", line)
|
tableMatch = re.match(r"^\|(.+)\|$", line)
|
||||||
if tableMatch and (i + 1) < len(lines) and re.match(r"^\|[\s\-:|]+\|$", lines[i + 1]):
|
if tableMatch and (i + 1) < len(lines) and re.match(r"^\|[\s\-:|]+\|$", lines[i + 1]):
|
||||||
headerCells = [c.strip() for c in tableMatch.group(1).split("|")]
|
headerCells = [_parseInlineRuns(c.strip()) for c in tableMatch.group(1).split("|")]
|
||||||
i += 2
|
i += 2
|
||||||
rows = []
|
rows = []
|
||||||
while i < len(lines) and re.match(r"^\|(.+)\|$", lines[i]):
|
while i < len(lines) and re.match(r"^\|(.+)\|$", lines[i]):
|
||||||
rowCells = [c.strip() for c in lines[i][1:-1].split("|")]
|
rowCells = [_parseInlineRuns(c.strip()) for c in lines[i][1:-1].split("|")]
|
||||||
rows.append(rowCells)
|
rows.append(rowCells)
|
||||||
i += 1
|
i += 1
|
||||||
sections.append({
|
sections.append({
|
||||||
|
|
@ -75,14 +134,14 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Bullet / numbered lists
|
# Bullet / numbered lists - items are List[List[InlineRun]]
|
||||||
listMatch = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", line)
|
listMatch = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", line)
|
||||||
if listMatch:
|
if listMatch:
|
||||||
isNumbered = bool(re.match(r"\d+[.)]", listMatch.group(2)))
|
isNumbered = bool(re.match(r"\d+[.)]", listMatch.group(2)))
|
||||||
items = []
|
items = []
|
||||||
while i < len(lines) and re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i]):
|
while i < len(lines) and re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i]):
|
||||||
m = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i])
|
m = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i])
|
||||||
items.append({"text": m.group(3).strip()})
|
items.append(_parseInlineRuns(m.group(3).strip()))
|
||||||
i += 1
|
i += 1
|
||||||
sections.append({
|
sections.append({
|
||||||
"id": _nextId(), "content_type": "bullet_list", "order": order,
|
"id": _nextId(), "content_type": "bullet_list", "order": order,
|
||||||
|
|
@ -95,46 +154,50 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Images (simplified: store as paragraph with ref for now - full resolution needs Knowledge Store)
|
# Standalone image on its own line -> block-level image section
|
||||||
imgMatch = re.match(r"^!\[([^\]]*)\]\(([^)]+)\)", line)
|
imgMatch = re.match(r"^!\[([^\]]*)\]\(([^)\"]+)(?:\s+\"(\d+)pt\")?\)\s*$", line)
|
||||||
if imgMatch:
|
if imgMatch:
|
||||||
altText = imgMatch.group(1).strip() or "Image"
|
altText = imgMatch.group(1).strip() or "Image"
|
||||||
src = imgMatch.group(2).strip()
|
src = imgMatch.group(2).strip()
|
||||||
|
widthStr = imgMatch.group(3)
|
||||||
fileId = src[5:] if src.startswith("file:") else ""
|
fileId = src[5:] if src.startswith("file:") else ""
|
||||||
|
content = {
|
||||||
|
"altText": altText,
|
||||||
|
"base64Data": "",
|
||||||
|
"_fileRef": fileId,
|
||||||
|
"_srcUrl": src if not fileId else "",
|
||||||
|
}
|
||||||
|
if widthStr:
|
||||||
|
content["widthPt"] = int(widthStr)
|
||||||
sections.append({
|
sections.append({
|
||||||
"id": _nextId(), "content_type": "image", "order": order,
|
"id": _nextId(), "content_type": "image", "order": order,
|
||||||
"elements": [{
|
"elements": [{"content": content}],
|
||||||
"content": {
|
|
||||||
"altText": altText,
|
|
||||||
"base64Data": "",
|
|
||||||
"_fileRef": fileId,
|
|
||||||
"_srcUrl": src if not fileId else "",
|
|
||||||
}
|
|
||||||
}],
|
|
||||||
})
|
})
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Paragraph
|
# Paragraph - produces inlineRuns
|
||||||
paraLines = []
|
paraLines = []
|
||||||
while i < len(lines) and lines[i].strip() and not re.match(
|
while i < len(lines) and lines[i].strip() and not re.match(
|
||||||
r"^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )", lines[i]
|
r"^(#{1,6}\s|```|\|.+\||!\[[^\]]*\]\([^)]+\)\s*$|(\s*)([-*+]|\d+[.)]) )", lines[i]
|
||||||
):
|
):
|
||||||
paraLines.append(lines[i])
|
paraLines.append(lines[i])
|
||||||
i += 1
|
i += 1
|
||||||
if paraLines:
|
if paraLines:
|
||||||
|
combinedText = " ".join(paraLines)
|
||||||
sections.append({
|
sections.append({
|
||||||
"id": _nextId(), "content_type": "paragraph", "order": order,
|
"id": _nextId(), "content_type": "paragraph", "order": order,
|
||||||
"elements": [{"content": {"text": " ".join(paraLines)}}],
|
"elements": [{"content": {"inlineRuns": _parseInlineRuns(combinedText)}}],
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
if not sections:
|
if not sections:
|
||||||
|
fallbackText = markdown.strip() or "(empty)"
|
||||||
sections.append({
|
sections.append({
|
||||||
"id": _nextId(), "content_type": "paragraph", "order": order,
|
"id": _nextId(), "content_type": "paragraph", "order": order,
|
||||||
"elements": [{"content": {"text": markdown.strip() or "(empty)"}}],
|
"elements": [{"content": {"inlineRuns": _parseInlineRuns(fallbackText)}}],
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -302,6 +302,30 @@ async def _executeWithRetry(executor, node, context, maxRetries: int = 0, retryD
|
||||||
raise lastError
|
raise lastError
|
||||||
|
|
||||||
|
|
||||||
|
def _substituteFeatureInstancePlaceholders(
|
||||||
|
graph: Dict[str, Any],
|
||||||
|
targetFeatureInstanceId: str,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Replace ``{{featureInstanceId}}`` placeholders in the serialised graph.
|
||||||
|
|
||||||
|
Works on the full JSON representation so that placeholders inside nested
|
||||||
|
parameter dicts, prompt strings, etc. are all caught. Already-resolved
|
||||||
|
concrete UUIDs (pre-baked by ``_copyTemplateWorkflows``) are left untouched
|
||||||
|
because the placeholder literal ``{{featureInstanceId}}`` will not match.
|
||||||
|
"""
|
||||||
|
import json as _json
|
||||||
|
raw = _json.dumps(graph)
|
||||||
|
if "{{featureInstanceId}}" not in raw:
|
||||||
|
return graph
|
||||||
|
replaced = raw.replace("{{featureInstanceId}}", targetFeatureInstanceId)
|
||||||
|
logger.debug(
|
||||||
|
"_substituteFeatureInstancePlaceholders: resolved %d occurrence(s) -> %s",
|
||||||
|
raw.count("{{featureInstanceId}}"),
|
||||||
|
targetFeatureInstanceId,
|
||||||
|
)
|
||||||
|
return _json.loads(replaced)
|
||||||
|
|
||||||
|
|
||||||
async def executeGraph(
|
async def executeGraph(
|
||||||
graph: Dict[str, Any],
|
graph: Dict[str, Any],
|
||||||
services: Any,
|
services: Any,
|
||||||
|
|
@ -315,6 +339,7 @@ async def executeGraph(
|
||||||
runId: Optional[str] = None,
|
runId: Optional[str] = None,
|
||||||
run_envelope: Optional[Dict[str, Any]] = None,
|
run_envelope: Optional[Dict[str, Any]] = None,
|
||||||
label: Optional[str] = None,
|
label: Optional[str] = None,
|
||||||
|
targetFeatureInstanceId: Optional[str] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Execute automation2 graph. Returns { success, nodeOutputs, error?, stopped? }.
|
Execute automation2 graph. Returns { success, nodeOutputs, error?, stopped? }.
|
||||||
|
|
@ -322,14 +347,16 @@ async def executeGraph(
|
||||||
pauses the run, and returns { success: False, paused: True, taskId, runId }.
|
pauses the run, and returns { success: False, paused: True, taskId, runId }.
|
||||||
For resume: pass initialNodeOutputs (with result for the human node) and startAfterNodeId.
|
For resume: pass initialNodeOutputs (with result for the human node) and startAfterNodeId.
|
||||||
For fresh runs: pass run_envelope (unified start payload for the start node); normalized with userId into context.runEnvelope.
|
For fresh runs: pass run_envelope (unified start payload for the start node); normalized with userId into context.runEnvelope.
|
||||||
|
targetFeatureInstanceId: resolves {{featureInstanceId}} placeholders in the graph JSON before execution.
|
||||||
"""
|
"""
|
||||||
logger.info(
|
logger.info(
|
||||||
"executeGraph start: instanceId=%s workflowId=%s userId=%s mandateId=%s resume=%s",
|
"executeGraph start: instanceId=%s workflowId=%s userId=%s mandateId=%s resume=%s targetInstance=%s",
|
||||||
instanceId,
|
instanceId,
|
||||||
workflowId,
|
workflowId,
|
||||||
userId,
|
userId,
|
||||||
mandateId,
|
mandateId,
|
||||||
startAfterNodeId is not None,
|
startAfterNodeId is not None,
|
||||||
|
targetFeatureInstanceId,
|
||||||
)
|
)
|
||||||
from modules.workflows.processing.shared.methodDiscovery import discoverMethods
|
from modules.workflows.processing.shared.methodDiscovery import discoverMethods
|
||||||
discoverMethods(services)
|
discoverMethods(services)
|
||||||
|
|
@ -338,6 +365,9 @@ async def executeGraph(
|
||||||
materializeFeatureInstanceRefs,
|
materializeFeatureInstanceRefs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if targetFeatureInstanceId:
|
||||||
|
graph = _substituteFeatureInstancePlaceholders(graph, targetFeatureInstanceId)
|
||||||
|
|
||||||
# Phase-5 Schicht-4: typed-ref envelopes are materialized FIRST so the
|
# Phase-5 Schicht-4: typed-ref envelopes are materialized FIRST so the
|
||||||
# subsequent connection-ref pass and validation see the canonical shape.
|
# subsequent connection-ref pass and validation see the canonical shape.
|
||||||
graph = materializeFeatureInstanceRefs(graph)
|
graph = materializeFeatureInstanceRefs(graph)
|
||||||
|
|
|
||||||
18
modules/workflows/methods/methodAi/_common.py
Normal file
18
modules/workflows/methods/methodAi/_common.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
|
||||||
|
"""Shared helpers for AI workflow actions."""
|
||||||
|
|
||||||
|
|
||||||
|
def applyCommonAiParams(parameters: dict, request) -> None:
|
||||||
|
"""Apply common AI parameters (requireNeutralization, allowedModels) from node to request."""
|
||||||
|
requireNeutralization = parameters.get("requireNeutralization")
|
||||||
|
if requireNeutralization is not None:
|
||||||
|
request.requireNeutralization = bool(requireNeutralization)
|
||||||
|
|
||||||
|
allowedModels = parameters.get("allowedModels")
|
||||||
|
if allowedModels and isinstance(allowedModels, list):
|
||||||
|
if not request.options:
|
||||||
|
from modules.datamodels.datamodelAi import AiCallOptions
|
||||||
|
request.options = AiCallOptions()
|
||||||
|
request.options.allowedModels = allowedModels
|
||||||
|
|
@ -67,6 +67,8 @@ async def consolidate(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE),
|
options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE),
|
||||||
)
|
)
|
||||||
|
from modules.workflows.methods.methodAi._common import applyCommonAiParams
|
||||||
|
applyCommonAiParams(parameters, req)
|
||||||
resp = await ai_service.callAi(req)
|
resp = await ai_service.callAi(req)
|
||||||
except (SubscriptionInactiveException, BillingContextError):
|
except (SubscriptionInactiveException, BillingContextError):
|
||||||
raise
|
raise
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,10 @@ async def convertDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
}
|
}
|
||||||
if parentOperationId:
|
if parentOperationId:
|
||||||
processParams["parentOperationId"] = parentOperationId
|
processParams["parentOperationId"] = parentOperationId
|
||||||
|
if parameters.get("allowedModels"):
|
||||||
|
processParams["allowedModels"] = parameters["allowedModels"]
|
||||||
|
if parameters.get("requireNeutralization") is not None:
|
||||||
|
processParams["requireNeutralization"] = parameters["requireNeutralization"]
|
||||||
|
|
||||||
return await self.process(processParams)
|
return await self.process(processParams)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,16 @@ async def generateCode(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
processingMode=ProcessingModeEnum.DETAILED
|
processingMode=ProcessingModeEnum.DETAILED
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Apply node-level AI params
|
||||||
|
allowedModels = parameters.get("allowedModels")
|
||||||
|
if allowedModels and isinstance(allowedModels, list):
|
||||||
|
options.allowedModels = allowedModels
|
||||||
|
requireNeutralization = parameters.get("requireNeutralization")
|
||||||
|
if requireNeutralization is not None:
|
||||||
|
_ctx = getattr(self.services, '_context', None)
|
||||||
|
if _ctx:
|
||||||
|
_ctx.requireNeutralization = bool(requireNeutralization)
|
||||||
|
|
||||||
# outputFormat: Optional - if None, formats determined from prompt by AI
|
# outputFormat: Optional - if None, formats determined from prompt by AI
|
||||||
aiResponse: AiResponse = await self.services.ai.callAiContent(
|
aiResponse: AiResponse = await self.services.ai.callAiContent(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,16 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
compressContext=False
|
compressContext=False
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Apply node-level AI params
|
||||||
|
allowedModels = parameters.get("allowedModels")
|
||||||
|
if allowedModels and isinstance(allowedModels, list):
|
||||||
|
options.allowedModels = allowedModels
|
||||||
|
requireNeutralization = parameters.get("requireNeutralization")
|
||||||
|
if requireNeutralization is not None:
|
||||||
|
_ctx = getattr(self.services, '_context', None)
|
||||||
|
if _ctx:
|
||||||
|
_ctx.requireNeutralization = bool(requireNeutralization)
|
||||||
|
|
||||||
# outputFormat: Optional - if None, formats determined from prompt by AI
|
# outputFormat: Optional - if None, formats determined from prompt by AI
|
||||||
aiResponse: AiResponse = await self.services.ai.callAiContent(
|
aiResponse: AiResponse = await self.services.ai.callAiContent(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,47 @@ def _action_docs_to_content_parts(services, docs: List[Any]) -> List[ContentPart
|
||||||
logger.info(f"ai.process: Extracted {len(ec.parts)} parts from {name} (no persistence)")
|
logger.info(f"ai.process: Extracted {len(ec.parts)} parts from {name} (no persistence)")
|
||||||
return all_parts
|
return all_parts
|
||||||
|
|
||||||
|
def _resolve_file_refs_to_content_parts(services, fileIdRefs) -> List[ContentPart]:
|
||||||
|
"""Fetch files by ID from the file store and extract content.
|
||||||
|
Used for automation2 workflows where documents are file-store references,
|
||||||
|
not chat message attachments."""
|
||||||
|
from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy
|
||||||
|
|
||||||
|
mgmt = getattr(services, 'interfaceDbComponent', None)
|
||||||
|
extraction = getattr(services, 'extraction', None)
|
||||||
|
if not mgmt or not extraction:
|
||||||
|
logger.warning("_resolve_file_refs_to_content_parts: missing interfaceDbComponent or extraction service")
|
||||||
|
return []
|
||||||
|
|
||||||
|
allParts: List[ContentPart] = []
|
||||||
|
opts = ExtractionOptions(prompt="", mergeStrategy=MergeStrategy())
|
||||||
|
for ref in fileIdRefs:
|
||||||
|
fileId = ref.documentId
|
||||||
|
fileMeta = mgmt.getFile(fileId)
|
||||||
|
if not fileMeta:
|
||||||
|
logger.warning(f"_resolve_file_refs_to_content_parts: file {fileId} not found")
|
||||||
|
continue
|
||||||
|
fileData = mgmt.getFileData(fileId)
|
||||||
|
if not fileData:
|
||||||
|
logger.warning(f"_resolve_file_refs_to_content_parts: no data for file {fileId}")
|
||||||
|
continue
|
||||||
|
fileName = getattr(fileMeta, 'fileName', fileId)
|
||||||
|
mimeType = getattr(fileMeta, 'mimeType', 'application/octet-stream')
|
||||||
|
ec = extraction.extractContentFromBytes(
|
||||||
|
documentBytes=fileData,
|
||||||
|
fileName=fileName,
|
||||||
|
mimeType=mimeType,
|
||||||
|
documentId=fileId,
|
||||||
|
options=opts,
|
||||||
|
)
|
||||||
|
for p in ec.parts:
|
||||||
|
if p.data or getattr(p, "typeGroup", "") == "image":
|
||||||
|
p.metadata.setdefault("originalFileName", fileName)
|
||||||
|
allParts.append(p)
|
||||||
|
logger.info(f"_resolve_file_refs_to_content_parts: extracted {len(ec.parts)} parts from {fileName}")
|
||||||
|
return allParts
|
||||||
|
|
||||||
|
|
||||||
async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
operationId = None
|
operationId = None
|
||||||
try:
|
try:
|
||||||
|
|
@ -130,6 +171,17 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
f"to DocumentReferenceList with {len(documentList.references)} references"
|
f"to DocumentReferenceList with {len(documentList.references)} references"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Resolve DocumentItemReferences (file-ID refs from automation2) directly
|
||||||
|
# from the file store. These cannot be resolved via chat messages.
|
||||||
|
from modules.datamodels.datamodelDocref import DocumentItemReference
|
||||||
|
fileIdRefs = [r for r in documentList.references if isinstance(r, DocumentItemReference)]
|
||||||
|
if fileIdRefs:
|
||||||
|
extractedParts = _resolve_file_refs_to_content_parts(self.services, fileIdRefs)
|
||||||
|
if extractedParts:
|
||||||
|
inline_content_parts = (inline_content_parts or []) + extractedParts
|
||||||
|
remaining = [r for r in documentList.references if not isinstance(r, DocumentItemReference)]
|
||||||
|
documentList = DocumentReferenceList(references=remaining)
|
||||||
|
|
||||||
# Optional: if omitted, formats determined from prompt. Default "txt" is validation fallback only.
|
# Optional: if omitted, formats determined from prompt. Default "txt" is validation fallback only.
|
||||||
resultType = parameters.get("resultType")
|
resultType = parameters.get("resultType")
|
||||||
simpleMode = parameters.get("simpleMode", False)
|
simpleMode = parameters.get("simpleMode", False)
|
||||||
|
|
@ -158,6 +210,18 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
mimeMap = {"txt": "text/plain", "json": "application/json", "html": "text/html", "md": "text/markdown", "csv": "text/csv", "xml": "application/xml"}
|
mimeMap = {"txt": "text/plain", "json": "application/json", "html": "text/html", "md": "text/markdown", "csv": "text/csv", "xml": "application/xml"}
|
||||||
output_mime_type = mimeMap.get(normalized_result_type, "text/plain") if normalized_result_type else "text/plain"
|
output_mime_type = mimeMap.get(normalized_result_type, "text/plain") if normalized_result_type else "text/plain"
|
||||||
|
|
||||||
|
# Normalize context: workflow refs may resolve to dict/list instead of str
|
||||||
|
paramContext = parameters.get("context")
|
||||||
|
if paramContext is not None and not isinstance(paramContext, str):
|
||||||
|
try:
|
||||||
|
paramContext = json.dumps(paramContext, ensure_ascii=False, default=str)
|
||||||
|
parameters["context"] = paramContext
|
||||||
|
logger.info(f"ai.process: Serialized non-string context ({type(parameters.get('context')).__name__}) to JSON ({len(paramContext)} chars)")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"ai.process: Failed to serialize context: {e}")
|
||||||
|
paramContext = str(paramContext)
|
||||||
|
parameters["context"] = paramContext
|
||||||
|
|
||||||
# Phase 7.3: Pass documentList and/or contentParts to AI service
|
# Phase 7.3: Pass documentList and/or contentParts to AI service
|
||||||
contentParts: Optional[List[ContentPart]] = inline_content_parts
|
contentParts: Optional[List[ContentPart]] = inline_content_parts
|
||||||
if "contentParts" in parameters and not inline_content_parts:
|
if "contentParts" in parameters and not inline_content_parts:
|
||||||
|
|
@ -212,6 +276,9 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from modules.workflows.methods.methodAi._common import applyCommonAiParams
|
||||||
|
applyCommonAiParams(parameters, request)
|
||||||
|
|
||||||
aiResponse_obj = await self.services.ai.callAi(request)
|
aiResponse_obj = await self.services.ai.callAi(request)
|
||||||
|
|
||||||
# Convert AiCallResponse to AiResponse format
|
# Convert AiCallResponse to AiResponse format
|
||||||
|
|
@ -243,6 +310,16 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
operationType=OperationTypeEnum.IMAGE_GENERATE if isImageGeneration else OperationTypeEnum.DATA_GENERATE
|
operationType=OperationTypeEnum.IMAGE_GENERATE if isImageGeneration else OperationTypeEnum.DATA_GENERATE
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Apply node-level AI params (allowedModels, requireNeutralization)
|
||||||
|
allowedModels = parameters.get("allowedModels")
|
||||||
|
if allowedModels and isinstance(allowedModels, list):
|
||||||
|
options.allowedModels = allowedModels
|
||||||
|
requireNeutralization = parameters.get("requireNeutralization")
|
||||||
|
if requireNeutralization is not None:
|
||||||
|
_ctx = getattr(self.services, '_context', None)
|
||||||
|
if _ctx:
|
||||||
|
_ctx.requireNeutralization = bool(requireNeutralization)
|
||||||
|
|
||||||
# Get generationIntent from parameters (required for DATA_GENERATE)
|
# Get generationIntent from parameters (required for DATA_GENERATE)
|
||||||
# Default to "document" if not provided (most common use case)
|
# Default to "document" if not provided (most common use case)
|
||||||
# For code generation, use ai.generateCode action or explicitly pass generationIntent="code"
|
# For code generation, use ai.generateCode action or explicitly pass generationIntent="code"
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,10 @@ async def summarizeDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
}
|
}
|
||||||
if parentOperationId:
|
if parentOperationId:
|
||||||
processParams["parentOperationId"] = parentOperationId
|
processParams["parentOperationId"] = parentOperationId
|
||||||
|
if parameters.get("allowedModels"):
|
||||||
|
processParams["allowedModels"] = parameters["allowedModels"]
|
||||||
|
if parameters.get("requireNeutralization") is not None:
|
||||||
|
processParams["requireNeutralization"] = parameters["requireNeutralization"]
|
||||||
|
|
||||||
return await self.process(processParams)
|
return await self.process(processParams)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,10 @@ async def translateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||||
processParams["resultType"] = resultType
|
processParams["resultType"] = resultType
|
||||||
if parentOperationId:
|
if parentOperationId:
|
||||||
processParams["parentOperationId"] = parentOperationId
|
processParams["parentOperationId"] = parentOperationId
|
||||||
|
if parameters.get("allowedModels"):
|
||||||
|
processParams["allowedModels"] = parameters["allowedModels"]
|
||||||
|
if parameters.get("requireNeutralization") is not None:
|
||||||
|
processParams["requireNeutralization"] = parameters["requireNeutralization"]
|
||||||
|
|
||||||
return await self.process(processParams)
|
return await self.process(processParams)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,23 @@ class MethodAi(MethodBase):
|
||||||
required=False,
|
required=False,
|
||||||
description="Document reference(s) in any format to use as input/context"
|
description="Document reference(s) in any format to use as input/context"
|
||||||
),
|
),
|
||||||
|
"context": WorkflowActionParameter(
|
||||||
|
name="context",
|
||||||
|
type="str",
|
||||||
|
frontendType=FrontendType.TEXTAREA,
|
||||||
|
required=False,
|
||||||
|
default="",
|
||||||
|
description="Additional context data (string or upstream-bound dict/list, e.g. accounting data) appended to the prompt. Non-string values are JSON-serialized."
|
||||||
|
),
|
||||||
|
"documentTheme": WorkflowActionParameter(
|
||||||
|
name="documentTheme",
|
||||||
|
type="str",
|
||||||
|
frontendType=FrontendType.SELECT,
|
||||||
|
frontendOptions=["general", "finance", "legal", "technical", "hr"],
|
||||||
|
required=False,
|
||||||
|
default="general",
|
||||||
|
description="Style hint for the document renderer (e.g. finance, legal). Used by the AI agent to choose colors and layout."
|
||||||
|
),
|
||||||
"resultType": WorkflowActionParameter(
|
"resultType": WorkflowActionParameter(
|
||||||
name="resultType",
|
name="resultType",
|
||||||
type="str",
|
type="str",
|
||||||
|
|
|
||||||
|
|
@ -243,6 +243,7 @@ class WorkflowScheduler:
|
||||||
runEnv = normalize_run_envelope(runEnv, user_id=str(eventUser.id) if eventUser else None)
|
runEnv = normalize_run_envelope(runEnv, user_id=str(eventUser.id) if eventUser else None)
|
||||||
|
|
||||||
_wfLabel = wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", None)
|
_wfLabel = wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", None)
|
||||||
|
_targetInstanceId = wf.get("targetFeatureInstanceId") if isinstance(wf, dict) else getattr(wf, "targetFeatureInstanceId", None)
|
||||||
|
|
||||||
result = await executeGraph(
|
result = await executeGraph(
|
||||||
graph=wf["graph"],
|
graph=wf["graph"],
|
||||||
|
|
@ -254,6 +255,7 @@ class WorkflowScheduler:
|
||||||
automation2_interface=iface,
|
automation2_interface=iface,
|
||||||
run_envelope=runEnv,
|
run_envelope=runEnv,
|
||||||
label=_wfLabel,
|
label=_wfLabel,
|
||||||
|
targetFeatureInstanceId=_targetInstanceId,
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
"WorkflowScheduler: executed workflow %s success=%s paused=%s",
|
"WorkflowScheduler: executed workflow %s success=%s paused=%s",
|
||||||
|
|
|
||||||
19
scripts/_archive/README.md
Normal file
19
scripts/_archive/README.md
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# Archived one-shot scripts
|
||||||
|
|
||||||
|
Diese Scripts haben einmal eine konkrete Daten- oder Code-Migration ausgefuehrt
|
||||||
|
und werden nicht mehr aktiv aufgerufen. Sie bleiben hier liegen, falls jemand
|
||||||
|
spaeter auf einem alten DB-Dump oder einem alten Branch nochmal denselben Stand
|
||||||
|
herstellen muss.
|
||||||
|
|
||||||
|
KEIN aktives Tool. Nicht aus CI, nicht aus Docs verlinken. Bei Aufraeumarbeiten
|
||||||
|
(z.B. nach 6 Monaten ohne Anwendung) loeschen.
|
||||||
|
|
||||||
|
## Inhalt
|
||||||
|
|
||||||
|
| Datei | Migrationsthema | Archiviert am | Begruendung |
|
||||||
|
|-------|-----------------|---------------|-------------|
|
||||||
|
| `check_orphan_featureinstance.py` | Vor-Ort-Check mit hardcoded FeatureInstance-/Mandate-UUIDs | 2026-04-29 | Ad-hoc fuer einen konkreten Vorfall |
|
||||||
|
| `script_db_cleanup_duplicate_roles.py` | Cleanup doppelter Roles wegen `IS NULL`-Bug in `connectorDbPostgre` | 2026-04-29 | Bug ist laengst gefixt, Cleanup ueberall durchgelaufen |
|
||||||
|
| `migrate_async_to_sync.py` | One-shot Codemod `async def` -> `def` fuer FastAPI-Routes | 2026-04-29 | Refactor abgeschlossen |
|
||||||
|
| `i18n_rekey_plaintext_keys.py` | Frontend `t('dot.notation')` -> `t('Klartext')` Rekey | 2026-04-29 | Frontend-Migration abgeschlossen (siehe `wiki/c-work/4-done/2026-04-ui-i18n-dynamic-language-sets.md`) |
|
||||||
|
| `script_db_migrate_accessrules_objectkeys.py` | AccessRule-Items: kurz -> vollqualifiziert (Navigation-API) | 2026-04-29 | Navigation-API live, MIGRATION_MAP nur fuer trustee+realestate hardcoded |
|
||||||
|
|
@ -1,25 +0,0 @@
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
||||||
import psycopg2, psycopg2.extras
|
|
||||||
from modules.shared.configuration import APP_CONFIG
|
|
||||||
|
|
||||||
c = psycopg2.connect(
|
|
||||||
host=APP_CONFIG.get('DB_HOST','localhost'),
|
|
||||||
user=APP_CONFIG.get('DB_USER'),
|
|
||||||
password=APP_CONFIG.get('DB_PASSWORD_SECRET'),
|
|
||||||
port=int(APP_CONFIG.get('DB_PORT',5432)),
|
|
||||||
dbname='poweron_app',
|
|
||||||
)
|
|
||||||
cur = c.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
|
|
||||||
cur.execute('SELECT id, name, label, enabled, "deletedAt", "sysCreatedAt" FROM "Mandate" ORDER BY "sysCreatedAt"')
|
|
||||||
print("All Mandates in poweron_app:")
|
|
||||||
for r in cur.fetchall():
|
|
||||||
print(f" id={r['id']} name={r['name']} label={r['label']} enabled={r['enabled']} deletedAt={r['deletedAt']}")
|
|
||||||
|
|
||||||
cur.execute('SELECT COUNT(*) AS n FROM "FeatureInstance" WHERE "featureCode" = %s', ("redmine",))
|
|
||||||
print(f"\nTotal redmine FeatureInstances in poweron_app: {cur.fetchone()['n']}")
|
|
||||||
|
|
||||||
cur.execute('SELECT id, "mandateId", label, enabled FROM "FeatureInstance" WHERE "featureCode" = %s ORDER BY "sysCreatedAt"', ("redmine",))
|
|
||||||
for r in cur.fetchall():
|
|
||||||
print(f" fi={r['id']} mandate={r['mandateId']} label={r['label']} enabled={r['enabled']}")
|
|
||||||
382
scripts/script_db_audit_legacy_state.py
Normal file
382
scripts/script_db_audit_legacy_state.py
Normal file
|
|
@ -0,0 +1,382 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Audit-Skript fuer Legacy-Bestaende vor Bootstrap-Cleanup (Plan C).
|
||||||
|
|
||||||
|
Prueft fuer jede der 5 Bootstrap-Migrationsroutinen, ob noch Restbestand
|
||||||
|
existiert. Wenn alle Checks 0 / GREEN liefern, kann die jeweilige Routine
|
||||||
|
sicher aus ``interfaceBootstrap.py`` / ``interfaceDbKnowledge.py`` entfernt
|
||||||
|
werden.
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
1. Mandate.description != NULL und Mandate.label leer
|
||||||
|
-> _migrateMandateDescriptionToLabel
|
||||||
|
2. Mandate.label leer ODER Mandate.name verstoesst gegen Slug-Regeln
|
||||||
|
-> _migrateMandateNameLabelSlugRules
|
||||||
|
3. Mandate mit name='Root' und isSystem=False
|
||||||
|
-> initRootMandate Legacy-Zweig
|
||||||
|
4. Role mit roleLabel='sysadmin' im Root-Mandat
|
||||||
|
-> _migrateAndDropSysAdminRole
|
||||||
|
5. FileContentIndex mit leerem mandateId UND leerem featureInstanceId
|
||||||
|
-> aggregateMandateRagTotalBytes Fallback-Block
|
||||||
|
|
||||||
|
Verwendung:
|
||||||
|
python -m scripts.script_db_audit_legacy_state # text-output
|
||||||
|
python -m scripts.script_db_audit_legacy_state --json # JSON-output
|
||||||
|
python -m scripts.script_db_audit_legacy_state --purge-rag-orphans
|
||||||
|
# loescht FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId
|
||||||
|
# (Voraussetzung fuer Removal des aggregateMandateRagTotalBytes-Fallback)
|
||||||
|
|
||||||
|
Exit-Code:
|
||||||
|
0 alle Checks GREEN (Removal sicher)
|
||||||
|
1 mind. ein Check RED (erst Daten bereinigen)
|
||||||
|
2 Skript-Fehler (DB nicht erreichbar etc.)
|
||||||
|
|
||||||
|
Lese-Zugriffe sind die Default. Schreibzugriffe NUR mit explizitem
|
||||||
|
``--purge-*``-Flag.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
_gatewayDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
if _gatewayDir not in sys.path:
|
||||||
|
sys.path.insert(0, _gatewayDir)
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
_envPath = os.path.join(_gatewayDir, "env_dev.env")
|
||||||
|
if os.path.exists(_envPath):
|
||||||
|
load_dotenv(_envPath)
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelUam import Mandate
|
||||||
|
from modules.datamodels.datamodelRbac import Role
|
||||||
|
from modules.datamodels.datamodelKnowledge import FileContentIndex
|
||||||
|
from modules.security.rootAccess import getRootDbAppConnector
|
||||||
|
from modules.interfaces.interfaceDbKnowledge import KnowledgeObjects
|
||||||
|
from modules.shared.mandateNameUtils import isValidMandateName
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.WARNING, format="%(message)s")
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _CheckResult:
|
||||||
|
"""Ergebnis eines einzelnen Audit-Checks."""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
routine: str
|
||||||
|
location: str
|
||||||
|
count: int
|
||||||
|
status: str
|
||||||
|
samples: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
def toDict(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"name": self.name,
|
||||||
|
"routine": self.routine,
|
||||||
|
"location": self.location,
|
||||||
|
"count": self.count,
|
||||||
|
"status": self.status,
|
||||||
|
"samples": self.samples,
|
||||||
|
"error": self.error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _getAppDb():
|
||||||
|
return getRootDbAppConnector()
|
||||||
|
|
||||||
|
|
||||||
|
def _getKnowledgeDb():
|
||||||
|
return KnowledgeObjects().db
|
||||||
|
|
||||||
|
|
||||||
|
def _checkMandateDescription(db) -> _CheckResult:
|
||||||
|
"""Mandate.description noch vorhanden und label leer?"""
|
||||||
|
rows = db.getRecordset(Mandate)
|
||||||
|
legacy = [
|
||||||
|
{
|
||||||
|
"id": r.get("id"),
|
||||||
|
"name": r.get("name"),
|
||||||
|
"description": str(r.get("description"))[:60] if r.get("description") else None,
|
||||||
|
"label": r.get("label"),
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
if r.get("description") and not r.get("label")
|
||||||
|
]
|
||||||
|
return _CheckResult(
|
||||||
|
name="mandate-description-to-label",
|
||||||
|
routine="_migrateMandateDescriptionToLabel",
|
||||||
|
location="interfaces/interfaceBootstrap.py:422-445",
|
||||||
|
count=len(legacy),
|
||||||
|
status="GREEN" if not legacy else "RED",
|
||||||
|
samples=legacy[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkMandateSlugRules(db) -> _CheckResult:
|
||||||
|
"""Mandate.name verletzt Slug-Regeln ODER Mandate.label leer?"""
|
||||||
|
rows = db.getRecordset(Mandate)
|
||||||
|
legacy = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for r in sorted(rows, key=lambda x: str(x.get("id", ""))):
|
||||||
|
name = (r.get("name") or "").strip()
|
||||||
|
labelRaw = r.get("label")
|
||||||
|
labelEmpty = not (labelRaw or "").strip() if labelRaw is not None else True
|
||||||
|
nameInvalid = not isValidMandateName(name)
|
||||||
|
nameCollides = name in seen
|
||||||
|
if not nameInvalid and not nameCollides:
|
||||||
|
seen.add(name)
|
||||||
|
if labelEmpty or nameInvalid or nameCollides:
|
||||||
|
legacy.append(
|
||||||
|
{
|
||||||
|
"id": r.get("id"),
|
||||||
|
"name": name,
|
||||||
|
"label": r.get("label"),
|
||||||
|
"labelEmpty": labelEmpty,
|
||||||
|
"nameInvalid": nameInvalid,
|
||||||
|
"nameCollides": nameCollides,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return _CheckResult(
|
||||||
|
name="mandate-name-slug-rules",
|
||||||
|
routine="_migrateMandateNameLabelSlugRules",
|
||||||
|
location="interfaces/interfaceBootstrap.py:448-511",
|
||||||
|
count=len(legacy),
|
||||||
|
status="GREEN" if not legacy else "RED",
|
||||||
|
samples=legacy[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkRootMandateLegacy(db) -> _CheckResult:
|
||||||
|
"""Mandate mit name='Root' (case-sensitive) ODER isSystem=False fuer root?"""
|
||||||
|
legacyByName = db.getRecordset(Mandate, recordFilter={"name": "Root"})
|
||||||
|
rows = db.getRecordset(Mandate, recordFilter={"name": "root"})
|
||||||
|
legacyByFlag = [r for r in rows if not r.get("isSystem")]
|
||||||
|
combined = list(legacyByName) + legacyByFlag
|
||||||
|
samples = [
|
||||||
|
{
|
||||||
|
"id": r.get("id"),
|
||||||
|
"name": r.get("name"),
|
||||||
|
"isSystem": r.get("isSystem"),
|
||||||
|
}
|
||||||
|
for r in combined
|
||||||
|
]
|
||||||
|
return _CheckResult(
|
||||||
|
name="root-mandate-legacy",
|
||||||
|
routine="initRootMandate-legacy-branch",
|
||||||
|
location="interfaces/interfaceBootstrap.py:406-412",
|
||||||
|
count=len(samples),
|
||||||
|
status="GREEN" if not samples else "RED",
|
||||||
|
samples=samples[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkSysadminRole(db) -> _CheckResult:
|
||||||
|
"""Legacy 'sysadmin'-Rolle im Root-Mandat?"""
|
||||||
|
rootMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
|
||||||
|
if not rootMandates:
|
||||||
|
return _CheckResult(
|
||||||
|
name="sysadmin-role",
|
||||||
|
routine="_migrateAndDropSysAdminRole",
|
||||||
|
location="interfaces/interfaceBootstrap.py:840-932",
|
||||||
|
count=0,
|
||||||
|
status="GREEN",
|
||||||
|
samples=[],
|
||||||
|
error="kein Root-Mandat gefunden -- Check uebersprungen (kann nicht relevant sein)",
|
||||||
|
)
|
||||||
|
rootId = str(rootMandates[0].get("id"))
|
||||||
|
rows = db.getRecordset(
|
||||||
|
Role,
|
||||||
|
recordFilter={"roleLabel": "sysadmin", "mandateId": rootId, "featureInstanceId": None},
|
||||||
|
)
|
||||||
|
samples = [{"id": r.get("id"), "roleLabel": r.get("roleLabel")} for r in rows]
|
||||||
|
return _CheckResult(
|
||||||
|
name="sysadmin-role",
|
||||||
|
routine="_migrateAndDropSysAdminRole",
|
||||||
|
location="interfaces/interfaceBootstrap.py:840-932",
|
||||||
|
count=len(samples),
|
||||||
|
status="GREEN" if not samples else "RED",
|
||||||
|
samples=samples[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _checkRagFallback(knowDb) -> _CheckResult:
|
||||||
|
"""FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId?"""
|
||||||
|
rows = knowDb.getRecordset(FileContentIndex)
|
||||||
|
legacy = [
|
||||||
|
{
|
||||||
|
"id": r.get("id"),
|
||||||
|
"fileName": r.get("fileName"),
|
||||||
|
"totalSize": r.get("totalSize"),
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
if not (r.get("mandateId") or "").strip() and not (r.get("featureInstanceId") or "").strip()
|
||||||
|
]
|
||||||
|
return _CheckResult(
|
||||||
|
name="rag-fallback-orphan-index",
|
||||||
|
routine="aggregateMandateRagTotalBytes-fallback",
|
||||||
|
location="interfaces/interfaceDbKnowledge.py:609-635",
|
||||||
|
count=len(legacy),
|
||||||
|
status="GREEN" if not legacy else "RED",
|
||||||
|
samples=legacy[:5],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _runChecks() -> List[_CheckResult]:
|
||||||
|
appDb = _getAppDb()
|
||||||
|
knowDb = _getKnowledgeDb()
|
||||||
|
|
||||||
|
appChecks: List[Callable[[Any], _CheckResult]] = [
|
||||||
|
_checkMandateDescription,
|
||||||
|
_checkMandateSlugRules,
|
||||||
|
_checkRootMandateLegacy,
|
||||||
|
_checkSysadminRole,
|
||||||
|
]
|
||||||
|
|
||||||
|
results: List[_CheckResult] = []
|
||||||
|
for fn in appChecks:
|
||||||
|
try:
|
||||||
|
results.append(fn(appDb))
|
||||||
|
except Exception as exc:
|
||||||
|
results.append(
|
||||||
|
_CheckResult(
|
||||||
|
name=fn.__name__,
|
||||||
|
routine="?",
|
||||||
|
location="?",
|
||||||
|
count=-1,
|
||||||
|
status="ERROR",
|
||||||
|
error=f"{type(exc).__name__}: {exc}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
results.append(_checkRagFallback(knowDb))
|
||||||
|
except Exception as exc:
|
||||||
|
results.append(
|
||||||
|
_CheckResult(
|
||||||
|
name="rag-fallback-orphan-index",
|
||||||
|
routine="aggregateMandateRagTotalBytes-fallback",
|
||||||
|
location="interfaces/interfaceDbKnowledge.py:609-635",
|
||||||
|
count=-1,
|
||||||
|
status="ERROR",
|
||||||
|
error=f"{type(exc).__name__}: {exc}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _printText(results: List[_CheckResult]) -> None:
|
||||||
|
print("=" * 78)
|
||||||
|
print("BOOTSTRAP-MIGRATIONS LEGACY-STATE-AUDIT")
|
||||||
|
print("=" * 78)
|
||||||
|
for r in results:
|
||||||
|
marker = {
|
||||||
|
"GREEN": "[OK]",
|
||||||
|
"RED": "[!!]",
|
||||||
|
"ERROR": "[ERR]",
|
||||||
|
}.get(r.status, "[?]")
|
||||||
|
print(f"\n{marker} {r.name}")
|
||||||
|
print(f" Routine : {r.routine}")
|
||||||
|
print(f" Location: {r.location}")
|
||||||
|
print(f" Count : {r.count}")
|
||||||
|
print(f" Status : {r.status}")
|
||||||
|
if r.error:
|
||||||
|
print(f" Note : {r.error}")
|
||||||
|
if r.samples:
|
||||||
|
print(f" Samples : (max 5)")
|
||||||
|
for s in r.samples:
|
||||||
|
print(f" {s}")
|
||||||
|
|
||||||
|
print("\n" + "=" * 78)
|
||||||
|
greens = sum(1 for r in results if r.status == "GREEN")
|
||||||
|
reds = sum(1 for r in results if r.status == "RED")
|
||||||
|
errs = sum(1 for r in results if r.status == "ERROR")
|
||||||
|
print(f"SUMMARY: {greens} GREEN {reds} RED {errs} ERROR ({len(results)} total)")
|
||||||
|
if reds == 0 and errs == 0:
|
||||||
|
print("VERDICT: alle Migrationsroutinen koennen entfernt werden.")
|
||||||
|
elif errs > 0:
|
||||||
|
print("VERDICT: Audit unvollstaendig (Fehler) -- bitte Skript fixen.")
|
||||||
|
else:
|
||||||
|
print("VERDICT: erst Daten bereinigen, dann Routinen entfernen.")
|
||||||
|
print("=" * 78)
|
||||||
|
|
||||||
|
|
||||||
|
def _purgeRagOrphans() -> int:
|
||||||
|
"""Loescht alle FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId.
|
||||||
|
|
||||||
|
Returns: Anzahl geloeschter Rows.
|
||||||
|
"""
|
||||||
|
knowDb = _getKnowledgeDb()
|
||||||
|
rows = knowDb.getRecordset(FileContentIndex)
|
||||||
|
orphans = [
|
||||||
|
r for r in rows
|
||||||
|
if not (r.get("mandateId") or "").strip()
|
||||||
|
and not (r.get("featureInstanceId") or "").strip()
|
||||||
|
]
|
||||||
|
if not orphans:
|
||||||
|
print("Keine RAG-Orphans gefunden -- nichts zu purgen.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
print(f"Purge {len(orphans)} RAG-Orphan(s):")
|
||||||
|
deleted = 0
|
||||||
|
for r in orphans:
|
||||||
|
rid = r.get("id")
|
||||||
|
try:
|
||||||
|
knowDb.recordDelete(FileContentIndex, str(rid))
|
||||||
|
deleted += 1
|
||||||
|
print(f" geloescht: {rid} {r.get('fileName')}")
|
||||||
|
except Exception as exc:
|
||||||
|
print(f" FEHLER {rid}: {type(exc).__name__}: {exc}", file=sys.stderr)
|
||||||
|
print(f"Purge abgeschlossen: {deleted}/{len(orphans)} geloescht.")
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Audit-Skript fuer Legacy-Bestaende (Bootstrap-Cleanup Plan C)"
|
||||||
|
)
|
||||||
|
parser.add_argument("--json", action="store_true", help="JSON-Output statt Text")
|
||||||
|
parser.add_argument(
|
||||||
|
"--purge-rag-orphans",
|
||||||
|
action="store_true",
|
||||||
|
help="WRITE: loescht FileContentIndex-Rows ohne mandateId UND featureInstanceId",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.purge_rag_orphans:
|
||||||
|
try:
|
||||||
|
_purgeRagOrphans()
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"FATAL: Purge fehlgeschlagen -- {type(exc).__name__}: {exc}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
print()
|
||||||
|
|
||||||
|
try:
|
||||||
|
results = _runChecks()
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"FATAL: konnte Audit nicht starten -- {type(exc).__name__}: {exc}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps([r.toDict() for r in results], indent=2, default=str))
|
||||||
|
else:
|
||||||
|
_printText(results)
|
||||||
|
|
||||||
|
if any(r.status == "ERROR" for r in results):
|
||||||
|
return 2
|
||||||
|
if any(r.status == "RED" for r in results):
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
0
tests/serviceAi/__init__.py
Normal file
0
tests/serviceAi/__init__.py
Normal file
14
tests/serviceAi/test_allowed_models_whitelist.py
Normal file
14
tests/serviceAi/test_allowed_models_whitelist.py
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
import pytest
|
||||||
|
from modules.datamodels.datamodelAi import AiCallOptions
|
||||||
|
|
||||||
|
|
||||||
|
def test_allowed_models_field_exists():
|
||||||
|
opts = AiCallOptions(allowedModels=["gpt-5-mini", "claude-4-7-opus"])
|
||||||
|
assert opts.allowedModels == ["gpt-5-mini", "claude-4-7-opus"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_allowed_models_default_none():
|
||||||
|
opts = AiCallOptions()
|
||||||
|
assert opts.allowedModels is None
|
||||||
0
tests/serviceGeneration/__init__.py
Normal file
0
tests/serviceGeneration/__init__.py
Normal file
23
tests/serviceGeneration/test_inline_image_paragraph.py
Normal file
23
tests/serviceGeneration/test_inline_image_paragraph.py
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
import pytest
|
||||||
|
from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_image_in_paragraph():
|
||||||
|
md = "Results show  clearly."
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
|
||||||
|
types = [r["type"] for r in runs]
|
||||||
|
assert "text" in types
|
||||||
|
assert "image" in types
|
||||||
|
imgRun = next(r for r in runs if r["type"] == "image")
|
||||||
|
assert imgRun.get("fileId") == "abc"
|
||||||
|
|
||||||
|
|
||||||
|
def test_multiple_inline_images():
|
||||||
|
md = "A  B  C"
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
|
||||||
|
images = [r for r in runs if r["type"] == "image"]
|
||||||
|
assert len(images) == 2
|
||||||
71
tests/serviceGeneration/test_md_to_json_consolidation.py
Normal file
71
tests/serviceGeneration/test_md_to_json_consolidation.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
import pytest
|
||||||
|
from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson
|
||||||
|
|
||||||
|
|
||||||
|
def test_basic_paragraph():
|
||||||
|
result = markdownToDocumentJson("Hello world", "Test")
|
||||||
|
doc = result["documents"][0]
|
||||||
|
section = doc["sections"][0]
|
||||||
|
assert section["content_type"] == "paragraph"
|
||||||
|
assert section["elements"][0]["content"]["inlineRuns"][0] == {"type": "text", "value": "Hello world"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_bold():
|
||||||
|
result = markdownToDocumentJson("This is **bold** text", "Test")
|
||||||
|
runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
|
||||||
|
assert any(r["type"] == "bold" and r["value"] == "bold" for r in runs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_image():
|
||||||
|
result = markdownToDocumentJson("Text  more", "Test")
|
||||||
|
runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
|
||||||
|
assert any(r["type"] == "image" and r.get("fileId") == "abc123" for r in runs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_link():
|
||||||
|
result = markdownToDocumentJson("Click [here](https://example.com)", "Test")
|
||||||
|
runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
|
||||||
|
assert any(r["type"] == "link" and r.get("href") == "https://example.com" for r in runs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_table_cells_are_inline_runs():
|
||||||
|
md = "| A | B |\n| --- | --- |\n| **x** | y |"
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
section = result["documents"][0]["sections"][0]
|
||||||
|
assert section["content_type"] == "table"
|
||||||
|
rows = section["elements"][0]["content"]["rows"]
|
||||||
|
assert isinstance(rows[0][0], list)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bullet_list_inline_runs():
|
||||||
|
md = "- Item **one**\n- Item two"
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
section = result["documents"][0]["sections"][0]
|
||||||
|
assert section["content_type"] == "bullet_list"
|
||||||
|
items = section["elements"][0]["content"]["items"]
|
||||||
|
assert isinstance(items[0], list)
|
||||||
|
|
||||||
|
|
||||||
|
def test_standalone_image_block():
|
||||||
|
md = ""
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
section = result["documents"][0]["sections"][0]
|
||||||
|
assert section["content_type"] == "image"
|
||||||
|
|
||||||
|
|
||||||
|
def test_heading_unchanged():
|
||||||
|
result = markdownToDocumentJson("# Title", "Test")
|
||||||
|
section = result["documents"][0]["sections"][0]
|
||||||
|
assert section["content_type"] == "heading"
|
||||||
|
assert section["elements"][0]["content"]["text"] == "Title"
|
||||||
|
assert section["elements"][0]["content"]["level"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_code_block_unchanged():
|
||||||
|
md = "```python\nprint('hi')\n```"
|
||||||
|
result = markdownToDocumentJson(md, "Test")
|
||||||
|
section = result["documents"][0]["sections"][0]
|
||||||
|
assert section["content_type"] == "code_block"
|
||||||
|
assert section["elements"][0]["content"]["code"] == "print('hi')"
|
||||||
39
tests/serviceGeneration/test_style_resolver.py
Normal file
39
tests/serviceGeneration/test_style_resolver.py
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
import pytest
|
||||||
|
from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle, DEFAULT_STYLE
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_none_returns_defaults():
|
||||||
|
result = resolveStyle(None)
|
||||||
|
assert result == DEFAULT_STYLE
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_empty_returns_defaults():
|
||||||
|
result = resolveStyle({})
|
||||||
|
assert result == DEFAULT_STYLE
|
||||||
|
|
||||||
|
|
||||||
|
def test_override_single_color():
|
||||||
|
result = resolveStyle({"colors": {"primary": "#FF0000"}})
|
||||||
|
assert result["colors"]["primary"] == "#FF0000"
|
||||||
|
assert result["colors"]["secondary"] == DEFAULT_STYLE["colors"]["secondary"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_override_nested_heading():
|
||||||
|
result = resolveStyle({"headings": {"h1": {"sizePt": 30}}})
|
||||||
|
assert result["headings"]["h1"]["sizePt"] == 30
|
||||||
|
assert result["headings"]["h1"]["weight"] == "bold"
|
||||||
|
|
||||||
|
|
||||||
|
def test_override_font():
|
||||||
|
result = resolveStyle({"fonts": {"primary": "Arial"}})
|
||||||
|
assert result["fonts"]["primary"] == "Arial"
|
||||||
|
assert result["fonts"]["monospace"] == "Consolas"
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_style_passthrough():
|
||||||
|
custom = {"fonts": {"primary": "Helvetica", "monospace": "Monaco"}}
|
||||||
|
result = resolveStyle(custom)
|
||||||
|
assert result["fonts"]["primary"] == "Helvetica"
|
||||||
|
assert result["fonts"]["monospace"] == "Monaco"
|
||||||
|
|
@ -1,133 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# Copyright (c) 2025 Patrick Motsch
|
|
||||||
# All rights reserved.
|
|
||||||
"""
|
|
||||||
Unit tests for ``_migrateMandateNameLabelSlugRules`` in interfaceBootstrap.
|
|
||||||
|
|
||||||
Covers:
|
|
||||||
- legacy ``name``/``label`` rows get fixed (label fill, slug rename),
|
|
||||||
- collisions across legacy rows resolve via -2/-3 suffixes in stable id order,
|
|
||||||
- valid rows are left untouched (idempotency),
|
|
||||||
- second invocation is a no-op.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from modules.datamodels.datamodelUam import Mandate
|
|
||||||
from modules.interfaces.interfaceBootstrap import _migrateMandateNameLabelSlugRules
|
|
||||||
from modules.shared.mandateNameUtils import isValidMandateName
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeDb:
|
|
||||||
"""Minimal connector simulating getRecordset(Mandate)+recordModify(Mandate, id, data)."""
|
|
||||||
|
|
||||||
def __init__(self, rows: List[Dict[str, Any]]):
|
|
||||||
self.rows: List[Dict[str, Any]] = [dict(r) for r in rows]
|
|
||||||
self.modifyCalls: List[Dict[str, Any]] = []
|
|
||||||
|
|
||||||
def getRecordset(self, model, recordFilter: Optional[Dict[str, Any]] = None):
|
|
||||||
if model is not Mandate:
|
|
||||||
return []
|
|
||||||
if not recordFilter:
|
|
||||||
return [dict(r) for r in self.rows]
|
|
||||||
out = []
|
|
||||||
for r in self.rows:
|
|
||||||
if all(r.get(k) == v for k, v in recordFilter.items()):
|
|
||||||
out.append(dict(r))
|
|
||||||
return out
|
|
||||||
|
|
||||||
def recordModify(self, model, recordId: str, data: Dict[str, Any]):
|
|
||||||
self.modifyCalls.append({"id": str(recordId), "data": dict(data)})
|
|
||||||
for r in self.rows:
|
|
||||||
if str(r.get("id")) == str(recordId):
|
|
||||||
r.update(data)
|
|
||||||
return r
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _row(mid: str, name: Any, label: Any = None) -> Dict[str, Any]:
|
|
||||||
return {"id": mid, "name": name, "label": label}
|
|
||||||
|
|
||||||
|
|
||||||
class TestMigrationFillsLabel:
|
|
||||||
def test_emptyLabelGetsNameAsLabel(self):
|
|
||||||
db = _FakeDb([_row("a1", "good-name", None)])
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.rows[0]["label"] == "good-name"
|
|
||||||
assert db.rows[0]["name"] == "good-name"
|
|
||||||
|
|
||||||
def test_emptyLabelAndEmptyNameFallsBackToMandate(self):
|
|
||||||
db = _FakeDb([_row("a1", "", "")])
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.rows[0]["label"] == "Mandate"
|
|
||||||
assert isValidMandateName(db.rows[0]["name"])
|
|
||||||
|
|
||||||
|
|
||||||
class TestMigrationRenamesInvalidNames:
|
|
||||||
def test_invalidNameGetsSlugFromLabel(self):
|
|
||||||
db = _FakeDb([_row("a1", "Home patrick", "Home Patrick")])
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.rows[0]["name"] == "home-patrick"
|
|
||||||
assert db.rows[0]["label"] == "Home Patrick"
|
|
||||||
|
|
||||||
def test_umlautsTransliterated(self):
|
|
||||||
db = _FakeDb([_row("a1", "Müller AG", "Müller AG")])
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.rows[0]["name"] == "mueller-ag"
|
|
||||||
|
|
||||||
|
|
||||||
class TestMigrationCollisions:
|
|
||||||
def test_collisionsResolveByStableIdOrder(self):
|
|
||||||
rows = [
|
|
||||||
_row("z1", "Home patrick", "Home Patrick"),
|
|
||||||
_row("a1", "home-patrick", "Home Patrick Two"),
|
|
||||||
]
|
|
||||||
db = _FakeDb(rows)
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
byId = {r["id"]: r for r in db.rows}
|
|
||||||
assert byId["a1"]["name"] == "home-patrick"
|
|
||||||
assert byId["z1"]["name"] == "home-patrick-2"
|
|
||||||
|
|
||||||
def test_threeWayCollisionGetsThirdSuffix(self):
|
|
||||||
rows = [
|
|
||||||
_row("id-aaa", "home-patrick", "Home Patrick"),
|
|
||||||
_row("id-bbb", "Home patrick", "Home Patrick"),
|
|
||||||
_row("id-ccc", "home patrick", "Home Patrick"),
|
|
||||||
]
|
|
||||||
db = _FakeDb(rows)
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
names = sorted(r["name"] for r in db.rows)
|
|
||||||
assert names == ["home-patrick", "home-patrick-2", "home-patrick-3"]
|
|
||||||
|
|
||||||
|
|
||||||
class TestMigrationIdempotency:
|
|
||||||
def test_secondRunIsNoop(self):
|
|
||||||
rows = [
|
|
||||||
_row("a1", "home-patrick", "Home Patrick"),
|
|
||||||
_row("b1", "Home Müller", ""),
|
|
||||||
]
|
|
||||||
db = _FakeDb(rows)
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert all(isValidMandateName(r["name"]) for r in db.rows)
|
|
||||||
firstChanges = list(db.modifyCalls)
|
|
||||||
db.modifyCalls.clear()
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.modifyCalls == [], (
|
|
||||||
f"expected no further changes after first migration, got {db.modifyCalls}; "
|
|
||||||
f"firstRun changes: {firstChanges}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_validRowsLeftUntouched(self):
|
|
||||||
rows = [_row("a1", "root", "Root"), _row("b1", "alpina-treuhand", "Alpina Treuhand AG")]
|
|
||||||
db = _FakeDb(rows)
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.modifyCalls == []
|
|
||||||
|
|
||||||
|
|
||||||
class TestMigrationEmpty:
|
|
||||||
def test_emptyDbDoesNothing(self):
|
|
||||||
db = _FakeDb([])
|
|
||||||
_migrateMandateNameLabelSlugRules(db)
|
|
||||||
assert db.modifyCalls == []
|
|
||||||
|
|
@ -1,209 +0,0 @@
|
||||||
# Copyright (c) 2025 Patrick Motsch
|
|
||||||
# All rights reserved.
|
|
||||||
"""
|
|
||||||
Unit tests for the one-shot sysadmin role -> isPlatformAdmin migration.
|
|
||||||
|
|
||||||
Covers acceptance criteria from
|
|
||||||
``wiki/c-work/4-done/2026-04-sysadmin-authority-split.md``:
|
|
||||||
|
|
||||||
- AC#4 -> Existing sysadmin role-holders are promoted to ``isPlatformAdmin=True``
|
|
||||||
and the legacy role is removed (Role + UserMandateRole + AccessRules)
|
|
||||||
when the gateway boots.
|
|
||||||
- AC#10 -> The migration is idempotent and removes ALL artefacts (Role,
|
|
||||||
AccessRules, UserMandateRole) of the legacy ``sysadmin`` role.
|
|
||||||
|
|
||||||
Strategy: use an in-memory fake ``DatabaseConnector`` that records calls
|
|
||||||
and returns deterministic recordsets for ``Role``/``UserMandateRole``/
|
|
||||||
``UserMandate``/``UserInDB``/``AccessRule`` lookups.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any, Dict, List
|
|
||||||
from unittest.mock import Mock
|
|
||||||
|
|
||||||
from modules.interfaces.interfaceBootstrap import _migrateAndDropSysAdminRole
|
|
||||||
from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole
|
|
||||||
from modules.datamodels.datamodelRbac import AccessRule, Role
|
|
||||||
from modules.datamodels.datamodelUam import UserInDB
|
|
||||||
|
|
||||||
|
|
||||||
_ROOT_MANDATE_ID = "root-mandate-id"
|
|
||||||
_SYSADMIN_ROLE_ID = "sysadmin-role-id"
|
|
||||||
_USER_MANDATE_ID = "user-mandate-id"
|
|
||||||
_USER_ID = "legacy-user-id"
|
|
||||||
_UMR_ROW_ID = "umr-row-id"
|
|
||||||
_ACCESS_RULE_ID = "access-rule-id"
|
|
||||||
|
|
||||||
|
|
||||||
def _buildFakeDb(
|
|
||||||
*,
|
|
||||||
sysadminRoles: List[Dict[str, Any]],
|
|
||||||
umRoleRows: List[Dict[str, Any]],
|
|
||||||
userMandateRows: List[Dict[str, Any]],
|
|
||||||
users: List[Dict[str, Any]],
|
|
||||||
accessRules: List[Dict[str, Any]],
|
|
||||||
) -> Mock:
|
|
||||||
"""Build a fake ``DatabaseConnector`` that maps model -> recordset."""
|
|
||||||
|
|
||||||
deletes: List[tuple] = []
|
|
||||||
modifies: List[tuple] = []
|
|
||||||
|
|
||||||
def _getRecordset(model, recordFilter=None, **_): # noqa: ANN001
|
|
||||||
recordFilter = recordFilter or {}
|
|
||||||
if model is Role:
|
|
||||||
label = recordFilter.get("roleLabel")
|
|
||||||
mandateId = recordFilter.get("mandateId")
|
|
||||||
if label == "sysadmin" and mandateId == _ROOT_MANDATE_ID:
|
|
||||||
return list(sysadminRoles)
|
|
||||||
return []
|
|
||||||
if model is UserMandateRole:
|
|
||||||
wanted = recordFilter.get("roleId")
|
|
||||||
return [r for r in umRoleRows if r.get("roleId") == wanted]
|
|
||||||
if model is UserMandate:
|
|
||||||
wanted = recordFilter.get("id")
|
|
||||||
return [r for r in userMandateRows if r.get("id") == wanted]
|
|
||||||
if model is UserInDB:
|
|
||||||
wanted = recordFilter.get("id")
|
|
||||||
return [r for r in users if r.get("id") == wanted]
|
|
||||||
if model is AccessRule:
|
|
||||||
wanted = recordFilter.get("roleId")
|
|
||||||
return [r for r in accessRules if r.get("roleId") == wanted]
|
|
||||||
return []
|
|
||||||
|
|
||||||
def _recordModify(model, recordId, payload): # noqa: ANN001
|
|
||||||
modifies.append((model, recordId, payload))
|
|
||||||
# Reflect the change so a subsequent migration call is idempotent.
|
|
||||||
if model is UserInDB:
|
|
||||||
for u in users:
|
|
||||||
if u.get("id") == recordId:
|
|
||||||
u.update(payload)
|
|
||||||
return True
|
|
||||||
|
|
||||||
def _recordDelete(model, recordId): # noqa: ANN001
|
|
||||||
deletes.append((model, recordId))
|
|
||||||
if model is UserMandateRole:
|
|
||||||
umRoleRows[:] = [r for r in umRoleRows if r.get("id") != recordId]
|
|
||||||
elif model is AccessRule:
|
|
||||||
accessRules[:] = [r for r in accessRules if r.get("id") != recordId]
|
|
||||||
elif model is Role:
|
|
||||||
sysadminRoles[:] = [r for r in sysadminRoles if r.get("id") != recordId]
|
|
||||||
return True
|
|
||||||
|
|
||||||
db = Mock()
|
|
||||||
db.getRecordset = Mock(side_effect=_getRecordset)
|
|
||||||
db.recordModify = Mock(side_effect=_recordModify)
|
|
||||||
db.recordDelete = Mock(side_effect=_recordDelete)
|
|
||||||
db._modifies = modifies # exposed for assertions
|
|
||||||
db._deletes = deletes
|
|
||||||
return db
|
|
||||||
|
|
||||||
|
|
||||||
def _seed():
|
|
||||||
return {
|
|
||||||
"sysadminRoles": [{"id": _SYSADMIN_ROLE_ID, "roleLabel": "sysadmin",
|
|
||||||
"mandateId": _ROOT_MANDATE_ID}],
|
|
||||||
"umRoleRows": [{"id": _UMR_ROW_ID, "roleId": _SYSADMIN_ROLE_ID,
|
|
||||||
"userMandateId": _USER_MANDATE_ID}],
|
|
||||||
"userMandateRows": [{"id": _USER_MANDATE_ID, "userId": _USER_ID,
|
|
||||||
"mandateId": _ROOT_MANDATE_ID}],
|
|
||||||
"users": [{"id": _USER_ID, "username": "legacy",
|
|
||||||
"isSysAdmin": False, "isPlatformAdmin": False}],
|
|
||||||
"accessRules": [{"id": _ACCESS_RULE_ID, "roleId": _SYSADMIN_ROLE_ID}],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# AC #4 — promote + drop on first run
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
def testMigrationPromotesUserAndDropsArtefacts():
|
|
||||||
"""AC#4: legacy holder is promoted; Role+AccessRule+UMR are deleted."""
|
|
||||||
seed = _seed()
|
|
||||||
db = _buildFakeDb(**seed)
|
|
||||||
|
|
||||||
_migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
|
|
||||||
|
|
||||||
# User got isPlatformAdmin=True
|
|
||||||
assert seed["users"][0]["isPlatformAdmin"] is True
|
|
||||||
assert any(
|
|
||||||
m[0] is UserInDB and m[2] == {"isPlatformAdmin": True}
|
|
||||||
for m in db._modifies
|
|
||||||
), "Expected UserInDB.isPlatformAdmin promotion call"
|
|
||||||
|
|
||||||
# All three artefact tables had their rows deleted.
|
|
||||||
deletedModels = {m[0] for m in db._deletes}
|
|
||||||
assert UserMandateRole in deletedModels, "UserMandateRole row not deleted"
|
|
||||||
assert AccessRule in deletedModels, "AccessRule row not deleted"
|
|
||||||
assert Role in deletedModels, "Sysadmin Role record not deleted"
|
|
||||||
|
|
||||||
# And the seeded lists are empty after the migration.
|
|
||||||
assert seed["umRoleRows"] == []
|
|
||||||
assert seed["accessRules"] == []
|
|
||||||
assert seed["sysadminRoles"] == []
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# AC #10 — idempotent: a second run is a no-op
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
def testMigrationIsIdempotent():
|
|
||||||
"""AC#10: a second invocation finds no sysadmin role and exits silently."""
|
|
||||||
seed = _seed()
|
|
||||||
db = _buildFakeDb(**seed)
|
|
||||||
|
|
||||||
_migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
|
|
||||||
firstModifies = list(db._modifies)
|
|
||||||
firstDeletes = list(db._deletes)
|
|
||||||
|
|
||||||
_migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
|
|
||||||
|
|
||||||
# No additional writes on the second call.
|
|
||||||
assert db._modifies == firstModifies, (
|
|
||||||
"Second migration call must not perform additional writes"
|
|
||||||
)
|
|
||||||
assert db._deletes == firstDeletes, (
|
|
||||||
"Second migration call must not perform additional deletes"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def testMigrationSkipsAlreadyPromotedUsers():
|
|
||||||
"""If a user already has ``isPlatformAdmin=True``, no redundant write."""
|
|
||||||
seed = _seed()
|
|
||||||
seed["users"][0]["isPlatformAdmin"] = True # already promoted
|
|
||||||
db = _buildFakeDb(**seed)
|
|
||||||
|
|
||||||
_migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
|
|
||||||
|
|
||||||
# No promotion write for an already-promoted user.
|
|
||||||
promotionWrites = [
|
|
||||||
m for m in db._modifies
|
|
||||||
if m[0] is UserInDB and m[2].get("isPlatformAdmin") is True
|
|
||||||
]
|
|
||||||
assert promotionWrites == [], (
|
|
||||||
"Should not re-write isPlatformAdmin if user already has it"
|
|
||||||
)
|
|
||||||
|
|
||||||
# But role + access-rule cleanup still happens.
|
|
||||||
deletedModels = {m[0] for m in db._deletes}
|
|
||||||
assert Role in deletedModels
|
|
||||||
assert AccessRule in deletedModels
|
|
||||||
assert UserMandateRole in deletedModels
|
|
||||||
|
|
||||||
|
|
||||||
def testMigrationOnEmptyDbIsNoop():
|
|
||||||
"""No legacy sysadmin role at all -> no calls, no errors."""
|
|
||||||
db = _buildFakeDb(
|
|
||||||
sysadminRoles=[],
|
|
||||||
umRoleRows=[],
|
|
||||||
userMandateRows=[],
|
|
||||||
users=[],
|
|
||||||
accessRules=[],
|
|
||||||
)
|
|
||||||
|
|
||||||
_migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
|
|
||||||
|
|
||||||
assert db._modifies == []
|
|
||||||
assert db._deletes == []
|
|
||||||
|
|
@ -66,6 +66,17 @@ class TestResolveParameterReferences:
|
||||||
value = "Land: {{n1.country}}"
|
value = "Land: {{n1.country}}"
|
||||||
assert resolveParameterReferences(value, node_outputs) == "Land: CH"
|
assert resolveParameterReferences(value, node_outputs) == "Land: CH"
|
||||||
|
|
||||||
|
def test_legacy_string_template_loop_current_item_nested(self):
|
||||||
|
"""Same shape as executionEngine sets on loop node id during body iteration."""
|
||||||
|
node_outputs = {
|
||||||
|
"loop93": {
|
||||||
|
"currentItem": {"subject": "Hello", "body": {"content": "World"}},
|
||||||
|
"currentIndex": 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value = "Subj: {{loop93.currentItem.subject}} Body: {{loop93.currentItem.body.content}}"
|
||||||
|
assert resolveParameterReferences(value, node_outputs) == "Subj: Hello Body: World"
|
||||||
|
|
||||||
|
|
||||||
class TestWildcardIteration:
|
class TestWildcardIteration:
|
||||||
"""Phase-4 typed Bindings-Resolver: ``*`` segment iterates over a list.
|
"""Phase-4 typed Bindings-Resolver: ``*`` segment iterates over a list.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue