fix: looping node and content extraction

This commit is contained in:
Ida 2026-05-03 19:03:25 +02:00
parent 0659d0d21a
commit 51dfb007f6
15 changed files with 522 additions and 265 deletions

View file

@ -59,7 +59,9 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult"]}}, "inputPorts": {0: {"accepts": [
"FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult", "LoopItem", "TextResult",
]}},
"outputPorts": {0: {"schema": "AiResult"}}, "outputPorts": {0: {"schema": "AiResult"}},
"meta": {"icon": "mdi-magnify", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-magnify", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",
@ -79,7 +81,7 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}}, "inputPorts": {0: {"accepts": ["DocumentList", "Transit", "LoopItem"]}},
"outputPorts": {0: {"schema": "AiResult"}}, "outputPorts": {0: {"schema": "AiResult"}},
"meta": {"icon": "mdi-file-document-outline", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-file-document-outline", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",
@ -98,7 +100,7 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}}, "inputPorts": {0: {"accepts": ["DocumentList", "Transit", "LoopItem"]}},
"outputPorts": {0: {"schema": "AiResult"}}, "outputPorts": {0: {"schema": "AiResult"}},
"meta": {"icon": "mdi-translate", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-translate", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",
@ -118,7 +120,7 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["DocumentList", "Transit"]}}, "inputPorts": {0: {"accepts": ["DocumentList", "Transit", "LoopItem"]}},
"outputPorts": {0: {"schema": "DocumentList"}}, "outputPorts": {0: {"schema": "DocumentList"}},
"meta": {"icon": "mdi-file-convert", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-file-convert", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",
@ -147,7 +149,9 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult"]}}, "inputPorts": {0: {"accepts": [
"FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult", "LoopItem", "TextResult",
]}},
"outputPorts": {0: {"schema": "DocumentList"}}, "outputPorts": {0: {"schema": "DocumentList"}},
"meta": {"icon": "mdi-file-plus", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-file-plus", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",
@ -171,7 +175,9 @@ AI_NODES = [
] + _AI_COMMON_PARAMS, ] + _AI_COMMON_PARAMS,
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult"]}}, "inputPorts": {0: {"accepts": [
"FormPayload", "Transit", "AiResult", "DocumentList", "ActionResult", "LoopItem", "TextResult",
]}},
"outputPorts": {0: {"schema": "AiResult"}}, "outputPorts": {0: {"schema": "AiResult"}},
"meta": {"icon": "mdi-code-tags", "color": "#9C27B0", "usesAi": True}, "meta": {"icon": "mdi-code-tags", "color": "#9C27B0", "usesAi": True},
"_method": "ai", "_method": "ai",

View file

@ -10,25 +10,17 @@ FILE_NODES = [
"label": t("Datei erstellen"), "label": t("Datei erstellen"),
"description": t("Erstellt eine Datei aus Kontext (Text/Markdown von KI)."), "description": t("Erstellt eine Datei aus Kontext (Text/Markdown von KI)."),
"parameters": [ "parameters": [
{"name": "contentSources", "type": "json", "required": False, "frontendType": "json",
"description": t("Kontext-Quellen"), "default": []},
{"name": "outputFormat", "type": "str", "required": True, "frontendType": "select", {"name": "outputFormat", "type": "str", "required": True, "frontendType": "select",
"frontendOptions": {"options": ["docx", "pdf", "txt", "html", "md"]}, "frontendOptions": {"options": ["docx", "pdf", "txt", "html", "md"]},
"description": t("Ausgabeformat"), "default": "docx"}, "description": t("Ausgabeformat"), "default": "docx"},
{"name": "title", "type": "str", "required": False, "frontendType": "text", {"name": "title", "type": "str", "required": False, "frontendType": "text",
"description": t("Dokumenttitel")}, "description": t("Dokumenttitel")},
{"name": "templateName", "type": "str", "required": False, "frontendType": "select",
"frontendOptions": {"options": ["default", "corporate", "minimal"]},
"description": t("Stil-Vorlage")},
{"name": "language", "type": "str", "required": False, "frontendType": "select",
"frontendOptions": {"options": ["de", "en", "fr"]},
"description": t("Sprache"), "default": "de"},
{"name": "context", "type": "Any", "required": False, "frontendType": "contextBuilder", {"name": "context", "type": "Any", "required": False, "frontendType": "contextBuilder",
"description": t("Daten aus vorherigen Schritten"), "default": ""}, "description": t("Daten aus vorherigen Schritten"), "default": ""},
], ],
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["AiResult", "TextResult", "Transit", "FormPayload"]}}, "inputPorts": {0: {"accepts": ["AiResult", "TextResult", "Transit", "FormPayload", "LoopItem", "ActionResult"]}},
"outputPorts": {0: {"schema": "DocumentList"}}, "outputPorts": {0: {"schema": "DocumentList"}},
"meta": {"icon": "mdi-file-plus-outline", "color": "#2196F3", "usesAi": False}, "meta": {"icon": "mdi-file-plus-outline", "color": "#2196F3", "usesAi": False},
"_method": "file", "_method": "file",

View file

@ -3,25 +3,46 @@
from modules.shared.i18nRegistry import t from modules.shared.i18nRegistry import t
# Ports, die typische Schritt-Ausgaben durchreichen (nicht nur leerer Transit).
_FLOW_INPUT_SCHEMAS = [
"Transit",
"FormPayload",
"AiResult",
"TextResult",
"ActionResult",
"DocumentList",
"FileList",
"EmailList",
"TaskList",
"QueryResult",
"MergeResult",
"LoopItem",
"BoolResult",
"UdmDocument",
]
FLOW_NODES = [ FLOW_NODES = [
{ {
"id": "flow.ifElse", "id": "flow.ifElse",
"category": "flow", "category": "flow",
"label": t("Wenn / Sonst"), "label": t("Wenn / Sonst"),
"description": t("Verzweigung nach Bedingung"), "description": t(
"Verzweigt anhand einer Bedingung auf ein vorheriges Feld oder einen Ausdruck. "
"Die Daten vom Eingangskanal werden an den gewählten Ausgang durchgereicht."
),
"parameters": [ "parameters": [
{ {
"name": "condition", "name": "condition",
"type": "str", "type": "json",
"required": True, "required": True,
"frontendType": "condition", "frontendType": "condition",
"description": t("Bedingung"), "description": t("Bedingung: Feld aus einem vorherigen Schritt und Vergleich"),
}, },
], ],
"inputs": 1, "inputs": 1,
"outputs": 2, "outputs": 2,
"outputLabels": [t("Ja"), t("Nein")], "outputLabels": [t("Ja"), t("Nein")],
"inputPorts": {0: {"accepts": ["Transit"]}}, "inputPorts": {0: {"accepts": list(_FLOW_INPUT_SCHEMAS)}},
"outputPorts": {0: {"schema": "Transit"}, 1: {"schema": "Transit"}}, "outputPorts": {0: {"schema": "Transit"}, 1: {"schema": "Transit"}},
"executor": "flow", "executor": "flow",
"meta": {"icon": "mdi-source-branch", "color": "#FF9800", "usesAi": False}, "meta": {"icon": "mdi-source-branch", "color": "#FF9800", "usesAi": False},
@ -30,26 +51,29 @@ FLOW_NODES = [
"id": "flow.switch", "id": "flow.switch",
"category": "flow", "category": "flow",
"label": t("Switch"), "label": t("Switch"),
"description": t("Mehrere Zweige nach Wert"), "description": t(
"Mehrere Zweige nach einem Wert aus einem vorherigen Schritt (Data Picker). "
"Definiere Fälle mit Vergleichsoperator; der Eingang wird an den ersten passenden Zweig durchgereicht."
),
"parameters": [ "parameters": [
{ {
"name": "value", "name": "value",
"type": "str", "type": "Any",
"required": True, "required": True,
"frontendType": "text", "frontendType": "dataRef",
"description": t("Zu vergleichender Wert"), "description": t("Wert zum Vergleichen (Feld aus einem vorherigen Schritt)"),
}, },
{ {
"name": "cases", "name": "cases",
"type": "array", "type": "array",
"required": False, "required": False,
"frontendType": "caseList", "frontendType": "caseList",
"description": t("Fälle"), "description": t("Fälle: Operator und Vergleichswert"),
}, },
], ],
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["Transit"]}}, "inputPorts": {0: {"accepts": list(_FLOW_INPUT_SCHEMAS)}},
"outputPorts": {0: {"schema": "Transit"}}, "outputPorts": {0: {"schema": "Transit"}},
"executor": "flow", "executor": "flow",
"meta": {"icon": "mdi-swap-horizontal", "color": "#FF9800", "usesAi": False}, "meta": {"icon": "mdi-swap-horizontal", "color": "#FF9800", "usesAi": False},
@ -57,15 +81,18 @@ FLOW_NODES = [
{ {
"id": "flow.loop", "id": "flow.loop",
"category": "flow", "category": "flow",
"label": t("Schleife / Für Jedes"), "label": t("Schleife / Für jedes"),
"description": t("Über Array-Elemente oder UDM-Strukturebenen iterieren"), "description": t(
"Iteriert über ein Array aus einem vorherigen Schritt (z. B. documente, Zeilen, Listeneinträge). "
"Optional: UDM-Ebene für strukturierte Dokumente."
),
"parameters": [ "parameters": [
{ {
"name": "items", "name": "items",
"type": "str", "type": "Any",
"required": True, "required": True,
"frontendType": "text", "frontendType": "dataRef",
"description": t("Pfad zum Array"), "description": t("Liste oder Sammlung zum Durchlaufen (im Data Picker wählen)"),
}, },
{ {
"name": "level", "name": "level",
@ -73,7 +100,7 @@ FLOW_NODES = [
"required": False, "required": False,
"frontendType": "select", "frontendType": "select",
"frontendOptions": {"options": ["auto", "documents", "structuralNodes", "contentBlocks"]}, "frontendOptions": {"options": ["auto", "documents", "structuralNodes", "contentBlocks"]},
"description": t("UDM-Iterationsebene"), "description": t("Nur bei UDM-Daten: welche Strukturebene als Elemente verwendet wird"),
"default": "auto", "default": "auto",
}, },
{ {
@ -82,14 +109,15 @@ FLOW_NODES = [
"required": False, "required": False,
"frontendType": "number", "frontendType": "number",
"frontendOptions": {"min": 1, "max": 20}, "frontendOptions": {"min": 1, "max": 20},
"description": t("Parallele Iterationen (1 = sequentiell)"), "description": t("Parallele Durchläufe (1 = nacheinander)"),
"default": 1, "default": 1,
}, },
], ],
"inputs": 1, "inputs": 1,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": [ "inputPorts": {0: {"accepts": [
"Transit", "UdmDocument", "EmailList", "DocumentList", "FileList", "TaskList", "ActionResult", "Transit", "UdmDocument", "EmailList", "DocumentList", "FileList", "TaskList",
"ActionResult", "AiResult", "QueryResult", "FormPayload",
]}}, ]}},
"outputPorts": {0: {"schema": "LoopItem"}}, "outputPorts": {0: {"schema": "LoopItem"}},
"executor": "flow", "executor": "flow",
@ -99,7 +127,10 @@ FLOW_NODES = [
"id": "flow.merge", "id": "flow.merge",
"category": "flow", "category": "flow",
"label": t("Zusammenführen"), "label": t("Zusammenführen"),
"description": t("Mehrere Zweige zusammenführen (2-5 Eingänge)"), "description": t(
"Führt 25 Zweige zusammen, wenn alle verbunden sind. "
"Modus legt fest, wie die Eingabeobjekte im Ergebnis kombiniert werden."
),
"parameters": [ "parameters": [
{ {
"name": "mode", "name": "mode",
@ -107,7 +138,7 @@ FLOW_NODES = [
"required": False, "required": False,
"frontendType": "select", "frontendType": "select",
"frontendOptions": {"options": ["first", "all", "append"]}, "frontendOptions": {"options": ["first", "all", "append"]},
"description": t("Zusammenführungsmodus"), "description": t("first: erster Zweig; all: Dict-Felder zusammenführen; append: Listen anhängen"),
"default": "first", "default": "first",
}, },
{ {
@ -116,13 +147,16 @@ FLOW_NODES = [
"required": False, "required": False,
"frontendType": "number", "frontendType": "number",
"frontendOptions": {"min": 2, "max": 5}, "frontendOptions": {"min": 2, "max": 5},
"description": t("Anzahl Eingänge"), "description": t("Anzahl Eingänge dieses Nodes (25)"),
"default": 2, "default": 2,
}, },
], ],
"inputs": 2, "inputs": 2,
"outputs": 1, "outputs": 1,
"inputPorts": {0: {"accepts": ["Transit"]}, 1: {"accepts": ["Transit"]}}, "inputPorts": {
0: {"accepts": list(_FLOW_INPUT_SCHEMAS)},
1: {"accepts": list(_FLOW_INPUT_SCHEMAS)},
},
"outputPorts": {0: {"schema": "MergeResult"}}, "outputPorts": {0: {"schema": "MergeResult"}},
"executor": "flow", "executor": "flow",
"meta": {"icon": "mdi-call-merge", "color": "#FF9800", "usesAi": False}, "meta": {"icon": "mdi-call-merge", "color": "#FF9800", "usesAi": False},

View file

@ -644,6 +644,69 @@ def resolveSystemVariable(variable: str, context: Dict[str, Any]) -> Any:
# Output normalizers # Output normalizers
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _file_record_to_document(f: Any) -> Optional[Dict[str, Any]]:
"""Map API / task-upload file dicts onto PortSchema ``Document`` fields."""
if f is None:
return None
if isinstance(f, str) and f.strip():
return {"id": f.strip()}
if not isinstance(f, dict):
return None
inner = f.get("file") if isinstance(f.get("file"), dict) else None
src = inner or f
out: Dict[str, Any] = {}
fid = src.get("id") or f.get("id")
if fid is not None and str(fid).strip():
out["id"] = str(fid).strip()
name = (
src.get("name")
or src.get("fileName")
or f.get("fileName")
or f.get("name")
)
if name is not None and str(name).strip():
out["name"] = str(name).strip()
mime = src.get("mimeType") or src.get("mime") or f.get("mimeType")
if mime is not None and str(mime).strip():
out["mimeType"] = str(mime).strip()
for k in ("sizeBytes", "downloadUrl", "filePath"):
v = src.get(k) if k in src else f.get(k)
if v is not None and v != "":
out[k] = v
return out if out else None
def _coerce_document_list_upload_fields(result: Dict[str, Any]) -> None:
"""
Human task ``input.upload`` completes with ``file`` / ``files`` / ``fileIds``.
DocumentList expects ``documents``. Without this, resume adds ``documents: []`` and drops the real files.
"""
docs = result.get("documents")
if isinstance(docs, list) and len(docs) > 0:
return
collected: List[Dict[str, Any]] = []
files = result.get("files")
if isinstance(files, list):
for item in files:
d = _file_record_to_document(item)
if d:
collected.append(d)
if not collected:
single = result.get("file")
d = _file_record_to_document(single)
if d:
collected.append(d)
if not collected and isinstance(result.get("fileIds"), list):
for fid in result["fileIds"]:
if fid is not None and str(fid).strip():
collected.append({"id": str(fid).strip()})
if not collected:
return
result["documents"] = collected
if not result.get("count"):
result["count"] = len(collected)
def normalizeToSchema(raw: Any, schemaName: str) -> Dict[str, Any]: def normalizeToSchema(raw: Any, schemaName: str) -> Dict[str, Any]:
""" """
Normalize raw executor output to match the declared port schema. Normalize raw executor output to match the declared port schema.
@ -660,6 +723,9 @@ def normalizeToSchema(raw: Any, schemaName: str) -> Dict[str, Any]:
if not schema or schemaName == "Transit": if not schema or schemaName == "Transit":
return result return result
if schemaName == "DocumentList":
_coerce_document_list_upload_fields(result)
# Only default **required** fields. Optional fields stay absent so DataRefs / context # Only default **required** fields. Optional fields stay absent so DataRefs / context
# resolution never pick a synthetic `{}` or `[]` (e.g. AiResult.responseData when the # resolution never pick a synthetic `{}` or `[]` (e.g. AiResult.responseData when the
# model returned plain text only). # model returned plain text only).

View file

@ -57,8 +57,7 @@ from .subJsonResponseHandling import JsonResponseHandler
from .subLoopingUseCases import LoopingUseCaseRegistry from .subLoopingUseCases import LoopingUseCaseRegistry
from modules.workflows.processing.shared.stateTools import checkWorkflowStopped from modules.workflows.processing.shared.stateTools import checkWorkflowStopped
from modules.shared.jsonContinuation import getContexts from modules.shared.jsonContinuation import getContexts
from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson from modules.shared.jsonUtils import buildContinuationContext, tryParseJson
from modules.shared.jsonUtils import tryParseJson
from modules.shared.jsonUtils import closeJsonStructures from modules.shared.jsonUtils import closeJsonStructures
from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
@ -374,9 +373,8 @@ class AiCallLooper:
if lastValidCompletePart: if lastValidCompletePart:
try: try:
extracted = extractJsonString(lastValidCompletePart) parsed, parseErr, _ = tryParseJson(lastValidCompletePart)
parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None:
if parseErr is None and parsed:
normalized = self._normalizeJsonStructure(parsed, useCase) normalized = self._normalizeJsonStructure(parsed, useCase)
return json.dumps(normalized, indent=2, ensure_ascii=False) return json.dumps(normalized, indent=2, ensure_ascii=False)
except Exception: except Exception:
@ -404,11 +402,10 @@ class AiCallLooper:
# This ensures retry iterations use the correct base context # This ensures retry iterations use the correct base context
lastRawResponse = candidateJson lastRawResponse = candidateJson
# Try direct parse of candidate # Try direct parse of candidate (same pipeline as structure filling / getContexts)
try: try:
extracted = extractJsonString(candidateJson) parsed, parseErr, extracted = tryParseJson(candidateJson)
parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None:
if parseErr is None and parsed:
# Direct parse succeeded - FINISHED # Direct parse succeeded - FINISHED
# Commit candidate to jsonBase # Commit candidate to jsonBase
jsonBase = candidateJson jsonBase = candidateJson
@ -441,21 +438,18 @@ class AiCallLooper:
# STEP 6: DECIDE based on jsonParsingSuccess and overlapContext # STEP 6: DECIDE based on jsonParsingSuccess and overlapContext
if contexts.jsonParsingSuccess and contexts.overlapContext == "": if contexts.jsonParsingSuccess and contexts.overlapContext == "":
# JSON is complete (no cut point) - FINISHED # getContexts and downstream must agree with tryParseJson (same as structure filling).
# Use completePart for final result (closed, repaired JSON)
# No more merging needed, so we don't need the cut version
jsonBase = contexts.completePart
logger.info(f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext='', JSON complete") logger.info(f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext='', JSON complete")
# Store and parse completePart
lastValidCompletePart = contexts.completePart lastValidCompletePart = contexts.completePart
try: try:
extracted = extractJsonString(contexts.completePart) parsed, parseErr, extracted = tryParseJson(contexts.completePart)
parsed, parseErr, _ = tryParseJson(extracted) if parseErr is not None:
if parseErr is None and parsed: raise ValueError(str(parseErr))
normalized = self._normalizeJsonStructure(parsed, useCase) normalized = self._normalizeJsonStructure(parsed, useCase)
result = json.dumps(normalized, indent=2, ensure_ascii=False) result = json.dumps(normalized, indent=2, ensure_ascii=False)
jsonBase = contexts.completePart
if iterationOperationId: if iterationOperationId:
self.services.chat.progressLogFinish(iterationOperationId, True) self.services.chat.progressLogFinish(iterationOperationId, True)
@ -468,12 +462,26 @@ class AiCallLooper:
result, normalized, extracted, debugPrefix, self.services result, normalized, extracted, debugPrefix, self.services
) )
except Exception as e: except Exception as e:
logger.warning(f"Iteration {iteration}: Failed to parse completePart: {e}") logger.warning(
f"Iteration {iteration}: completePart not serializable after getContexts success: {e}"
# Fallback: return completePart as-is )
mergeFailCount += 1
if mergeFailCount >= MAX_MERGE_FAILS:
logger.error(
f"Iteration {iteration}: Max failures ({MAX_MERGE_FAILS}) "
"after output pipeline mismatch"
)
if iterationOperationId: if iterationOperationId:
self.services.chat.progressLogFinish(iterationOperationId, False)
return jsonBase if jsonBase else ""
if iterationOperationId:
self.services.chat.progressLogUpdate(
iterationOperationId,
0.7,
f"Output pipeline failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying",
)
self.services.chat.progressLogFinish(iterationOperationId, True) self.services.chat.progressLogFinish(iterationOperationId, True)
return contexts.completePart continue
elif contexts.jsonParsingSuccess and contexts.overlapContext != "": elif contexts.jsonParsingSuccess and contexts.overlapContext != "":
# JSON parseable but has cut point - CONTINUE to next iteration # JSON parseable but has cut point - CONTINUE to next iteration
@ -522,9 +530,8 @@ class AiCallLooper:
if lastValidCompletePart: if lastValidCompletePart:
try: try:
extracted = extractJsonString(lastValidCompletePart) parsed, parseErr, _ = tryParseJson(lastValidCompletePart)
parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None:
if parseErr is None and parsed:
normalized = self._normalizeJsonStructure(parsed, useCase) normalized = self._normalizeJsonStructure(parsed, useCase)
return json.dumps(normalized, indent=2, ensure_ascii=False) return json.dumps(normalized, indent=2, ensure_ascii=False)
except Exception: except Exception:
@ -552,9 +559,24 @@ class AiCallLooper:
if iteration >= maxIterations: if iteration >= maxIterations:
logger.warning(f"AI call stopped after maximum iterations ({maxIterations})") logger.warning(f"AI call stopped after maximum iterations ({maxIterations})")
# This code path should never be reached because all registered use cases # Prefer last repaired complete JSON from getContexts (raw `result` is only the last fragment).
# return early when JSON is complete. This would only execute for use cases that if lastValidCompletePart and useCase and not useCase.requiresExtraction:
# require section extraction, but no such use cases are currently registered. try:
parsed, parseErr, extracted = tryParseJson(lastValidCompletePart)
if parseErr is None:
normalized = self._normalizeJsonStructure(parsed, useCase)
out = json.dumps(normalized, indent=2, ensure_ascii=False)
if useCase.finalResultHandler:
logger.warning(
"callAiWithLooping: max iterations — returning last valid completePart for %r",
useCaseId,
)
return useCase.finalResultHandler(
out, normalized, extracted, debugPrefix, self.services
)
except Exception as e:
logger.debug("Max-iterations fallback on completePart failed: %s", e)
logger.error( logger.error(
"End of callAiWithLooping without success for use case %r (iterations=%s, lastResultLen=%s)", "End of callAiWithLooping without success for use case %r (iterations=%s, lastResultLen=%s)",
useCaseId, useCaseId,

View file

@ -54,6 +54,15 @@ def _handleCodeContentFinalResult(result: str, parsedJsonForUseCase: Any, extrac
return final_json return final_json
def _lift_section_plain_text(d: Dict[str, Any]) -> Optional[str]:
"""Models often return {\"text\": \"...\"} without an elements array; extract usable prose."""
for key in ("text", "body", "summary", "response", "output", "answer", "message", "content"):
v = d.get(key)
if isinstance(v, str) and v.strip():
return v.strip()
return None
def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any: def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any:
"""Normalize JSON structure for section_content use case.""" """Normalize JSON structure for section_content use case."""
# For section_content, expect {"elements": [...]} structure # For section_content, expect {"elements": [...]} structure
@ -77,14 +86,28 @@ def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any:
# Convert plain list of elements to elements structure # Convert plain list of elements to elements structure
return {"elements": parsed} return {"elements": parsed}
elif isinstance(parsed, dict): elif isinstance(parsed, dict):
# If it already has "elements", return as-is
if "elements" in parsed: if "elements" in parsed:
els = parsed.get("elements")
if isinstance(els, list) and len(els) > 0:
return parsed return parsed
# If it has "type" and looks like an element, wrap in elements array lifted = _lift_section_plain_text(parsed)
elif parsed.get("type"): if lifted:
out = dict(parsed)
out["elements"] = [{"type": "paragraph", "content": {"text": lifted}}]
logger.info(
"section_content: promoted plain-text field to elements (%d chars)",
len(lifted),
)
return out
return parsed
if parsed.get("type"):
return {"elements": [parsed]} return {"elements": [parsed]}
# Otherwise, assume it's already in correct format lifted = _lift_section_plain_text(parsed)
else: if lifted:
return {
**parsed,
"elements": [{"type": "paragraph", "content": {"text": lifted}}],
}
return parsed return parsed
# For other use cases, return as-is (they have their own structures) # For other use cases, return as-is (they have their own structures)

View file

@ -27,6 +27,36 @@ class _AiResponseFallback:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _elements_from_section_content_ai_json(parsed: Any) -> List[Any]:
"""Normalize section_content AI JSON (incl. models that return {\"text\": ...}) into elements."""
from modules.serviceCenter.services.serviceAi.subLoopingUseCases import _normalizeSectionContentJson
if parsed is None:
return []
if isinstance(parsed, dict):
has_nonempty_elements = (
isinstance(parsed.get("elements"), list) and len(parsed["elements"]) > 0
)
if not has_nonempty_elements:
# Valid full-document envelope (same normalized shape the renderer uses elsewhere)
docs = parsed.get("documents")
if isinstance(docs, list) and docs and isinstance(docs[0], dict):
secs = docs[0].get("sections")
if isinstance(secs, list) and secs and isinstance(secs[0], dict):
parsed = secs[0]
elif (
isinstance(parsed.get("sections"), list)
and parsed["sections"]
and isinstance(parsed["sections"][0], dict)
):
parsed = parsed["sections"][0]
norm = _normalizeSectionContentJson(parsed, "section_content")
if isinstance(norm, dict):
els = norm.get("elements")
return list(els) if isinstance(els, list) else []
return []
class StructureFiller: class StructureFiller:
"""Handles filling document structure with content.""" """Handles filling document structure with content."""
@ -524,36 +554,10 @@ class StructureFiller:
if generatedElements: if generatedElements:
elements.extend(generatedElements) elements.extend(generatedElements)
else: else:
# Fallback: Try to parse JSON response directly with repair logic logger.error(f"No elements produced for section {sectionId} (callAiWithLooping must return parseable JSON)")
try:
from modules.shared.jsonUtils import tryParseJson, repairBrokenJson
# Use tryParseJson which handles extraction and basic parsing
fallbackElements, parseError, cleanedStr = tryParseJson(aiResponse.content)
# If parsing failed, try repair
if parseError and isinstance(aiResponse.content, str):
logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}")
repairedJson = repairBrokenJson(aiResponse.content)
if repairedJson:
fallbackElements = repairedJson
parseError = None
logger.info(f"Successfully repaired JSON for section {sectionId}")
if parseError:
raise parseError
if isinstance(fallbackElements, list):
elements.extend(fallbackElements)
elif isinstance(fallbackElements, dict) and "elements" in fallbackElements:
elements.extend(fallbackElements["elements"])
elif isinstance(fallbackElements, dict) and fallbackElements.get("type"):
elements.append(fallbackElements)
except (json.JSONDecodeError, ValueError) as json_error:
logger.error(f"Error parsing JSON response for section {sectionId}: {str(json_error)}")
elements.append({ elements.append({
"type": "error", "type": "error",
"message": f"Failed to parse JSON response: {str(json_error)}", "message": f"No parsed content for section {sectionId}",
"sectionId": sectionId "sectionId": sectionId
}) })
@ -671,7 +675,7 @@ class StructureFiller:
try: try:
self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation")
operationType = OperationTypeEnum.DATA_ANALYSE operationType = OperationTypeEnum.DATA_GENERATE
options = AiCallOptions( options = AiCallOptions(
operationType=operationType, operationType=operationType,
priority=PriorityEnum.BALANCED, priority=PriorityEnum.BALANCED,
@ -703,22 +707,17 @@ class StructureFiller:
) )
try: try:
from modules.shared.jsonUtils import tryParseJson, repairBrokenJson from modules.shared.jsonUtils import tryParseJson
if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1): if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1):
generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId) generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId)
else: else:
parsedResponse, parseError, cleanedStr = tryParseJson(aiResponseJson) parsedResponse, parseError, _ = tryParseJson(aiResponseJson)
if parsedResponse is None: if parseError is not None:
logger.warning(f"Section {sectionId}: tryParseJson failed, attempting repair") logger.error(f"Section {sectionId}: tryParseJson failed: {parseError}")
repairedStr = repairBrokenJson(aiResponseJson)
parsedResponse, parseError2, _ = tryParseJson(repairedStr)
if parsedResponse and isinstance(parsedResponse, dict):
generatedElements = parsedResponse.get("elements", [])
elif parsedResponse and isinstance(parsedResponse, list):
generatedElements = parsedResponse
else:
generatedElements = [] generatedElements = []
else:
generatedElements = _elements_from_section_content_ai_json(parsedResponse)
except Exception as parseErr: except Exception as parseErr:
logger.error(f"Section {sectionId}: JSON parse error: {parseErr}") logger.error(f"Section {sectionId}: JSON parse error: {parseErr}")
generatedElements = [] generatedElements = []
@ -930,7 +929,7 @@ class StructureFiller:
self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation")
operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_GENERATE
if operationType == OperationTypeEnum.IMAGE_GENERATE: if operationType == OperationTypeEnum.IMAGE_GENERATE:
maxPromptLength = 4000 maxPromptLength = 4000
@ -996,43 +995,16 @@ class StructureFiller:
) )
try: try:
# Use tryParseJson which handles extraction and basic parsing from modules.shared.jsonUtils import tryParseJson
from modules.shared.jsonUtils import tryParseJson, repairBrokenJson
# Check if response contains multiple JSON blocks (separated by --- or multiple ```json blocks)
# This can happen when AI returns multiple complete responses
if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1): if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1):
logger.info(f"Section {sectionId}: Detected multiple JSON blocks in response, attempting to merge") logger.info(f"Section {sectionId}: Detected multiple JSON blocks in response, attempting to merge")
generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId) generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId)
else: else:
parsedResponse, parseError, cleanedStr = tryParseJson(aiResponseJson) parsedResponse, parseError, _ = tryParseJson(aiResponseJson)
if parseError is not None:
# If parsing failed, try repair
if parseError and isinstance(aiResponseJson, str):
logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}")
repairedJson = repairBrokenJson(aiResponseJson)
if repairedJson:
parsedResponse = repairedJson
parseError = None
logger.info(f"Successfully repaired JSON for section {sectionId}")
if parseError:
raise parseError raise parseError
generatedElements = _elements_from_section_content_ai_json(parsedResponse)
if isinstance(parsedResponse, list):
generatedElements = parsedResponse
elif isinstance(parsedResponse, dict):
if "elements" in parsedResponse:
generatedElements = parsedResponse["elements"]
elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0:
firstSection = parsedResponse["sections"][0]
generatedElements = firstSection.get("elements", [])
elif parsedResponse.get("type"):
generatedElements = [parsedResponse]
else:
generatedElements = []
else:
generatedElements = []
aiResponse = _AiResponseFallback(aiResponseJson) aiResponse = _AiResponseFallback(aiResponseJson)
except Exception as parseError: except Exception as parseError:
@ -1112,7 +1084,7 @@ class StructureFiller:
self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation")
operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_GENERATE
if operationType == OperationTypeEnum.IMAGE_GENERATE: if operationType == OperationTypeEnum.IMAGE_GENERATE:
maxPromptLength = 4000 maxPromptLength = 4000
@ -1135,6 +1107,7 @@ class StructureFiller:
processingMode=ProcessingModeEnum.DETAILED processingMode=ProcessingModeEnum.DETAILED
) )
) )
checkWorkflowStopped(self.services)
aiResponse = await self.aiService.callAi(request) aiResponse = await self.aiService.callAi(request)
generatedElements = [] generatedElements = []
@ -1179,22 +1152,16 @@ class StructureFiller:
) )
try: try:
parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) from modules.shared.jsonUtils import tryParseJson
if isinstance(parsedResponse, list):
generatedElements = parsedResponse
elif isinstance(parsedResponse, dict):
if "elements" in parsedResponse:
generatedElements = parsedResponse["elements"]
elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0:
firstSection = parsedResponse["sections"][0]
generatedElements = firstSection.get("elements", [])
elif parsedResponse.get("type"):
generatedElements = [parsedResponse]
else:
generatedElements = []
else:
generatedElements = []
parsedResponse, parseError, _ = tryParseJson(aiResponseJson)
if parseError is not None:
logger.error(
f"Error parsing response from _callAiWithLooping for section {sectionId}: {parseError}"
)
generatedElements = []
else:
generatedElements = _elements_from_section_content_ai_json(parsedResponse)
aiResponse = _AiResponseFallback(aiResponseJson) aiResponse = _AiResponseFallback(aiResponseJson)
except Exception as parseError: except Exception as parseError:
logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}")
@ -1371,7 +1338,7 @@ class StructureFiller:
self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation")
operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_GENERATE
if operationType == OperationTypeEnum.IMAGE_GENERATE: if operationType == OperationTypeEnum.IMAGE_GENERATE:
maxPromptLength = 4000 maxPromptLength = 4000
@ -1439,22 +1406,16 @@ class StructureFiller:
) )
try: try:
parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) from modules.shared.jsonUtils import tryParseJson
if isinstance(parsedResponse, list):
generatedElements = parsedResponse
elif isinstance(parsedResponse, dict):
if "elements" in parsedResponse:
generatedElements = parsedResponse["elements"]
elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0:
firstSection = parsedResponse["sections"][0]
generatedElements = firstSection.get("elements", [])
elif parsedResponse.get("type"):
generatedElements = [parsedResponse]
else:
generatedElements = []
else:
generatedElements = []
parsedResponse, parseError, _ = tryParseJson(aiResponseJson)
if parseError is not None:
logger.error(
f"Error parsing response from _callAiWithLooping for section {sectionId}: {parseError}"
)
generatedElements = []
else:
generatedElements = _elements_from_section_content_ai_json(parsedResponse)
aiResponse = _AiResponseFallback(aiResponseJson) aiResponse = _AiResponseFallback(aiResponseJson)
except Exception as parseError: except Exception as parseError:
logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}")

View file

@ -40,6 +40,26 @@ class ChatService:
"""Workflow from context (stable during workflow execution).""" """Workflow from context (stable during workflow execution)."""
return self._context.workflow return self._context.workflow
def _chat_document_from_management_file(self, file_id: str) -> Optional[ChatDocument]:
"""Build a ChatDocument when docItem references a management FileItem (e.g. automation uploads) without a chat message."""
try:
fi = self.interfaceDbComponent.getFile(file_id)
except Exception as e:
logger.debug("getFile(%s) failed: %s", file_id, e)
return None
if fi is None:
return None
wf = self._workflow
wf_id = wf.id if wf else "no-workflow"
return ChatDocument(
id=file_id,
messageId=f"_filestore:{wf_id}",
fileId=fi.id,
fileName=fi.fileName or "document",
fileSize=int(fi.fileSize or 0),
mimeType=fi.mimeType or "application/octet-stream",
)
def getChatDocumentsFromDocumentList(self, documentList) -> List[ChatDocument]: def getChatDocumentsFromDocumentList(self, documentList) -> List[ChatDocument]:
"""Get ChatDocuments from a DocumentReferenceList. """Get ChatDocuments from a DocumentReferenceList.
@ -130,14 +150,28 @@ class ChatService:
if message.documents: if message.documents:
for doc in message.documents: for doc in message.documents:
if doc.id == docId: if doc.id == docId or getattr(doc, "fileId", None) == docId:
allDocuments.append(doc) allDocuments.append(doc)
docFound = True docFound = True
logger.debug(f"Matched document reference '{docRef}' to document {doc.id} (fileName: {getattr(doc, 'fileName', 'unknown')}) by documentId") logger.debug(
f"Matched document reference '{docRef}' to document {doc.id} "
f"(fileName: {getattr(doc, 'fileName', 'unknown')}) by id/fileId"
)
break break
if docFound: if docFound:
break break
if not docFound:
synth = self._chat_document_from_management_file(docId)
if synth is not None:
allDocuments.append(synth)
docFound = True
logger.info(
"Resolved document reference %r via FileItem %s (automation / transient workflow)",
docRef,
docId,
)
# Fallback: If not found by documentId and it looks like a filename (has file extension), try filename matching # Fallback: If not found by documentId and it looks like a filename (has file extension), try filename matching
# This handles cases where AI incorrectly generates docItem:filename.docx # This handles cases where AI incorrectly generates docItem:filename.docx
if not docFound and '.' in docId and len(parts) == 2: if not docFound and '.' in docId and len(parts) == 2:

View file

@ -196,8 +196,10 @@ class RendererText(BaseRenderer):
textParts.append(f"[Reference: {label}]") textParts.append(f"[Reference: {label}]")
continue continue
elif element_type == "extracted_text": elif element_type == "extracted_text":
# Extracted text format # Extracted text format (str or raw bytes from ContentPart)
content = element.get("content", "") content = element.get("content", "")
if isinstance(content, (bytes, bytearray, memoryview)):
content = bytes(content).decode("utf-8", errors="replace")
source = element.get("source", "") source = element.get("source", "")
if content: if content:
source_text = f" (Source: {source})" if source else "" source_text = f" (Source: {source})" if source else ""
@ -323,22 +325,27 @@ class RendererText(BaseRenderer):
try: try:
# Extract from nested content structure: element.content.{text, level} # Extract from nested content structure: element.content.{text, level}
content = headingData.get("content", {}) content = headingData.get("content", {})
if not isinstance(content, dict): if isinstance(content, dict) and content:
return ""
text = self._stripMarkdownForPlainText(content.get("text", "")) text = self._stripMarkdownForPlainText(content.get("text", ""))
level = content.get("level", 1) level = content.get("level", 1)
if text:
level = max(1, min(6, level))
if level == 1:
return f"{text}\n{'=' * len(text)}"
elif level == 2:
return f"{text}\n{'-' * len(text)}"
else: else:
return f"{'#' * level} {text}" # AI shorthand: {"type":"heading","text":"...","level":2}
text = self._stripMarkdownForPlainText(str(headingData.get("text", "") or ""))
level = headingData.get("level", 1)
if not text:
return "" return ""
try:
level_i = int(level) if level is not None else 1
except (TypeError, ValueError):
level_i = 1
level_i = max(1, min(6, level_i))
if level_i == 1:
return f"{text}\n{'=' * len(text)}"
if level_i == 2:
return f"{text}\n{'-' * len(text)}"
return f"{'#' * level_i} {text}"
except Exception as e: except Exception as e:
self.logger.warning(f"Error rendering heading: {str(e)}") self.logger.warning(f"Error rendering heading: {str(e)}")
return "" return ""
@ -399,8 +406,19 @@ class RendererText(BaseRenderer):
def _renderJsonParagraph(self, paragraphData: Dict[str, Any]) -> str: def _renderJsonParagraph(self, paragraphData: Dict[str, Any]) -> str:
"""Render a JSON paragraph to text. Strips markdown for plain text output.""" """Render a JSON paragraph to text. Strips markdown for plain text output."""
try: try:
# Extract from nested content structure # Models often return {"type":"paragraph","text":"..."} without nested "content"
content = paragraphData.get("content", {}) top = paragraphData.get("text")
raw_content = paragraphData.get("content", {})
if isinstance(top, str) and top.strip():
if raw_content is None or raw_content == {}:
return self._stripMarkdownForPlainText(top)
if isinstance(raw_content, dict):
if not (raw_content.get("text") or raw_content.get("inlineRuns")):
return self._stripMarkdownForPlainText(top)
content = raw_content
if content is None:
content = {}
if isinstance(content, dict): if isinstance(content, dict):
runs = self._inlineRunsFromContent(content) runs = self._inlineRunsFromContent(content)
if runs: if runs:

View file

@ -2172,11 +2172,13 @@ def getContexts(
>>> print(contexts.overlapContext) # "" (empty - JSON is complete) >>> print(contexts.overlapContext) # "" (empty - JSON is complete)
>>> print(contexts.jsonParsingSuccess) # True >>> print(contexts.jsonParsingSuccess) # True
""" """
# First, check if original JSON is already complete (parseable without modification) # Completeness must use the same pipeline as callers (fences, balanced extract, normalization).
from modules.shared.jsonUtils import tryParseJson as _utils_try_parse_json
jsonIsComplete = False jsonIsComplete = False
if truncatedJson and truncatedJson.strip(): if truncatedJson and truncatedJson.strip():
parsed, error = _tryParseJson(truncatedJson.strip()) _parsed_hdr, error_hdr, _ = _utils_try_parse_json(truncatedJson)
if error is None: if error_hdr is None:
jsonIsComplete = True jsonIsComplete = True
logger.debug("Original JSON is already complete (no cut point)") logger.debug("Original JSON is already complete (no cut point)")
@ -2193,28 +2195,27 @@ def getContexts(
jsonParsingSuccess = False jsonParsingSuccess = False
if completePart and completePart.strip(): if completePart and completePart.strip():
# First attempt: parse as-is parsed, error, _ = _utils_try_parse_json(completePart)
parsed, error = _tryParseJson(completePart)
if error is None: if error is None:
jsonParsingSuccess = True jsonParsingSuccess = True
else: else:
# Second attempt: repair internal errors and retry logger.debug(f"Initial parse failed: {error}, attempting internal repair")
logger.debug(f"Initial parse failed: {error}, attempting repair")
repairedCompletePart = _repairInternalJsonErrors(completePart) repairedCompletePart = _repairInternalJsonErrors(completePart)
parsed, error, _ = _utils_try_parse_json(repairedCompletePart)
parsed, error = _tryParseJson(repairedCompletePart)
if error is None: if error is None:
# Repair succeeded - use repaired version
completePart = repairedCompletePart completePart = repairedCompletePart
jsonParsingSuccess = True jsonParsingSuccess = True
logger.debug("JSON repair successful") logger.debug("JSON repair successful")
else: else:
# Repair also failed - keep original completePart, mark as failed
logger.debug(f"JSON repair also failed: {error}") logger.debug(f"JSON repair also failed: {error}")
jsonParsingSuccess = False jsonParsingSuccess = False
# If completePart parses successfully, the merged/candidate JSON is structurally complete
# after repair/closing — overlap from extractContinuationContexts on the *raw* candidate
# would falsely signal truncation and trap callAiWithLooping in continuation iterations.
if jsonParsingSuccess:
overlap = ""
return JsonContinuationContexts( return JsonContinuationContexts(
overlapContext=overlap, overlapContext=overlap,
hierarchyContext=hierarchy, hierarchyContext=hierarchy,

View file

@ -393,9 +393,10 @@ async def executeGraph(
ordered_ids = [n.get("id") for n in ordered if n.get("id")] ordered_ids = [n.get("id") for n in ordered if n.get("id")]
logger.info("executeGraph topoSort order: %s", ordered_ids) logger.info("executeGraph topoSort order: %s", ordered_ids)
nodeOutputs: Dict[str, Any] = dict(initialNodeOutputs or {}) # Normalize resumed human-node output BEFORE copying into nodeOutputs — otherwise
# normalizeToSchema only updates initialNodeOutputs and loop/refs still see raw
# e.g. input.upload {files} without coerced DocumentList.documents.
is_resume = startAfterNodeId is not None is_resume = startAfterNodeId is not None
if is_resume and initialNodeOutputs and startAfterNodeId: if is_resume and initialNodeOutputs and startAfterNodeId:
resumedNode = next((n for n in nodes if n.get("id") == startAfterNodeId), None) resumedNode = next((n for n in nodes if n.get("id") == startAfterNodeId), None)
if resumedNode: if resumedNode:
@ -408,6 +409,8 @@ async def executeGraph(
initialNodeOutputs[startAfterNodeId] = normalizeToSchema(resumedOutput, schema) initialNodeOutputs[startAfterNodeId] = normalizeToSchema(resumedOutput, schema)
except Exception as valErr: except Exception as valErr:
logger.warning("executeGraph resume: schema validation failed for %s: %s", startAfterNodeId, valErr) logger.warning("executeGraph resume: schema validation failed for %s: %s", startAfterNodeId, valErr)
nodeOutputs: Dict[str, Any] = dict(initialNodeOutputs or {})
if not runId and automation2_interface and workflowId and not is_resume: if not runId and automation2_interface and workflowId and not is_resume:
run_context = { run_context = {
"connectionMap": connectionMap, "connectionMap": connectionMap,

View file

@ -1,7 +1,8 @@
# Copyright (c) 2025 Patrick Motsch # Copyright (c) 2025 Patrick Motsch
# Action node executor - maps ai.*, email.*, sharepoint.*, clickup.*, file.*, trustee.* to method actions. # Action node executor - maps ai.*, email.*, sharepoint.*, clickup.*, file.*, trustee.* to method actions.
# #
# Typed Port System: explicit DataRefs / static parameters only (no runtime wire-handover). # Typed Port System: explicit DataRefs / static parameters; optional ``documentList`` from input port 0
# when the param is empty (same idea as IOExecutor wire fill).
# ``materializeConnectionRefs`` (see pickNotPushMigration) may still rewrite empty connectionReference at run start. # ``materializeConnectionRefs`` (see pickNotPushMigration) may still rewrite empty connectionReference at run start.
import json import json
@ -18,6 +19,25 @@ from modules.serviceCenter.services.serviceBilling.mainServiceBilling import Bil
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _coerce_document_data_to_bytes(raw: Any) -> Optional[bytes]:
"""Normalize documentData (bytes/str/buffer) for DB file persistence."""
if raw is None:
return None
if isinstance(raw, bytes):
return raw if len(raw) > 0 else None
if isinstance(raw, bytearray):
b = bytes(raw)
return b if len(b) > 0 else None
if isinstance(raw, memoryview):
b = raw.tobytes()
return b if len(b) > 0 else None
if isinstance(raw, str):
b = raw.encode("utf-8")
return b if len(b) > 0 else None
return None
_USER_CONNECTION_ID_RE = re.compile( _USER_CONNECTION_ID_RE = re.compile(
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$",
re.IGNORECASE, re.IGNORECASE,
@ -219,6 +239,78 @@ def _getOutputSchemaName(nodeDef: Dict) -> str:
return port0.get("schema", "ActionResult") return port0.get("schema", "ActionResult")
def _extract_wired_document_list(inp: Any) -> Optional[Dict[str, Any]]:
"""
Build a DocumentList-shaped dict from upstream node output (matches IOExecutor wire behavior).
Handles DocumentList, human upload shapes (file / files / fileIds), FileList, loop file items.
During flow.loop body execution the loop node's output is
{items, count, currentItem, currentIndex}; wired document actions must use currentItem.
"""
if inp is None:
return None
from modules.features.graphicalEditor.portTypes import (
unwrapTransit,
_coerce_document_list_upload_fields,
_file_record_to_document,
)
data = unwrapTransit(inp)
if isinstance(data, str):
one = _file_record_to_document(data)
return {"documents": [one], "count": 1} if one else None
if not isinstance(data, dict):
return None
d = dict(data)
_coerce_document_list_upload_fields(d)
# Per-iteration payload from executionEngine (flow.loop → downstream in loop body)
if "currentItem" in d:
ci = d.get("currentItem")
if ci is not None:
nested = _extract_wired_document_list(ci)
if nested:
return nested
docs = d.get("documents")
if isinstance(docs, list) and len(docs) > 0:
return {"documents": docs, "count": d.get("count", len(docs))}
raw_list = d.get("documentList")
if isinstance(raw_list, list) and len(raw_list) > 0 and isinstance(raw_list[0], dict):
return {"documents": raw_list, "count": len(raw_list)}
doc_id = d.get("documentId") or d.get("id")
if doc_id and str(doc_id).strip():
one: Dict[str, Any] = {"id": str(doc_id).strip()}
fn = d.get("fileName") or d.get("name")
if fn:
one["name"] = str(fn)
mt = d.get("mimeType")
if mt:
one["mimeType"] = str(mt)
return {"documents": [one], "count": 1}
files = d.get("files")
if isinstance(files, list) and files:
collected = []
for item in files:
conv = _file_record_to_document(item) if isinstance(item, dict) else None
if conv:
collected.append(conv)
if collected:
return {"documents": collected, "count": len(collected)}
return None
def _document_list_param_is_empty(val: Any) -> bool:
if val is None or val == "":
return True
if isinstance(val, list) and len(val) == 0:
return True
if isinstance(val, dict):
if val.get("documents") or val.get("references") or val.get("items"):
return False
if val.get("documentId") or val.get("id"):
return False
return True
return False
class ActionNodeExecutor: class ActionNodeExecutor:
"""Execute action nodes by mapping to method actions via ActionExecutor.""" """Execute action nodes by mapping to method actions via ActionExecutor."""
@ -260,6 +352,17 @@ class ActionNodeExecutor:
if pName and pName not in resolvedParams and "default" in pDef: if pName and pName not in resolvedParams and "default" in pDef:
resolvedParams[pName] = pDef["default"] resolvedParams[pName] = pDef["default"]
_param_names = {p.get("name") for p in nodeDef.get("parameters", []) if p.get("name")}
if "documentList" in _param_names and _document_list_param_is_empty(resolvedParams.get("documentList")):
_src_map = (context.get("inputSources") or {}).get(nodeId) or {}
_entry = _src_map.get(0)
if _entry:
_src_node_id, _ = _entry
_upstream = (context.get("nodeOutputs") or {}).get(_src_node_id)
_wired = _extract_wired_document_list(_upstream)
if _wired:
resolvedParams["documentList"] = _wired
# 3. Resolve connectionReference # 3. Resolve connectionReference
chatService = getattr(self.services, "chat", None) chatService = getattr(self.services, "chat", None)
_resolveConnectionParam(resolvedParams, chatService, self.services) _resolveConnectionParam(resolvedParams, chatService, self.services)
@ -323,7 +426,8 @@ class ActionNodeExecutor:
for d in (result.documents or []): for d in (result.documents or []):
dumped = d.model_dump() if hasattr(d, "model_dump") else dict(d) if isinstance(d, dict) else d dumped = d.model_dump() if hasattr(d, "model_dump") else dict(d) if isinstance(d, dict) else d
rawData = getattr(d, "documentData", None) if hasattr(d, "documentData") else (dumped.get("documentData") if isinstance(dumped, dict) else None) rawData = getattr(d, "documentData", None) if hasattr(d, "documentData") else (dumped.get("documentData") if isinstance(dumped, dict) else None)
if isinstance(dumped, dict) and isinstance(rawData, bytes) and len(rawData) > 0: rawBytes = _coerce_document_data_to_bytes(rawData)
if isinstance(dumped, dict) and rawBytes:
try: try:
from modules.interfaces.interfaceDbManagement import getInterface as _getMgmtInterface from modules.interfaces.interfaceDbManagement import getInterface as _getMgmtInterface
from modules.interfaces.interfaceDbApp import getInterface as _getAppInterface from modules.interfaces.interfaceDbApp import getInterface as _getAppInterface
@ -347,8 +451,8 @@ class ActionNodeExecutor:
_mgmt = _getMgmtInterface(_owner, mandateId=_mandateId, featureInstanceId=_instanceId) _mgmt = _getMgmtInterface(_owner, mandateId=_mandateId, featureInstanceId=_instanceId)
_docName = dumped.get("documentName") or f"workflow-result-{nodeId}.bin" _docName = dumped.get("documentName") or f"workflow-result-{nodeId}.bin"
_mimeType = dumped.get("mimeType") or "application/octet-stream" _mimeType = dumped.get("mimeType") or "application/octet-stream"
_fileItem = _mgmt.createFile(_docName, _mimeType, rawData) _fileItem = _mgmt.createFile(_docName, _mimeType, rawBytes)
_mgmt.createFileData(_fileItem.id, rawData) _mgmt.createFileData(_fileItem.id, rawBytes)
dumped["fileId"] = _fileItem.id dumped["fileId"] = _fileItem.id
dumped["id"] = _fileItem.id dumped["id"] = _fileItem.id
dumped["fileName"] = _fileItem.fileName dumped["fileName"] = _fileItem.fileName

View file

@ -45,10 +45,12 @@ class IOExecutor:
if 0 in inputSources: if 0 in inputSources:
srcId, _ = inputSources[0] srcId, _ = inputSources[0]
inp = nodeOutputs.get(srcId) inp = nodeOutputs.get(srcId)
from modules.workflows.automation2.executors.actionNodeExecutor import _getDocumentsFromUpstream from modules.workflows.automation2.executors.actionNodeExecutor import _extract_wired_document_list
docs = _getDocumentsFromUpstream(inp) if isinstance(inp, dict) else []
wired = _extract_wired_document_list(inp)
docs = (wired or {}).get("documents") if isinstance(wired, dict) else None
if docs: if docs:
resolvedParams.setdefault("documentList", docs) resolvedParams.setdefault("documentList", wired)
elif inp is not None: elif inp is not None:
resolvedParams.setdefault("input", inp) resolvedParams.setdefault("input", inp)

View file

@ -21,7 +21,6 @@ async def generateCode(self, parameters: Dict[str, Any]) -> ActionResult:
if not prompt.strip(): if not prompt.strip():
return ActionResult.isFailure(error="prompt is required") return ActionResult.isFailure(error="prompt is required")
documentList = parameters.get("documentList", [])
# Optional: if omitted, formats determined from prompt by AI # Optional: if omitted, formats determined from prompt by AI
resultType = parameters.get("resultType") resultType = parameters.get("resultType")
@ -34,19 +33,15 @@ async def generateCode(self, parameters: Dict[str, Any]) -> ActionResult:
parentOperationId = parameters.get('parentOperationId') parentOperationId = parameters.get('parentOperationId')
try: try:
# Convert documentList to DocumentReferenceList if needed from modules.datamodels.datamodelDocref import coerceDocumentReferenceList
docRefList = None
if documentList:
from modules.datamodels.datamodelDocref import DocumentReferenceList
if isinstance(documentList, DocumentReferenceList): raw_dl = parameters.get("documentList")
docRefList = documentList if raw_dl is None or raw_dl == "":
elif isinstance(documentList, str): docRefList = None
docRefList = DocumentReferenceList.from_string_list([documentList])
elif isinstance(documentList, list):
docRefList = DocumentReferenceList.from_string_list(documentList)
else: else:
docRefList = DocumentReferenceList(references=[]) docRefList = coerceDocumentReferenceList(raw_dl)
if not docRefList.references:
docRefList = None
# Prepare title # Prepare title
title = "Generated Code" title = "Generated Code"

View file

@ -21,7 +21,6 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
if not prompt.strip(): if not prompt.strip():
return ActionResult.isFailure(error="prompt is required") return ActionResult.isFailure(error="prompt is required")
documentList = parameters.get("documentList", [])
documentType = parameters.get("documentType") documentType = parameters.get("documentType")
# Prefer explicit outputFormat (flow UI); resultType remains for legacy / API callers. # Prefer explicit outputFormat (flow UI); resultType remains for legacy / API callers.
resultType = parameters.get("outputFormat") or parameters.get("resultType") resultType = parameters.get("outputFormat") or parameters.get("resultType")
@ -37,19 +36,16 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
parentOperationId = parameters.get('parentOperationId') parentOperationId = parameters.get('parentOperationId')
try: try:
# Convert documentList to DocumentReferenceList if needed # Convert documentList to DocumentReferenceList (handles dict {"documents": [...]}, list of ids, str, etc.)
docRefList = None from modules.datamodels.datamodelDocref import coerceDocumentReferenceList
if documentList:
from modules.datamodels.datamodelDocref import DocumentReferenceList
if isinstance(documentList, DocumentReferenceList): raw_dl = parameters.get("documentList")
docRefList = documentList if raw_dl is None or raw_dl == "":
elif isinstance(documentList, str): docRefList = None
docRefList = DocumentReferenceList.from_string_list([documentList])
elif isinstance(documentList, list):
docRefList = DocumentReferenceList.from_string_list(documentList)
else: else:
docRefList = DocumentReferenceList(references=[]) docRefList = coerceDocumentReferenceList(raw_dl)
if not docRefList.references:
docRefList = None
title_raw = parameters.get("title") title_raw = parameters.get("title")
title = (title_raw.strip() if isinstance(title_raw, str) else "") or None title = (title_raw.strip() if isinstance(title_raw, str) else "") or None