258 lines
13 KiB
Python
258 lines
13 KiB
Python
# Copyright (c) 2025 Patrick Motsch
|
|
# All rights reserved.
|
|
"""
|
|
Document Generation Path
|
|
|
|
Handles document generation using existing chapter/section model.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
from typing import Dict, Any, List, Optional
|
|
from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData
|
|
from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent
|
|
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum
|
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class DocumentGenerationPath:
|
|
"""Document generation path (existing functionality, refactored)."""
|
|
|
|
def __init__(self, services):
|
|
self.services = services
|
|
|
|
async def generateDocument(
|
|
self,
|
|
userPrompt: str,
|
|
documentList: Optional[Any] = None, # DocumentReferenceList
|
|
documentIntents: Optional[List[DocumentIntent]] = None,
|
|
contentParts: Optional[List[ContentPart]] = None,
|
|
outputFormat: str = "txt",
|
|
title: Optional[str] = None,
|
|
parentOperationId: Optional[str] = None
|
|
) -> AiResponse:
|
|
"""
|
|
Generate document using existing chapter/section model.
|
|
|
|
Returns: AiResponse with documents list
|
|
"""
|
|
# Create operation ID
|
|
workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
|
|
docOperationId = f"doc_gen_{workflowId}_{int(time.time())}"
|
|
|
|
# Start progress tracking
|
|
self.services.chat.progressLogStart(
|
|
docOperationId,
|
|
"Document Generation",
|
|
"Document Generation",
|
|
f"Format: {outputFormat}",
|
|
parentOperationId=parentOperationId
|
|
)
|
|
|
|
try:
|
|
# Schritt 5A: Kläre Dokument-Intents
|
|
documents = []
|
|
if documentList:
|
|
documents = self.services.chat.getChatDocumentsFromDocumentList(documentList)
|
|
|
|
if not documentIntents and documents:
|
|
documentIntents = await self.services.ai.clarifyDocumentIntents(
|
|
documents,
|
|
userPrompt,
|
|
{"outputFormat": outputFormat},
|
|
docOperationId
|
|
)
|
|
|
|
# Schritt 5B: Extrahiere und bereite Content vor
|
|
if documents:
|
|
preparedContentParts = await self.services.ai.extractAndPrepareContent(
|
|
documents,
|
|
documentIntents or [],
|
|
docOperationId
|
|
)
|
|
|
|
# Merge mit bereitgestellten contentParts (falls vorhanden)
|
|
if contentParts:
|
|
# Prüfe auf pre-extracted Content
|
|
for part in contentParts:
|
|
if part.metadata.get("skipExtraction", False):
|
|
# Bereits extrahiert - verwende as-is, stelle sicher dass Metadaten vollständig
|
|
part.metadata.setdefault("contentFormat", "extracted")
|
|
part.metadata.setdefault("isPreExtracted", True)
|
|
preparedContentParts.extend(contentParts)
|
|
|
|
contentParts = preparedContentParts
|
|
|
|
# Schritt 5B.5: Process contentParts with AI extraction (if provided)
|
|
# This extracts text from images, processes content, and updates contentParts with extracted data
|
|
# This matches the original flow: extract content first (no AI), then process with AI
|
|
if contentParts:
|
|
# Filter out binary/other parts that shouldn't be processed
|
|
processableParts = []
|
|
skippedParts = []
|
|
for p in contentParts:
|
|
if p.typeGroup in ["image", "text", "table", "structure"] or (p.mimeType and (p.mimeType.startswith("image/") or p.mimeType.startswith("text/"))):
|
|
processableParts.append(p)
|
|
else:
|
|
skippedParts.append(p)
|
|
|
|
if skippedParts:
|
|
logger.debug(f"Skipping {len(skippedParts)} binary/other parts from document generation")
|
|
|
|
if processableParts:
|
|
# Count images for progress update
|
|
imageCount = len([p for p in processableParts if p.typeGroup == "image" or (p.mimeType and p.mimeType.startswith("image/"))])
|
|
if imageCount > 0:
|
|
self.services.chat.progressLogUpdate(docOperationId, 0.25, f"Extracting data from {imageCount} images using vision models")
|
|
|
|
# Build proper extraction prompt using buildExtractionPrompt
|
|
# This creates a focused extraction prompt, not the user's generation prompt
|
|
from modules.services.serviceExtraction.subPromptBuilderExtraction import buildExtractionPrompt
|
|
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum
|
|
|
|
# Determine renderer for format-specific guidelines
|
|
renderer = None
|
|
if outputFormat:
|
|
try:
|
|
from modules.services.serviceGeneration.mainServiceGeneration import GenerationService
|
|
generationService = GenerationService(self.services)
|
|
renderer = generationService.getRendererForFormat(outputFormat)
|
|
except Exception as e:
|
|
logger.debug(f"Could not get renderer for format {outputFormat}: {e}")
|
|
|
|
extractionPrompt = await buildExtractionPrompt(
|
|
outputFormat=outputFormat or "txt",
|
|
userPrompt=userPrompt, # User's prompt as context for what to extract
|
|
title=title or "Document",
|
|
aiService=self.services.ai if hasattr(self.services.ai, 'aiObjects') and self.services.ai.aiObjects else None,
|
|
services=self.services,
|
|
renderer=renderer
|
|
)
|
|
|
|
logger.info(f"Processing {len(processableParts)} content parts ({imageCount} images) with extraction prompt")
|
|
|
|
# Update progress - starting extraction
|
|
self.services.chat.progressLogUpdate(docOperationId, 0.26, f"Starting AI extraction from {len(processableParts)} content parts")
|
|
|
|
# Use DATA_EXTRACT operation type for extraction
|
|
extractionOptions = AiCallOptions(
|
|
operationType=OperationTypeEnum.DATA_EXTRACT, # Use DATA_EXTRACT for extraction
|
|
compressPrompt=False,
|
|
compressContext=False
|
|
)
|
|
|
|
# Create progress callback for per-part progress updates
|
|
def extractionProgressCallback(progress: float, message: str):
|
|
"""Progress callback for extraction - updates parent operation."""
|
|
# Map progress from 0.0-1.0 to 0.26-0.35 range (extraction phase)
|
|
mappedProgress = 0.26 + (progress * 0.09) # 0.26 to 0.35
|
|
self.services.chat.progressLogUpdate(docOperationId, mappedProgress, message)
|
|
|
|
extractionRequest = AiCallRequest(
|
|
prompt=extractionPrompt, # Use proper extraction prompt, not user's generation prompt
|
|
context="",
|
|
options=extractionOptions,
|
|
contentParts=processableParts
|
|
)
|
|
|
|
# Write debug file for extraction prompt (all parts)
|
|
self.services.utils.writeDebugFile(extractionPrompt, "content_extraction_prompt")
|
|
|
|
# Call AI to extract content from contentParts (with progress callback)
|
|
extractionResponse = await self.services.ai.callAi(extractionRequest, progressCallback=extractionProgressCallback)
|
|
|
|
# Update progress - extraction completed
|
|
self.services.chat.progressLogUpdate(docOperationId, 0.35, f"Completed AI extraction from {len(processableParts)} content parts")
|
|
|
|
# Write debug file for extraction response
|
|
if extractionResponse.content:
|
|
self.services.utils.writeDebugFile(extractionResponse.content, "content_extraction_response")
|
|
else:
|
|
self.services.utils.writeDebugFile(f"Error: No content returned (errorCount={extractionResponse.errorCount})", "content_extraction_response")
|
|
logger.warning(f"Content extraction returned no content (errorCount={extractionResponse.errorCount})")
|
|
|
|
# Update contentParts with extracted content (matching original flow)
|
|
if extractionResponse.errorCount == 0 and extractionResponse.content:
|
|
# The extracted content is already merged - update the first processable part with it
|
|
# This matches the original behavior where extracted text was used for generation
|
|
if processableParts:
|
|
# Store extracted content in metadata for use in structure generation
|
|
processableParts[0].metadata["extractedContent"] = extractionResponse.content
|
|
logger.info(f"Successfully extracted content from {len(processableParts)} parts ({len(extractionResponse.content)} chars)")
|
|
else:
|
|
# Extraction failed - log warning but continue
|
|
logger.warning(f"Content extraction failed, continuing with original contentParts")
|
|
|
|
# Schritt 5C: Generiere Struktur
|
|
structure = await self.services.ai.generateStructure(
|
|
userPrompt,
|
|
contentParts or [],
|
|
outputFormat,
|
|
docOperationId
|
|
)
|
|
|
|
# Schritt 5D: Fülle Struktur
|
|
# Language will be extracted from services (user intention analysis) in fillStructure
|
|
filledStructure = await self.services.ai.fillStructure(
|
|
structure,
|
|
contentParts or [],
|
|
userPrompt,
|
|
docOperationId
|
|
)
|
|
|
|
# Schritt 5E: Rendere Resultat
|
|
# Jedes Dokument wird einzeln gerendert, kann 1..n Dateien zurückgeben (z.B. HTML + Bilder)
|
|
renderedDocuments = await self.services.ai.renderResult(
|
|
filledStructure,
|
|
outputFormat,
|
|
title or "Generated Document",
|
|
userPrompt,
|
|
docOperationId
|
|
)
|
|
|
|
# Baue Response: Konvertiere alle gerenderten Dokumente zu DocumentData
|
|
documentDataList = []
|
|
for renderedDoc in renderedDocuments:
|
|
try:
|
|
# Erstelle DocumentData für jedes gerenderte Dokument
|
|
docDataObj = DocumentData(
|
|
documentName=renderedDoc.filename,
|
|
documentData=renderedDoc.documentData,
|
|
mimeType=renderedDoc.mimeType,
|
|
sourceJson=filledStructure if len(documentDataList) == 0 else None # Nur für erstes Dokument
|
|
)
|
|
documentDataList.append(docDataObj)
|
|
logger.debug(f"Added rendered document: {renderedDoc.filename} ({len(renderedDoc.documentData)} bytes, {renderedDoc.mimeType})")
|
|
except Exception as e:
|
|
logger.warning(f"Error creating document {renderedDoc.filename}: {str(e)}")
|
|
|
|
if not documentDataList:
|
|
raise ValueError("No documents were rendered")
|
|
|
|
metadata = AiResponseMetadata(
|
|
title=title or filledStructure.get("metadata", {}).get("title", "Generated Document"),
|
|
operationType=OperationTypeEnum.DATA_GENERATE.value
|
|
)
|
|
|
|
# Debug-Log (harmonisiert)
|
|
self.services.utils.writeDebugFile(
|
|
json.dumps(filledStructure, indent=2, ensure_ascii=False, default=str),
|
|
"document_generation_response"
|
|
)
|
|
|
|
self.services.chat.progressLogFinish(docOperationId, True)
|
|
|
|
return AiResponse(
|
|
content=json.dumps(filledStructure),
|
|
metadata=metadata,
|
|
documents=documentDataList
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in document generation: {str(e)}")
|
|
self.services.chat.progressLogFinish(docOperationId, False)
|
|
raise
|
|
|