267 lines
11 KiB
Python
267 lines
11 KiB
Python
# Copyright (c) 2025 Patrick Motsch
|
|
# All rights reserved.
|
|
"""
|
|
Structure Generation Module
|
|
|
|
Handles document structure generation, including:
|
|
- Generating document structure with sections
|
|
- Building structure prompts
|
|
"""
|
|
import json
|
|
import logging
|
|
from typing import Dict, Any, List
|
|
|
|
from modules.datamodels.datamodelExtraction import ContentPart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StructureGenerator:
|
|
"""Handles document structure generation."""
|
|
|
|
def __init__(self, services, aiService):
|
|
"""Initialize StructureGenerator with service center and AI service access."""
|
|
self.services = services
|
|
self.aiService = aiService
|
|
|
|
async def generateStructure(
|
|
self,
|
|
userPrompt: str,
|
|
contentParts: List[ContentPart],
|
|
outputFormat: str,
|
|
parentOperationId: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Phase 5C: Generiert Chapter-Struktur (Table of Contents).
|
|
Definiert für jedes Chapter:
|
|
- Level, Title
|
|
- contentPartIds
|
|
- contentPartInstructions
|
|
- generationHint
|
|
|
|
Args:
|
|
userPrompt: User-Anfrage
|
|
contentParts: Alle vorbereiteten ContentParts mit Metadaten
|
|
outputFormat: Ziel-Format (html, docx, pdf, etc.)
|
|
parentOperationId: Parent Operation-ID für ChatLog-Hierarchie
|
|
|
|
Returns:
|
|
Struktur-Dict mit documents und chapters (nicht sections!)
|
|
"""
|
|
# Erstelle Operation-ID für Struktur-Generierung
|
|
structureOperationId = f"{parentOperationId}_structure_generation"
|
|
|
|
# Starte ChatLog mit Parent-Referenz
|
|
self.services.chat.progressLogStart(
|
|
structureOperationId,
|
|
"Chapter Structure Generation",
|
|
"Structure",
|
|
f"Generating chapter structure for {outputFormat}",
|
|
parentOperationId=parentOperationId
|
|
)
|
|
|
|
try:
|
|
# Baue Chapter-Struktur-Prompt mit Content-Index
|
|
structurePrompt = self._buildChapterStructurePrompt(
|
|
userPrompt=userPrompt,
|
|
contentParts=contentParts,
|
|
outputFormat=outputFormat
|
|
)
|
|
|
|
# AI-Call für Chapter-Struktur-Generierung
|
|
# Note: Debug logging is handled by callAiPlanning
|
|
aiResponse = await self.aiService.callAiPlanning(
|
|
prompt=structurePrompt,
|
|
debugType="chapter_structure_generation"
|
|
)
|
|
|
|
# Parse Struktur
|
|
# Use tryParseJson which handles malformed JSON and unterminated strings
|
|
extractedJson = self.services.utils.jsonExtractString(aiResponse)
|
|
parsedJson, parseError, cleanedJson = self.services.utils.jsonTryParse(extractedJson)
|
|
|
|
if parseError is not None:
|
|
# Try to repair broken JSON (handles unterminated strings, incomplete structures, etc.)
|
|
logger.warning(f"Initial JSON parsing failed: {str(parseError)}. Attempting repair...")
|
|
from modules.shared import jsonUtils
|
|
repairedJson = jsonUtils.repairBrokenJson(extractedJson)
|
|
if repairedJson:
|
|
# Try parsing repaired JSON
|
|
parsedJson, parseError, _ = self.services.utils.jsonTryParse(json.dumps(repairedJson))
|
|
if parseError is None:
|
|
logger.info("Successfully repaired and parsed JSON structure")
|
|
structure = parsedJson
|
|
else:
|
|
logger.error(f"Failed to parse repaired JSON: {str(parseError)}")
|
|
raise ValueError(f"Failed to parse JSON structure after repair: {str(parseError)}")
|
|
else:
|
|
logger.error(f"Failed to repair JSON. Parse error: {str(parseError)}")
|
|
logger.error(f"Cleaned JSON preview (first 500 chars): {cleanedJson[:500]}")
|
|
raise ValueError(f"Failed to parse JSON structure: {str(parseError)}")
|
|
else:
|
|
structure = parsedJson
|
|
|
|
# ChatLog abschließen
|
|
self.services.chat.progressLogFinish(structureOperationId, True)
|
|
|
|
return structure
|
|
|
|
except Exception as e:
|
|
self.services.chat.progressLogFinish(structureOperationId, False)
|
|
logger.error(f"Error in generateStructure: {str(e)}")
|
|
raise
|
|
|
|
def _buildChapterStructurePrompt(
|
|
self,
|
|
userPrompt: str,
|
|
contentParts: List[ContentPart],
|
|
outputFormat: str
|
|
) -> str:
|
|
"""Baue Prompt für Chapter-Struktur-Generierung."""
|
|
# Baue ContentParts-Index - filtere leere Parts heraus
|
|
contentPartsIndex = ""
|
|
validParts = []
|
|
filteredParts = []
|
|
|
|
for part in contentParts:
|
|
contentFormat = part.metadata.get("contentFormat", "unknown")
|
|
|
|
# WICHTIG: Reference Parts haben absichtlich leere Daten - immer einschließen
|
|
if contentFormat == "reference":
|
|
validParts.append(part)
|
|
logger.debug(f"Including reference ContentPart {part.id} (intentionally empty data)")
|
|
continue
|
|
|
|
# Überspringe leere Parts (keine Daten oder nur Container ohne Inhalt)
|
|
# ABER: Reference Parts wurden bereits oben behandelt
|
|
if not part.data or (isinstance(part.data, str) and len(part.data.strip()) == 0):
|
|
# Überspringe Container-Parts ohne Daten
|
|
if part.typeGroup == "container" and not part.data:
|
|
filteredParts.append((part.id, "container without data"))
|
|
continue
|
|
# Überspringe andere leere Parts (aber nicht Reference, die wurden bereits behandelt)
|
|
if not part.data:
|
|
filteredParts.append((part.id, f"no data (format: {contentFormat})"))
|
|
continue
|
|
|
|
validParts.append(part)
|
|
logger.debug(f"Including ContentPart {part.id}: format={contentFormat}, type={part.typeGroup}, dataLength={len(str(part.data)) if part.data else 0}")
|
|
|
|
if filteredParts:
|
|
logger.debug(f"Filtered out {len(filteredParts)} empty ContentParts: {filteredParts}")
|
|
|
|
logger.info(f"Building structure prompt with {len(validParts)} valid ContentParts (from {len(contentParts)} total)")
|
|
|
|
# Baue Index nur für gültige Parts
|
|
for i, part in enumerate(validParts, 1):
|
|
contentFormat = part.metadata.get("contentFormat", "unknown")
|
|
originalFileName = part.metadata.get('originalFileName', 'N/A')
|
|
|
|
contentPartsIndex += f"\n{i}. ContentPart ID: {part.id}\n"
|
|
contentPartsIndex += f" Format: {contentFormat}\n"
|
|
contentPartsIndex += f" Type: {part.typeGroup}\n"
|
|
contentPartsIndex += f" MIME Type: {part.mimeType or 'N/A'}\n"
|
|
contentPartsIndex += f" Source: {part.metadata.get('documentId', 'unknown')}\n"
|
|
contentPartsIndex += f" Original file name: {originalFileName}\n"
|
|
contentPartsIndex += f" Usage hint: {part.metadata.get('usageHint', 'N/A')}\n"
|
|
|
|
if not contentPartsIndex:
|
|
contentPartsIndex = "\n(No content parts available)"
|
|
|
|
# Extract language from user prompt or default to "de" (can be detected from userPrompt)
|
|
# For now, default to "de" - can be enhanced with language detection later
|
|
language = "en" # Default language
|
|
|
|
prompt = f"""USER REQUEST (for context):
|
|
```
|
|
{userPrompt}
|
|
```
|
|
|
|
LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}.
|
|
|
|
AVAILABLE CONTENT PARTS:
|
|
{contentPartsIndex}
|
|
|
|
TASK: Generate Chapter Structure for the documents to be generated.
|
|
|
|
IMPORTANT - CHAPTER INDEPENDENCE:
|
|
- Each chapter is independent and self-contained
|
|
- One chapter does NOT have information about another chapter
|
|
- Each chapter must provide its own context and be understandable alone
|
|
|
|
CRITICAL - CONTENT ASSIGNMENT TO CHAPTERS:
|
|
- You MUST assign available ContentParts to chapters using contentPartIds
|
|
- Based on the user request, determine which content should be used in which chapter
|
|
- If the user request mentions specific content, assign the corresponding ContentPart to the appropriate chapter
|
|
- Chapters WITHOUT contentPartIds can only generate generic content, NOT document-specific analysis
|
|
- To include document content analysis, chapters MUST have contentPartIds assigned
|
|
- Review the user request carefully to match ContentParts to chapters based on context and purpose
|
|
|
|
CRITICAL - CHAPTERS WITHOUT CONTENT PARTS:
|
|
- If contentPartIds is EMPTY, generationHint MUST be VERY DETAILED with all context needed to generate content from scratch
|
|
- Include: what to generate, what information to include, purpose, specific details
|
|
- Without content parts, AI relies ENTIRELY on generationHint and CANNOT analyze document content
|
|
|
|
IMPORTANT - FORMATTING:
|
|
- Formatting (fonts, colors, layouts, styles) is handled AUTOMATICALLY by the renderer
|
|
- Do NOT specify formatting details in generationHint unless it's content-specific (e.g., "pie chart with 3 segments")
|
|
- Focus on CONTENT and STRUCTURE, not visual formatting
|
|
- The renderer will apply appropriate styling based on the output format ({outputFormat})
|
|
|
|
For each chapter:
|
|
- chapter id
|
|
- level (1, 2, 3, etc.)
|
|
- title
|
|
- contentPartIds: [List of ContentPart IDs] - ASSIGN content based on user request and chapter purpose
|
|
- contentPartInstructions: {{
|
|
"partId": {{
|
|
"instruction": "How content should be structured"
|
|
}}
|
|
}}
|
|
- generationHint: Description of the content (must be self-contained with all necessary context)
|
|
* If contentPartIds is EMPTY, generationHint MUST be VERY DETAILED with all context needed to generate content from scratch
|
|
* Focus on content and structure, NOT formatting details
|
|
|
|
OUTPUT FORMAT: {outputFormat}
|
|
|
|
RETURN JSON:
|
|
{{
|
|
"metadata": {{
|
|
"title": "Document Title",
|
|
"language": "de"
|
|
}},
|
|
"documents": [{{
|
|
"id": "doc_1",
|
|
"title": "Document Title",
|
|
"filename": "document.{outputFormat}",
|
|
"chapters": [
|
|
{{
|
|
"id": "chapter_1",
|
|
"level": 1,
|
|
"title": "Introduction",
|
|
"contentPartIds": ["part_ext_1"],
|
|
"contentPartInstructions": {{
|
|
"part_ext_1": {{
|
|
"instruction": "Use full extracted text"
|
|
}}
|
|
}},
|
|
"generationHint": "Create introduction section",
|
|
"sections": []
|
|
}},
|
|
{{
|
|
"id": "chapter_2",
|
|
"level": 1,
|
|
"title": "Main Title",
|
|
"contentPartIds": [],
|
|
"contentPartInstructions": {{}},
|
|
"generationHint": "Create [specific content description] with [formatting details]. Include [required information]. Purpose: [explanation of what this chapter provides].",
|
|
"sections": []
|
|
}}
|
|
]
|
|
}}]
|
|
}}
|
|
|
|
Return ONLY valid JSON following the structure above.
|
|
"""
|
|
return prompt
|
|
|