fixed iteration ai calls for documents sections
This commit is contained in:
parent
9f433743a6
commit
e4b633cac6
7 changed files with 699 additions and 43 deletions
|
|
@ -155,14 +155,17 @@ class AiCallLooper:
|
||||||
)
|
)
|
||||||
|
|
||||||
# Write the ACTUAL prompt sent to AI
|
# Write the ACTUAL prompt sent to AI
|
||||||
# For section content generation: only write one prompt file (first iteration)
|
# For section content generation: write prompt for first iteration and continuation iterations
|
||||||
# For document generation: write prompt for each iteration
|
# For document generation: write prompt for each iteration
|
||||||
isSectionContent = "_section_" in debugPrefix
|
isSectionContent = "_section_" in debugPrefix
|
||||||
if iteration == 1 or not isSectionContent:
|
if iteration == 1:
|
||||||
if iteration == 1:
|
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt")
|
||||||
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt")
|
elif isSectionContent:
|
||||||
elif not isSectionContent:
|
# Save continuation prompts for section_content debugging
|
||||||
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}")
|
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}")
|
||||||
|
else:
|
||||||
|
# Document generation - save all iteration prompts
|
||||||
|
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}")
|
||||||
|
|
||||||
response = await self.aiService.callAi(request)
|
response = await self.aiService.callAi(request)
|
||||||
result = response.content
|
result = response.content
|
||||||
|
|
@ -183,13 +186,16 @@ class AiCallLooper:
|
||||||
self.services.chat.progressLogUpdate(iterationOperationId, 0.6, f"AI response received ({bytesDisplay})")
|
self.services.chat.progressLogUpdate(iterationOperationId, 0.6, f"AI response received ({bytesDisplay})")
|
||||||
|
|
||||||
# Write raw AI response to debug file
|
# Write raw AI response to debug file
|
||||||
# For section content generation: only write one response file (first iteration)
|
# For section content generation: write response for first iteration and continuation iterations
|
||||||
# For document generation: write response for each iteration
|
# For document generation: write response for each iteration
|
||||||
if iteration == 1 or not isSectionContent:
|
if iteration == 1:
|
||||||
if iteration == 1:
|
self.services.utils.writeDebugFile(result, f"{debugPrefix}_response")
|
||||||
self.services.utils.writeDebugFile(result, f"{debugPrefix}_response")
|
elif isSectionContent:
|
||||||
elif not isSectionContent:
|
# Save continuation responses for section_content debugging
|
||||||
self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}")
|
self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}")
|
||||||
|
else:
|
||||||
|
# Document generation - save all iteration responses
|
||||||
|
self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}")
|
||||||
|
|
||||||
# Emit stats for this iteration (only if workflow exists and has id)
|
# Emit stats for this iteration (only if workflow exists and has id)
|
||||||
if self.services.workflow and hasattr(self.services.workflow, 'id') and self.services.workflow.id:
|
if self.services.workflow and hasattr(self.services.workflow, 'id') and self.services.workflow.id:
|
||||||
|
|
@ -279,23 +285,57 @@ class AiCallLooper:
|
||||||
# JSON is complete - merge accumulated responses if any
|
# JSON is complete - merge accumulated responses if any
|
||||||
if accumulatedDirectJson:
|
if accumulatedDirectJson:
|
||||||
logger.info(f"Iteration {iteration}: Merging {len(accumulatedDirectJson) + 1} accumulated responses")
|
logger.info(f"Iteration {iteration}: Merging {len(accumulatedDirectJson) + 1} accumulated responses")
|
||||||
# Merge accumulated JSON strings with current response
|
|
||||||
mergedJsonString = accumulatedDirectJson[0] if accumulatedDirectJson else result
|
|
||||||
for prevJson in accumulatedDirectJson[1:]:
|
|
||||||
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, prevJson)
|
|
||||||
# Finally merge with current response
|
|
||||||
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, result)
|
|
||||||
|
|
||||||
# Re-parse merged JSON
|
# Use generic data-based merging for all use cases
|
||||||
try:
|
try:
|
||||||
extractedMerged = extractJsonString(mergedJsonString)
|
# Parse all accumulated JSON strings and current response
|
||||||
parsedMerged, parseError, _ = tryParseJson(extractedMerged)
|
allParsed = []
|
||||||
if parseError is None and parsedMerged:
|
for jsonStr in accumulatedDirectJson + [result]:
|
||||||
parsedJsonForUseCase = parsedMerged
|
extracted = extractJsonString(jsonStr)
|
||||||
|
parsed, parseErr, _ = tryParseJson(extracted)
|
||||||
|
if parseErr is None and parsed:
|
||||||
|
# Normalize structure: ensure consistent format
|
||||||
|
normalized = self._normalizeJsonStructure(parsed, useCaseId)
|
||||||
|
allParsed.append(normalized)
|
||||||
|
|
||||||
|
if allParsed and len(allParsed) > 1:
|
||||||
|
# Generic recursive merge of parsed JSON objects
|
||||||
|
mergedJsonObj = self._mergeJsonObjectsRecursively(allParsed)
|
||||||
|
|
||||||
|
# Reconstruct merged JSON string
|
||||||
|
mergedJsonString = json.dumps(mergedJsonObj, indent=2, ensure_ascii=False)
|
||||||
|
parsedJsonForUseCase = mergedJsonObj
|
||||||
|
result = mergedJsonString
|
||||||
|
logger.info(f"Successfully merged {len(accumulatedDirectJson) + 1} JSON fragments using generic recursive merge")
|
||||||
|
elif allParsed:
|
||||||
|
# Only one parsed JSON, use it directly
|
||||||
|
parsedJsonForUseCase = allParsed[0]
|
||||||
|
result = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False)
|
||||||
|
else:
|
||||||
|
# Fallback to string merging if parsing fails
|
||||||
|
logger.warning("Failed to parse all JSON fragments for data-based merge, falling back to string merging")
|
||||||
|
mergedJsonString = accumulatedDirectJson[0] if accumulatedDirectJson else result
|
||||||
|
for prevJson in accumulatedDirectJson[1:]:
|
||||||
|
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, prevJson)
|
||||||
|
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, result)
|
||||||
result = mergedJsonString
|
result = mergedJsonString
|
||||||
logger.info(f"Successfully merged and parsed {len(accumulatedDirectJson) + 1} JSON fragments")
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to parse merged JSON, using last response: {e}")
|
logger.warning(f"Failed data-based merge, falling back to string merging: {e}")
|
||||||
|
# Fallback to string merging
|
||||||
|
mergedJsonString = accumulatedDirectJson[0] if accumulatedDirectJson else result
|
||||||
|
for prevJson in accumulatedDirectJson[1:]:
|
||||||
|
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, prevJson)
|
||||||
|
mergedJsonString = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, result)
|
||||||
|
result = mergedJsonString
|
||||||
|
|
||||||
|
# Try to parse the string-merged result
|
||||||
|
try:
|
||||||
|
extractedMerged = extractJsonString(result)
|
||||||
|
parsedMerged, parseError, _ = tryParseJson(extractedMerged)
|
||||||
|
if parseError is None and parsedMerged:
|
||||||
|
parsedJsonForUseCase = parsedMerged
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
logger.info(f"Iteration {iteration}: Use case '{useCaseId}' - JSON is complete")
|
logger.info(f"Iteration {iteration}: Use case '{useCaseId}' - JSON is complete")
|
||||||
|
|
||||||
|
|
@ -307,10 +347,12 @@ class AiCallLooper:
|
||||||
# The merging logic in subStructureFilling.py will handle extraction and merging
|
# The merging logic in subStructureFilling.py will handle extraction and merging
|
||||||
if useCaseId == "section_content":
|
if useCaseId == "section_content":
|
||||||
final_json = result # Return raw response to preserve all JSON blocks
|
final_json = result # Return raw response to preserve all JSON blocks
|
||||||
|
# Write final merged result for section_content (overwrites iteration 1 response with complete merged result)
|
||||||
|
self.services.utils.writeDebugFile(final_json, f"{debugPrefix}_response")
|
||||||
else:
|
else:
|
||||||
final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result)
|
final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result)
|
||||||
|
|
||||||
# Write final result for chapter structure and code structure (section_content skips it)
|
# Write final result for chapter structure and code structure
|
||||||
if useCaseId in ["chapter_structure", "code_structure"]:
|
if useCaseId in ["chapter_structure", "code_structure"]:
|
||||||
self.services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result")
|
self.services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result")
|
||||||
|
|
||||||
|
|
@ -567,6 +609,166 @@ class AiCallLooper:
|
||||||
|
|
||||||
return final_result
|
return final_result
|
||||||
|
|
||||||
|
def _normalizeJsonStructure(self, parsed: Any, useCaseId: str) -> Any:
|
||||||
|
"""
|
||||||
|
Normalize JSON structure to ensure consistent format before merging.
|
||||||
|
Handles different response formats and converts them to expected structure.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
parsed: Parsed JSON object (can be dict, list, or primitive)
|
||||||
|
useCaseId: Use case ID to determine expected structure
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Normalized JSON structure
|
||||||
|
"""
|
||||||
|
# For section_content, expect {"elements": [...]} structure
|
||||||
|
if useCaseId == "section_content":
|
||||||
|
if isinstance(parsed, list):
|
||||||
|
# Check if list contains strings (invalid format) or element objects
|
||||||
|
if parsed and isinstance(parsed[0], str):
|
||||||
|
# Invalid format - list of strings instead of elements
|
||||||
|
# This shouldn't happen, but we'll log a warning and return empty structure
|
||||||
|
logger.warning(f"Invalid response format: received list of strings instead of elements array. Expected {{'elements': [...]}} structure.")
|
||||||
|
return {"elements": []}
|
||||||
|
else:
|
||||||
|
# Convert plain list of elements to elements structure
|
||||||
|
return {"elements": parsed}
|
||||||
|
elif isinstance(parsed, dict):
|
||||||
|
# If it already has "elements", return as-is
|
||||||
|
if "elements" in parsed:
|
||||||
|
return parsed
|
||||||
|
# If it has "type" and looks like an element, wrap in elements array
|
||||||
|
elif parsed.get("type"):
|
||||||
|
return {"elements": [parsed]}
|
||||||
|
# Otherwise, assume it's already in correct format
|
||||||
|
else:
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
# For other use cases, return as-is (they have their own structures)
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
def _mergeJsonObjectsRecursively(self, jsonObjects: List[Any]) -> Any:
|
||||||
|
"""
|
||||||
|
GENERIC recursive merge function for JSON objects.
|
||||||
|
Works for ANY JSON structure - handles lists, dicts, and primitives intelligently.
|
||||||
|
|
||||||
|
Merge strategy:
|
||||||
|
- Lists/Arrays: Merge by removing duplicates based on content (works for rows, items, elements, etc.)
|
||||||
|
- Dicts/Objects: Merge properties, recursively merging nested structures
|
||||||
|
- Primitives: Use the latest value
|
||||||
|
|
||||||
|
Args:
|
||||||
|
jsonObjects: List of parsed JSON objects to merge
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Merged JSON object
|
||||||
|
"""
|
||||||
|
if not jsonObjects:
|
||||||
|
return None
|
||||||
|
if len(jsonObjects) == 1:
|
||||||
|
return jsonObjects[0]
|
||||||
|
|
||||||
|
# Start with first object and merge others into it
|
||||||
|
merged = jsonObjects[0]
|
||||||
|
|
||||||
|
for obj in jsonObjects[1:]:
|
||||||
|
merged = self._mergeTwoObjects(merged, obj)
|
||||||
|
|
||||||
|
return merged
|
||||||
|
|
||||||
|
def _mergeTwoObjects(self, obj1: Any, obj2: Any) -> Any:
|
||||||
|
"""
|
||||||
|
Merge two JSON objects recursively.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
obj1: First object (base)
|
||||||
|
obj2: Second object (to merge into obj1)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Merged object
|
||||||
|
"""
|
||||||
|
# Handle None values
|
||||||
|
if obj1 is None:
|
||||||
|
return obj2
|
||||||
|
if obj2 is None:
|
||||||
|
return obj1
|
||||||
|
|
||||||
|
# Handle different types
|
||||||
|
if isinstance(obj1, dict) and isinstance(obj2, dict):
|
||||||
|
# Merge dictionaries
|
||||||
|
merged = dict(obj1) # Start with copy of obj1
|
||||||
|
|
||||||
|
for key, value2 in obj2.items():
|
||||||
|
if key in merged:
|
||||||
|
# Key exists in both - recursively merge
|
||||||
|
merged[key] = self._mergeTwoObjects(merged[key], value2)
|
||||||
|
else:
|
||||||
|
# New key - add it
|
||||||
|
merged[key] = value2
|
||||||
|
|
||||||
|
return merged
|
||||||
|
|
||||||
|
elif isinstance(obj1, list) and isinstance(obj2, list):
|
||||||
|
# Merge lists by removing duplicates based on content
|
||||||
|
merged = list(obj1) # Start with copy of obj1
|
||||||
|
seenItems = set() # Track seen items to avoid duplicates
|
||||||
|
|
||||||
|
# Add all items from obj1 with their keys
|
||||||
|
for item in merged:
|
||||||
|
itemKey = self._createItemKey(item)
|
||||||
|
seenItems.add(itemKey)
|
||||||
|
|
||||||
|
# Add items from obj2 that aren't duplicates
|
||||||
|
for item in obj2:
|
||||||
|
itemKey = self._createItemKey(item)
|
||||||
|
if itemKey not in seenItems:
|
||||||
|
seenItems.add(itemKey)
|
||||||
|
merged.append(item)
|
||||||
|
|
||||||
|
return merged
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Different types or primitives - use obj2 (latest value)
|
||||||
|
return obj2
|
||||||
|
|
||||||
|
def _createItemKey(self, item: Any) -> Any:
|
||||||
|
"""
|
||||||
|
Create a key for an item to detect duplicates.
|
||||||
|
Works generically for any JSON structure.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
item: Item to create key for
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Key that can be used for duplicate detection
|
||||||
|
"""
|
||||||
|
if isinstance(item, dict):
|
||||||
|
# For dicts, create key from all values (or specific identifying fields)
|
||||||
|
# Try to find common identifying fields first
|
||||||
|
if "id" in item:
|
||||||
|
return ("id", item["id"])
|
||||||
|
elif "type" in item and "content" in item:
|
||||||
|
# For elements with type and content, use type + content hash
|
||||||
|
content = item.get("content", {})
|
||||||
|
if isinstance(content, dict):
|
||||||
|
# For tables/lists, use type + first few rows/items for key
|
||||||
|
if "rows" in content:
|
||||||
|
rows = content.get("rows", [])
|
||||||
|
return ("type", item["type"], "rows", tuple(rows[:3]) if rows else ())
|
||||||
|
elif "items" in content:
|
||||||
|
items = content.get("items", [])
|
||||||
|
return ("type", item["type"], "items", tuple(items[:3]) if items else ())
|
||||||
|
return ("type", item["type"], tuple(sorted(item.items())))
|
||||||
|
else:
|
||||||
|
# Generic: use sorted items tuple
|
||||||
|
return tuple(sorted(item.items()))
|
||||||
|
elif isinstance(item, (list, tuple)):
|
||||||
|
# For lists/tuples, use the tuple itself as key
|
||||||
|
return tuple(item) if isinstance(item, list) else item
|
||||||
|
else:
|
||||||
|
# For primitives, use the value itself
|
||||||
|
return item
|
||||||
|
|
||||||
async def _defineKpisFromPrompt(
|
async def _defineKpisFromPrompt(
|
||||||
self,
|
self,
|
||||||
userPrompt: str,
|
userPrompt: str,
|
||||||
|
|
|
||||||
|
|
@ -69,24 +69,59 @@ class CodeGenerationPath:
|
||||||
codeFiles = await self._generateCodeContent(codeStructure, codeOperationId)
|
codeFiles = await self._generateCodeContent(codeStructure, codeOperationId)
|
||||||
|
|
||||||
# Phase 3: Code formatting & validation
|
# Phase 3: Code formatting & validation
|
||||||
self.services.chat.progressLogUpdate(codeOperationId, 0.9, "Formatting code files")
|
self.services.chat.progressLogUpdate(codeOperationId, 0.8, "Formatting code files")
|
||||||
formattedFiles = await self._formatAndValidateCode(codeFiles)
|
formattedFiles = await self._formatAndValidateCode(codeFiles)
|
||||||
|
|
||||||
# Convert to unified document format
|
# Phase 4: Code Rendering (Renderer-Based)
|
||||||
documents = []
|
self.services.chat.progressLogUpdate(codeOperationId, 0.9, "Rendering code files")
|
||||||
|
|
||||||
|
# Group files by format
|
||||||
|
filesByFormat = {}
|
||||||
for file in formattedFiles:
|
for file in formattedFiles:
|
||||||
mimeType = self._getMimeType(file.get("fileType", outputFormat or "txt"))
|
fileType = file.get("fileType", outputFormat or "txt")
|
||||||
content = file.get("content", "")
|
if fileType not in filesByFormat:
|
||||||
if isinstance(content, str):
|
filesByFormat[fileType] = []
|
||||||
contentBytes = content.encode('utf-8')
|
filesByFormat[fileType].append(file)
|
||||||
else:
|
|
||||||
contentBytes = content
|
# Render each format group using appropriate renderer
|
||||||
|
allRenderedDocuments = []
|
||||||
|
for fileType, files in filesByFormat.items():
|
||||||
|
# Get renderer for this format
|
||||||
|
renderer = self._getCodeRenderer(fileType)
|
||||||
|
|
||||||
|
if renderer:
|
||||||
|
# Use code renderer
|
||||||
|
renderedDocs = await renderer.renderCodeFiles(
|
||||||
|
codeFiles=files,
|
||||||
|
metadata=codeStructure.get("metadata", {}),
|
||||||
|
userPrompt=userPrompt
|
||||||
|
)
|
||||||
|
allRenderedDocuments.extend(renderedDocs)
|
||||||
|
else:
|
||||||
|
# Fallback: output directly (for formats without renderers)
|
||||||
|
for file in files:
|
||||||
|
mimeType = self._getMimeType(file.get("fileType", "txt"))
|
||||||
|
content = file.get("content", "")
|
||||||
|
contentBytes = content.encode('utf-8') if isinstance(content, str) else content
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
||||||
|
allRenderedDocuments.append(
|
||||||
|
RenderedDocument(
|
||||||
|
documentData=contentBytes,
|
||||||
|
mimeType=mimeType,
|
||||||
|
filename=file.get("filename", "generated.txt"),
|
||||||
|
metadata=codeStructure.get("metadata", {})
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Convert RenderedDocument to DocumentData
|
||||||
|
documents = []
|
||||||
|
for renderedDoc in allRenderedDocuments:
|
||||||
documents.append(DocumentData(
|
documents.append(DocumentData(
|
||||||
documentName=file.get("filename", "generated.txt"),
|
documentName=renderedDoc.filename,
|
||||||
documentData=contentBytes,
|
documentData=renderedDoc.documentData,
|
||||||
mimeType=mimeType,
|
mimeType=renderedDoc.mimeType,
|
||||||
sourceJson=file
|
sourceJson=renderedDoc.metadata if hasattr(renderedDoc, 'metadata') else None
|
||||||
))
|
))
|
||||||
|
|
||||||
metadata = AiResponseMetadata(
|
metadata = AiResponseMetadata(
|
||||||
|
|
@ -579,6 +614,28 @@ Return ONLY valid JSON in this format:
|
||||||
"md": "text/markdown",
|
"md": "text/markdown",
|
||||||
"java": "text/x-java-source",
|
"java": "text/x-java-source",
|
||||||
"cpp": "text/x-c++src",
|
"cpp": "text/x-c++src",
|
||||||
"c": "text/x-csrc"
|
"c": "text/x-csrc",
|
||||||
|
"csv": "text/csv",
|
||||||
|
"xml": "application/xml"
|
||||||
}
|
}
|
||||||
return mimeTypes.get(fileType.lower(), "text/plain")
|
return mimeTypes.get(fileType.lower(), "text/plain")
|
||||||
|
|
||||||
|
def _getCodeRenderer(self, fileType: str):
|
||||||
|
"""Get code renderer for file type."""
|
||||||
|
from modules.services.serviceGeneration.renderers.registry import getRenderer
|
||||||
|
|
||||||
|
# Map file types to renderer formats
|
||||||
|
formatMap = {
|
||||||
|
'json': 'json',
|
||||||
|
'csv': 'csv',
|
||||||
|
'xml': 'xml'
|
||||||
|
}
|
||||||
|
|
||||||
|
rendererFormat = formatMap.get(fileType.lower())
|
||||||
|
if rendererFormat:
|
||||||
|
renderer = getRenderer(rendererFormat, self.services)
|
||||||
|
# Check if renderer supports code rendering
|
||||||
|
if renderer and hasattr(renderer, 'renderCodeFiles'):
|
||||||
|
return renderer
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""
|
||||||
|
Base renderer class for code format renderers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from .rendererBaseTemplate import BaseRenderer
|
||||||
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class BaseCodeRenderer(BaseRenderer):
|
||||||
|
"""Base class for code format renderers."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def renderCodeFiles(
|
||||||
|
self,
|
||||||
|
codeFiles: List[Dict[str, Any]],
|
||||||
|
metadata: Dict[str, Any],
|
||||||
|
userPrompt: str = None
|
||||||
|
) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render code files to format-specific output.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
codeFiles: List of file dictionaries with:
|
||||||
|
- filename: str
|
||||||
|
- fileType: str (json, csv, xml, etc.)
|
||||||
|
- content: str (generated code)
|
||||||
|
- id: str (optional)
|
||||||
|
metadata: Project metadata (language, projectType, etc.)
|
||||||
|
userPrompt: Original user prompt
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of RenderedDocument objects (can be 1..n files)
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _validateCodeFile(self, codeFile: Dict[str, Any]) -> bool:
|
||||||
|
"""Validate code file structure."""
|
||||||
|
required = ['filename', 'fileType', 'content']
|
||||||
|
return all(key in codeFile for key in required)
|
||||||
|
|
@ -76,9 +76,26 @@ class RendererRegistry:
|
||||||
# Get supported formats from the renderer class
|
# Get supported formats from the renderer class
|
||||||
supportedFormats = rendererClass.getSupportedFormats()
|
supportedFormats = rendererClass.getSupportedFormats()
|
||||||
|
|
||||||
|
# Get priority (default to 0 if not specified)
|
||||||
|
priority = rendererClass.getPriority() if hasattr(rendererClass, 'getPriority') else 0
|
||||||
|
|
||||||
for formatName in supportedFormats:
|
for formatName in supportedFormats:
|
||||||
# Register primary format
|
formatKey = formatName.lower()
|
||||||
self._renderers[formatName.lower()] = rendererClass
|
|
||||||
|
# Check if format already registered - use priority to decide
|
||||||
|
if formatKey in self._renderers:
|
||||||
|
existingRenderer = self._renderers[formatKey]
|
||||||
|
existingPriority = existingRenderer.getPriority() if hasattr(existingRenderer, 'getPriority') else 0
|
||||||
|
|
||||||
|
# Only replace if new renderer has higher priority
|
||||||
|
if priority > existingPriority:
|
||||||
|
logger.debug(f"Replacing {existingRenderer.__name__} with {rendererClass.__name__} for format '{formatName}' (priority {priority} > {existingPriority})")
|
||||||
|
self._renderers[formatKey] = rendererClass
|
||||||
|
else:
|
||||||
|
logger.debug(f"Keeping {existingRenderer.__name__} for format '{formatName}' (priority {existingPriority} >= {priority})")
|
||||||
|
else:
|
||||||
|
# Register primary format
|
||||||
|
self._renderers[formatKey] = rendererClass
|
||||||
|
|
||||||
# Register aliases if any
|
# Register aliases if any
|
||||||
if hasattr(rendererClass, 'getFormatAliases'):
|
if hasattr(rendererClass, 'getFormatAliases'):
|
||||||
|
|
@ -86,7 +103,7 @@ class RendererRegistry:
|
||||||
for alias in aliases:
|
for alias in aliases:
|
||||||
self._format_mappings[alias.lower()] = formatName.lower()
|
self._format_mappings[alias.lower()] = formatName.lower()
|
||||||
|
|
||||||
logger.debug(f"Registered {rendererClass.__name__} for formats: {supportedFormats}")
|
logger.debug(f"Registered {rendererClass.__name__} for formats: {supportedFormats} (priority: {priority})")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error registering renderer {rendererClass.__name__}: {str(e)}")
|
logger.error(f"Error registering renderer {rendererClass.__name__}: {str(e)}")
|
||||||
|
|
|
||||||
125
modules/services/serviceGeneration/renderers/rendererCodeCsv.py
Normal file
125
modules/services/serviceGeneration/renderers/rendererCodeCsv.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""
|
||||||
|
CSV code renderer for code generation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .codeRendererBaseTemplate import BaseCodeRenderer
|
||||||
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
|
||||||
|
class RendererCodeCsv(BaseCodeRenderer):
|
||||||
|
"""Renders CSV code files."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getSupportedFormats(cls) -> List[str]:
|
||||||
|
"""Return supported CSV formats."""
|
||||||
|
return ['csv']
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getFormatAliases(cls) -> List[str]:
|
||||||
|
"""Return format aliases."""
|
||||||
|
return []
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getPriority(cls) -> int:
|
||||||
|
"""Return priority for CSV code renderer."""
|
||||||
|
return 75 # Higher than document renderer (70) for code generation
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getOutputStyle(cls, formatName: Optional[str] = None) -> str:
|
||||||
|
"""Return output style classification: CSV requires specific structure."""
|
||||||
|
return 'code'
|
||||||
|
|
||||||
|
async def renderCodeFiles(
|
||||||
|
self,
|
||||||
|
codeFiles: List[Dict[str, Any]],
|
||||||
|
metadata: Dict[str, Any],
|
||||||
|
userPrompt: str = None
|
||||||
|
) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render CSV code files.
|
||||||
|
For single file: output as-is (validate structure)
|
||||||
|
For multiple files: output separately (each is independent CSV)
|
||||||
|
"""
|
||||||
|
renderedDocs = []
|
||||||
|
|
||||||
|
for codeFile in codeFiles:
|
||||||
|
if not self._validateCodeFile(codeFile):
|
||||||
|
self.logger.warning(f"Invalid code file: {codeFile.get('filename', 'unknown')}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
filename = codeFile['filename']
|
||||||
|
content = codeFile['content']
|
||||||
|
|
||||||
|
# Validate CSV structure (header row, consistent columns)
|
||||||
|
validatedContent = self._validateAndFixCsv(content)
|
||||||
|
|
||||||
|
renderedDocs.append(
|
||||||
|
RenderedDocument(
|
||||||
|
documentData=validatedContent.encode('utf-8'),
|
||||||
|
mimeType="text/csv",
|
||||||
|
filename=filename,
|
||||||
|
metadata=metadata
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return renderedDocs
|
||||||
|
|
||||||
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render method for document generation compatibility.
|
||||||
|
Delegates to document renderer if needed, or handles code files directly.
|
||||||
|
"""
|
||||||
|
# Check if this is code generation (has files array) or document generation (has documents array)
|
||||||
|
if "files" in extractedContent:
|
||||||
|
# Code generation path - use renderCodeFiles
|
||||||
|
files = extractedContent.get("files", [])
|
||||||
|
metadata = extractedContent.get("metadata", {})
|
||||||
|
return await self.renderCodeFiles(files, metadata, userPrompt)
|
||||||
|
else:
|
||||||
|
# Document generation path - delegate to document renderer
|
||||||
|
from .rendererCsv import RendererCsv
|
||||||
|
documentRenderer = RendererCsv(self.services)
|
||||||
|
return await documentRenderer.render(extractedContent, title, userPrompt, aiService)
|
||||||
|
|
||||||
|
def _validateAndFixCsv(self, content: str) -> str:
|
||||||
|
"""Validate CSV structure and fix common issues."""
|
||||||
|
try:
|
||||||
|
# Parse CSV to validate structure
|
||||||
|
reader = csv.reader(io.StringIO(content))
|
||||||
|
rows = list(reader)
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return content # Empty CSV
|
||||||
|
|
||||||
|
# Check header row exists
|
||||||
|
headerRow = rows[0]
|
||||||
|
headerCount = len(headerRow)
|
||||||
|
|
||||||
|
# Validate all rows have same column count
|
||||||
|
fixedRows = [headerRow] # Start with header
|
||||||
|
|
||||||
|
for i, row in enumerate(rows[1:], 1):
|
||||||
|
if len(row) != headerCount:
|
||||||
|
self.logger.warning(f"Row {i} has {len(row)} columns, expected {headerCount}. Fixing...")
|
||||||
|
# Pad or truncate to match header
|
||||||
|
if len(row) < headerCount:
|
||||||
|
row.extend([''] * (headerCount - len(row)))
|
||||||
|
else:
|
||||||
|
row = row[:headerCount]
|
||||||
|
fixedRows.append(row)
|
||||||
|
|
||||||
|
# Convert back to CSV string
|
||||||
|
output = io.StringIO()
|
||||||
|
writer = csv.writer(output)
|
||||||
|
for row in fixedRows:
|
||||||
|
writer.writerow(row)
|
||||||
|
|
||||||
|
return output.getvalue()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.warning(f"CSV validation failed: {e}, returning original content")
|
||||||
|
return content
|
||||||
|
|
@ -0,0 +1,97 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""
|
||||||
|
JSON code renderer for code generation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .codeRendererBaseTemplate import BaseCodeRenderer
|
||||||
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
import json
|
||||||
|
|
||||||
|
class RendererCodeJson(BaseCodeRenderer):
|
||||||
|
"""Renders JSON code files."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getSupportedFormats(cls) -> List[str]:
|
||||||
|
"""Return supported JSON formats."""
|
||||||
|
return ['json']
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getFormatAliases(cls) -> List[str]:
|
||||||
|
"""Return format aliases."""
|
||||||
|
return []
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getPriority(cls) -> int:
|
||||||
|
"""Return priority for JSON code renderer."""
|
||||||
|
return 85 # Higher than document renderer (80) for code generation
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getOutputStyle(cls, formatName: Optional[str] = None) -> str:
|
||||||
|
"""Return output style classification: JSON is structured data format."""
|
||||||
|
return 'code'
|
||||||
|
|
||||||
|
async def renderCodeFiles(
|
||||||
|
self,
|
||||||
|
codeFiles: List[Dict[str, Any]],
|
||||||
|
metadata: Dict[str, Any],
|
||||||
|
userPrompt: str = None
|
||||||
|
) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render JSON code files.
|
||||||
|
For single file: output as-is
|
||||||
|
For multiple files: output separately (each file is independent JSON)
|
||||||
|
"""
|
||||||
|
renderedDocs = []
|
||||||
|
|
||||||
|
for codeFile in codeFiles:
|
||||||
|
if not self._validateCodeFile(codeFile):
|
||||||
|
self.logger.warning(f"Invalid code file: {codeFile.get('filename', 'unknown')}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
filename = codeFile['filename']
|
||||||
|
content = codeFile['content']
|
||||||
|
|
||||||
|
# Validate JSON syntax
|
||||||
|
try:
|
||||||
|
json.loads(content) # Validate JSON
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
self.logger.warning(f"Invalid JSON in {filename}: {e}")
|
||||||
|
# Could fix/format JSON here if needed
|
||||||
|
|
||||||
|
# Format JSON (pretty print)
|
||||||
|
try:
|
||||||
|
parsed = json.loads(content)
|
||||||
|
formattedContent = json.dumps(parsed, indent=2, ensure_ascii=False)
|
||||||
|
except Exception:
|
||||||
|
formattedContent = content # Use original if formatting fails
|
||||||
|
|
||||||
|
renderedDocs.append(
|
||||||
|
RenderedDocument(
|
||||||
|
documentData=formattedContent.encode('utf-8'),
|
||||||
|
mimeType="application/json",
|
||||||
|
filename=filename,
|
||||||
|
metadata=metadata
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return renderedDocs
|
||||||
|
|
||||||
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render method for document generation compatibility.
|
||||||
|
Delegates to document renderer if needed, or handles code files directly.
|
||||||
|
"""
|
||||||
|
# Check if this is code generation (has files array) or document generation (has documents array)
|
||||||
|
if "files" in extractedContent:
|
||||||
|
# Code generation path - use renderCodeFiles
|
||||||
|
files = extractedContent.get("files", [])
|
||||||
|
metadata = extractedContent.get("metadata", {})
|
||||||
|
return await self.renderCodeFiles(files, metadata, userPrompt)
|
||||||
|
else:
|
||||||
|
# Document generation path - delegate to document renderer
|
||||||
|
# Import here to avoid circular dependency
|
||||||
|
from .rendererJson import RendererJson
|
||||||
|
documentRenderer = RendererJson(self.services)
|
||||||
|
return await documentRenderer.render(extractedContent, title, userPrompt, aiService)
|
||||||
113
modules/services/serviceGeneration/renderers/rendererCodeXml.py
Normal file
113
modules/services/serviceGeneration/renderers/rendererCodeXml.py
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""
|
||||||
|
XML code renderer for code generation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .codeRendererBaseTemplate import BaseCodeRenderer
|
||||||
|
from modules.datamodels.datamodelDocument import RenderedDocument
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
from xml.dom import minidom
|
||||||
|
|
||||||
|
class RendererCodeXml(BaseCodeRenderer):
|
||||||
|
"""Renders XML code files."""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getSupportedFormats(cls) -> List[str]:
|
||||||
|
"""Return supported XML formats."""
|
||||||
|
return ['xml']
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getFormatAliases(cls) -> List[str]:
|
||||||
|
"""Return format aliases."""
|
||||||
|
return []
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getPriority(cls) -> int:
|
||||||
|
"""Return priority for XML code renderer."""
|
||||||
|
return 80
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def getOutputStyle(cls, formatName: Optional[str] = None) -> str:
|
||||||
|
"""Return output style classification: XML is structured data format."""
|
||||||
|
return 'code'
|
||||||
|
|
||||||
|
async def renderCodeFiles(
|
||||||
|
self,
|
||||||
|
codeFiles: List[Dict[str, Any]],
|
||||||
|
metadata: Dict[str, Any],
|
||||||
|
userPrompt: str = None
|
||||||
|
) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render XML code files.
|
||||||
|
Validates XML syntax and formats (pretty print).
|
||||||
|
"""
|
||||||
|
renderedDocs = []
|
||||||
|
|
||||||
|
for codeFile in codeFiles:
|
||||||
|
if not self._validateCodeFile(codeFile):
|
||||||
|
self.logger.warning(f"Invalid code file: {codeFile.get('filename', 'unknown')}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
filename = codeFile['filename']
|
||||||
|
content = codeFile['content']
|
||||||
|
|
||||||
|
# Validate and format XML
|
||||||
|
formattedContent = self._validateAndFormatXml(content)
|
||||||
|
|
||||||
|
renderedDocs.append(
|
||||||
|
RenderedDocument(
|
||||||
|
documentData=formattedContent.encode('utf-8'),
|
||||||
|
mimeType="application/xml",
|
||||||
|
filename=filename,
|
||||||
|
metadata=metadata
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return renderedDocs
|
||||||
|
|
||||||
|
async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]:
|
||||||
|
"""
|
||||||
|
Render method for document generation compatibility.
|
||||||
|
For XML, we only support code generation (no document renderer exists yet).
|
||||||
|
"""
|
||||||
|
# Check if this is code generation (has files array)
|
||||||
|
if "files" in extractedContent:
|
||||||
|
# Code generation path - use renderCodeFiles
|
||||||
|
files = extractedContent.get("files", [])
|
||||||
|
metadata = extractedContent.get("metadata", {})
|
||||||
|
return await self.renderCodeFiles(files, metadata, userPrompt)
|
||||||
|
else:
|
||||||
|
# Document generation path - not supported yet, return error
|
||||||
|
self.logger.warning("XML document generation not supported, only code generation")
|
||||||
|
return [
|
||||||
|
RenderedDocument(
|
||||||
|
documentData=f"XML document generation not yet supported".encode('utf-8'),
|
||||||
|
mimeType="text/plain",
|
||||||
|
filename="error.txt",
|
||||||
|
metadata={}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
def _validateAndFormatXml(self, content: str) -> str:
|
||||||
|
"""Validate XML syntax and format (pretty print)."""
|
||||||
|
try:
|
||||||
|
# Parse XML to validate
|
||||||
|
root = ET.fromstring(content)
|
||||||
|
|
||||||
|
# Format XML (pretty print)
|
||||||
|
rough_string = ET.tostring(root, encoding='unicode')
|
||||||
|
reparsed = minidom.parseString(rough_string)
|
||||||
|
formatted = reparsed.toprettyxml(indent=" ")
|
||||||
|
|
||||||
|
# Remove extra blank lines
|
||||||
|
lines = [line for line in formatted.split('\n') if line.strip()]
|
||||||
|
return '\n'.join(lines)
|
||||||
|
|
||||||
|
except ET.ParseError as e:
|
||||||
|
self.logger.warning(f"Invalid XML: {e}, returning original content")
|
||||||
|
return content
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.warning(f"XML formatting failed: {e}, returning original content")
|
||||||
|
return content
|
||||||
Loading…
Reference in a new issue