From 1c0604106b38b3c7828f6647fe8d38d8a1cf84dd Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 4 Jan 2026 21:38:26 +0100 Subject: [PATCH] implemented enhanced standardized ai looping system with usecases --- modules/datamodels/datamodelAi.py | 52 ++- .../services/serviceAi/subAiCallLooping.py | 131 +++--- .../services/serviceAi/subLoopingUseCases.py | 95 ++++ .../services/serviceAi/subStructureFilling.py | 417 +++++------------- .../serviceAi/subStructureGeneration.py | 143 ++++-- .../serviceGeneration/paths/codePath.py | 291 ++++++------ modules/shared/jsonUtils.py | 65 ++- 7 files changed, 610 insertions(+), 584 deletions(-) diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py index 9e680164..b4ce76b7 100644 --- a/modules/datamodels/datamodelAi.py +++ b/modules/datamodels/datamodelAi.py @@ -6,8 +6,6 @@ from enum import Enum # Import ContentPart for runtime use (needed for Pydantic model rebuilding) from modules.datamodels.datamodelExtraction import ContentPart -# Import JSON utilities for safe conversion -from modules.shared.jsonUtils import extractJsonString, tryParseJson, repairBrokenJson # Operation Types class OperationTypeEnum(str, Enum): @@ -258,3 +256,53 @@ class JsonAccumulationState(BaseModel): description="KPI definitions with current values: [{id, description, jsonPath, targetValue, currentValue}, ...]" ) + +class ContinuationContext(BaseModel): + """Pydantic model for continuation context information.""" + section_count: int + delivered_summary: str + cut_off_element: Optional[str] = None + element_before_cutoff: Optional[str] = None + template_structure: Optional[str] = None + last_complete_part: Optional[str] = None + incomplete_part: Optional[str] = None + structure_context: Optional[str] = None + last_raw_json: Optional[str] = None + + +class SectionPromptArgs(BaseModel): + """Type-safe arguments for section content prompt builder.""" + section: Dict[str, Any] + contentParts: List[ContentPart] + userPrompt: str + generationHint: str + allSections: List[Dict[str, Any]] + sectionIndex: int + isAggregation: bool + language: str + + +class ChapterStructurePromptArgs(BaseModel): + """Type-safe arguments for chapter structure prompt builder.""" + userPrompt: str + contentParts: List[ContentPart] = Field(default_factory=list) + outputFormat: str + + +class CodeContentPromptArgs(BaseModel): + """Type-safe arguments for code content prompt builder.""" + filename: str + fileType: str + functions: List[Dict] = Field(default_factory=list) + classes: List[Dict] = Field(default_factory=list) + dependencies: List[str] = Field(default_factory=list) + metadata: Dict[str, Any] = Field(default_factory=dict) + userPrompt: str + contentParts: List[ContentPart] = Field(default_factory=list) + contextInfo: str = "" + + +class CodeStructurePromptArgs(BaseModel): + """Type-safe arguments for code structure prompt builder.""" + userPrompt: str + contentParts: List[ContentPart] = Field(default_factory=list) \ No newline at end of file diff --git a/modules/services/serviceAi/subAiCallLooping.py b/modules/services/serviceAi/subAiCallLooping.py index 2bb2afd8..2af600e5 100644 --- a/modules/services/serviceAi/subAiCallLooping.py +++ b/modules/services/serviceAi/subAiCallLooping.py @@ -12,7 +12,9 @@ import json import logging from typing import Dict, Any, List, Optional, Callable -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum, JsonAccumulationState +from modules.datamodels.datamodelAi import ( + AiCallRequest, AiCallOptions +) from modules.datamodels.datamodelExtraction import ContentPart from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler @@ -110,18 +112,38 @@ class AiCallLooper: # CRITICAL: Build continuation prompt if we have sections OR if we have a previous response (even if broken) # This ensures continuation prompts are built even when JSON is so broken that no sections can be extracted if (len(allSections) > 0 or lastRawResponse) and promptBuilder and promptArgs: + # Extract templateStructure and basePrompt from promptArgs (REQUIRED) + templateStructure = promptArgs.get("templateStructure") + if not templateStructure: + raise ValueError( + f"templateStructure is REQUIRED in promptArgs for use case '{useCaseId}'. " + "Prompt creation functions must return (prompt, templateStructure) tuple." + ) + + basePrompt = promptArgs.get("basePrompt") + if not basePrompt: + # Fallback: use prompt parameter (should be the same) + basePrompt = prompt + logger.warning( + f"basePrompt not found in promptArgs for use case '{useCaseId}', " + "using prompt parameter instead. This may indicate a bug." + ) + # This is a continuation - build continuation context with raw JSON and rebuild prompt - continuationContext = buildContinuationContext(allSections, lastRawResponse, useCaseId) + continuationContext = buildContinuationContext( + allSections, lastRawResponse, useCaseId, templateStructure + ) if not lastRawResponse: logger.warning(f"Iteration {iteration}: No previous response available for continuation!") - # Unified prompt builder call: All prompt builders accept continuationContext and **kwargs - # Each builder extracts only the parameters it needs from kwargs - # This ensures consistent architecture across all use cases - if not promptArgs.get('services') and hasattr(self, 'services'): - promptArgs['services'] = self.services - - iterationPrompt = await promptBuilder(continuationContext=continuationContext, **promptArgs) + # Unified prompt builder call: Continuation builders only need continuationContext, templateStructure, and basePrompt + # All initial context (section, userPrompt, etc.) is already in basePrompt, so promptArgs is not needed + # Extract templateStructure and basePrompt from promptArgs (they're explicit parameters) + iterationPrompt = await promptBuilder( + continuationContext=continuationContext, + templateStructure=templateStructure, + basePrompt=basePrompt + ) else: # First iteration - use original prompt iterationPrompt = prompt @@ -238,11 +260,10 @@ class AiCallLooper: pass # Handle use cases that return JSON directly (no section extraction needed) - directReturnUseCases = ["section_content", "chapter_structure", "code_structure", "code_content"] - if useCaseId in directReturnUseCases: - # For chapter_structure, code_structure, section_content, and code_content, check completeness and support looping - loopingUseCases = ["chapter_structure", "code_structure", "section_content", "code_content"] - if useCaseId in loopingUseCases: + # Check if use case supports direct return (all registered use cases do) + if useCase and not useCase.requiresExtraction: + # For all direct return use cases, check completeness and support looping + if True: # All registered use cases support looping # CRITICAL: Check if JSON string is incomplete BEFORE parsing # If JSON is truncated, it will be closed for parsing, making it appear complete # So we need to check the original string, not the parsed JSON @@ -310,7 +331,8 @@ class AiCallLooper: extracted = extractJsonString(mergedJsonString) parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None and parsed: - normalized = self._normalizeJsonStructure(parsed, useCaseId) + # Use callback to normalize JSON structure + normalized = self._normalizeJsonStructure(parsed, useCase) parsedJsonForUseCase = normalized result = json.dumps(normalized, indent=2, ensure_ascii=False) except Exception: @@ -322,8 +344,8 @@ class AiCallLooper: parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None and parsed: - # Parsing succeeded - normalize and use - normalized = self._normalizeJsonStructure(parsed, useCaseId) + # Parsing succeeded - normalize and use (via callback) + normalized = self._normalizeJsonStructure(parsed, useCase) parsedJsonForUseCase = normalized result = json.dumps(normalized, indent=2, ensure_ascii=False) else: @@ -334,7 +356,8 @@ class AiCallLooper: extracted = extractJsonString(jsonStr) parsed, parseErr, _ = tryParseJson(extracted) if parseErr is None and parsed: - normalized = self._normalizeJsonStructure(parsed, useCaseId) + # Use callback to normalize JSON structure + normalized = self._normalizeJsonStructure(parsed, useCase) allParsed.append(normalized) if allParsed: @@ -399,18 +422,16 @@ class AiCallLooper: if iterationOperationId: self.services.chat.progressLogFinish(iterationOperationId, True) - # For section_content, return raw result to allow merging of multiple JSON blocks - # The merging logic in subStructureFilling.py will handle extraction and merging - if useCaseId == "section_content": - final_json = result # Return raw response to preserve all JSON blocks - # Write final merged result for section_content (overwrites iteration 1 response with complete merged result) - self.services.utils.writeDebugFile(final_json, f"{debugPrefix}_response") - else: - final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) - - # Write final result for chapter structure and code structure - if useCaseId in ["chapter_structure", "code_structure"]: - self.services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") + # Use callback to handle final result formatting and debug file writing (REQUIRED - no fallback) + if not useCase.finalResultHandler: + raise ValueError( + f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback. " + "All use cases must provide a finalResultHandler function." + ) + final_json = useCase.finalResultHandler( + result, parsedJsonForUseCase, extractedJsonForUseCase, + debugPrefix, self.services + ) return final_json @@ -423,8 +444,8 @@ class AiCallLooper: if iteration >= maxIterations: logger.warning(f"AI call stopped after maximum iterations ({maxIterations})") - # This code path is never reached because all use cases are in directReturnUseCases - # and return early at line 417. This code would only execute for use cases that + # This code path should never be reached because all registered use cases + # return early when JSON is complete. This would only execute for use cases that # require section extraction, but no such use cases are currently registered. logger.error(f"Unexpected code path: reached end of loop without return for use case '{useCaseId}'") return result if result else "" @@ -539,51 +560,23 @@ class AiCallLooper: # Doesn't parse even after closing - might be malformed, but assume incomplete to be safe return True - def _normalizeJsonStructure(self, parsed: Any, useCaseId: str) -> Any: + def _normalizeJsonStructure(self, parsed: Any, useCase) -> Any: """ Normalize JSON structure to ensure consistent format before merging. Handles different response formats and converts them to expected structure. Args: parsed: Parsed JSON object (can be dict, list, or primitive) - useCaseId: Use case ID to determine expected structure + useCase: LoopingUseCase instance with jsonNormalizer callback Returns: Normalized JSON structure """ - # For section_content, expect {"elements": [...]} structure - if useCaseId == "section_content": - if isinstance(parsed, list): - # Check if list contains strings (invalid format) or element objects - if parsed and isinstance(parsed[0], str): - # Invalid format - list of strings instead of elements - # Try to convert strings to paragraph elements as fallback - # This can happen if AI returns raw text instead of structured JSON - logger.debug(f"Received list of strings instead of elements array, converting to paragraph elements") - elements = [] - for text in parsed: - if isinstance(text, str) and text.strip(): - elements.append({ - "type": "paragraph", - "content": { - "text": text.strip() - } - }) - return {"elements": elements} if elements else {"elements": []} - else: - # Convert plain list of elements to elements structure - return {"elements": parsed} - elif isinstance(parsed, dict): - # If it already has "elements", return as-is - if "elements" in parsed: - return parsed - # If it has "type" and looks like an element, wrap in elements array - elif parsed.get("type"): - return {"elements": [parsed]} - # Otherwise, assume it's already in correct format - else: - return parsed - - # For other use cases, return as-is (they have their own structures) - return parsed + # Use callback to normalize JSON structure (REQUIRED - no fallback) + if not useCase or not useCase.jsonNormalizer: + raise ValueError( + f"Use case '{useCase.useCaseId if useCase else 'unknown'}' is missing required 'jsonNormalizer' callback. " + "All use cases must provide a jsonNormalizer function." + ) + return useCase.jsonNormalizer(parsed, useCase.useCaseId) diff --git a/modules/services/serviceAi/subLoopingUseCases.py b/modules/services/serviceAi/subLoopingUseCases.py index dcf3e31e..a2828108 100644 --- a/modules/services/serviceAi/subLoopingUseCases.py +++ b/modules/services/serviceAi/subLoopingUseCases.py @@ -12,6 +12,89 @@ from typing import Dict, Any, List, Optional, Callable logger = logging.getLogger(__name__) +# Callback functions for use-case-specific logic + +def _handleSectionContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for section_content: return raw result to preserve all JSON blocks.""" + final_json = result # Return raw response to preserve all JSON blocks + # Write final merged result for section_content (overwrites iteration 1 response with complete merged result) + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_response") + return final_json + + +def _handleChapterStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for chapter_structure: format JSON and write debug file.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + # Write final result for chapter structure + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") + return final_json + + +def _handleCodeStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for code_structure: format JSON and write debug file.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + # Write final result for code structure + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") + return final_json + + +def _handleCodeContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for code_content: format JSON.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + return final_json + + +def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any: + """Normalize JSON structure for section_content use case.""" + # For section_content, expect {"elements": [...]} structure + if isinstance(parsed, list): + # Check if list contains strings (invalid format) or element objects + if parsed and isinstance(parsed[0], str): + # Invalid format - list of strings instead of elements + # Try to convert strings to paragraph elements as fallback + logger.debug(f"Received list of strings instead of elements array, converting to paragraph elements") + elements = [] + for text in parsed: + if isinstance(text, str) and text.strip(): + elements.append({ + "type": "paragraph", + "content": { + "text": text.strip() + } + }) + return {"elements": elements} if elements else {"elements": []} + else: + # Convert plain list of elements to elements structure + return {"elements": parsed} + elif isinstance(parsed, dict): + # If it already has "elements", return as-is + if "elements" in parsed: + return parsed + # If it has "type" and looks like an element, wrap in elements array + elif parsed.get("type"): + return {"elements": [parsed]} + # Otherwise, assume it's already in correct format + else: + return parsed + + # For other use cases, return as-is (they have their own structures) + return parsed + + +def _normalizeDefaultJson(parsed: Any, useCaseId: str) -> Any: + """Default normalizer: return as-is.""" + return parsed + @dataclass class LoopingUseCase: @@ -39,6 +122,10 @@ class LoopingUseCase: # Result Building resultBuilder: Optional[Callable] = None # Build final result from accumulated data + # Use-case-specific handlers (callbacks to avoid if/elif chains in generic code) + finalResultHandler: Optional[Callable] = None # Handle final result formatting and debug file writing + jsonNormalizer: Optional[Callable] = None # Normalize JSON structure for this use case + # Metadata supportsAccumulation: bool = True # Whether this use case supports accumulation requiresExtraction: bool = False # Whether this requires extraction (like sections) @@ -124,6 +211,8 @@ class LoopingUseCaseRegistry: merger=None, continuationContextBuilder=None, # Will use default continuation context resultBuilder=None, # Return JSON directly + finalResultHandler=_handleSectionContentFinalResult, + jsonNormalizer=_normalizeSectionContentJson, supportsAccumulation=False, requiresExtraction=False )) @@ -141,6 +230,8 @@ class LoopingUseCaseRegistry: merger=None, continuationContextBuilder=None, resultBuilder=None, # Return JSON directly + finalResultHandler=_handleChapterStructureFinalResult, + jsonNormalizer=_normalizeDefaultJson, supportsAccumulation=False, requiresExtraction=False )) @@ -174,6 +265,8 @@ class LoopingUseCaseRegistry: merger=None, continuationContextBuilder=None, resultBuilder=None, + finalResultHandler=_handleCodeStructureFinalResult, + jsonNormalizer=_normalizeDefaultJson, supportsAccumulation=False, requiresExtraction=False )) @@ -190,6 +283,8 @@ class LoopingUseCaseRegistry: merger=None, # Will use default merger continuationContextBuilder=None, resultBuilder=None, # Will use default result builder + finalResultHandler=_handleCodeContentFinalResult, + jsonNormalizer=_normalizeDefaultJson, supportsAccumulation=True, requiresExtraction=False )) diff --git a/modules/services/serviceAi/subStructureFilling.py b/modules/services/serviceAi/subStructureFilling.py index 81e82dcc..e3767305 100644 --- a/modules/services/serviceAi/subStructureFilling.py +++ b/modules/services/serviceAi/subStructureFilling.py @@ -753,7 +753,7 @@ class StructureFiller: if processedExtractedParts: logger.debug(f"Section {sectionId}: Aggregating {len(processedExtractedParts)} extracted parts with AI") isAggregation = True - generationPrompt = self._buildSectionGenerationPrompt( + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( section=section, contentParts=processedExtractedParts, userPrompt=userPrompt, @@ -811,106 +811,8 @@ class StructureFiller: f"{chapterId}_section_{sectionId}_response" ) else: - async def buildSectionPromptWithContinuation( - continuationContext: Dict[str, Any], - **kwargs - ) -> str: - """Build section prompt with continuation context. Extracts section-specific parameters from kwargs.""" - # Extract parameters from kwargs (for section_content use case) - section = kwargs.get("section") - contentParts = kwargs.get("contentParts", []) - userPrompt = kwargs.get("userPrompt", "") - generationHint = kwargs.get("generationHint", "") - allSections = kwargs.get("allSections", []) - sectionIndex = kwargs.get("sectionIndex", 0) - isAggregation = kwargs.get("isAggregation", False) - basePrompt = self._buildSectionGenerationPrompt( - section=section, - contentParts=contentParts, - userPrompt=userPrompt, - generationHint=generationHint, - allSections=allSections, - sectionIndex=sectionIndex, - isAggregation=isAggregation, - language=language - ) - - # Extract JSON structure context for continuation - incompletePart = continuationContext.get("incomplete_part", "") - lastRawJson = continuationContext.get("last_raw_json", "") - - # Build overlap context: extract last ~100 characters from the response for overlap - overlapContext = "" - if lastRawJson: - # Get last 100 characters for overlap - overlapContext = lastRawJson[-100:].strip() - - # Build unified context showing structure hierarchy with cut point - # This combines structure template, last complete part, and incomplete part in one view - unifiedContext = "" - if lastRawJson: - # Find break position in raw JSON - if incompletePart: - breakPos = lastRawJson.find(incompletePart) - if breakPos == -1: - # Try to find where JSON ends - breakPos = len(lastRawJson.rstrip()) - else: - # No incomplete part found - assume end of JSON - breakPos = len(lastRawJson.rstrip()) - - # Build intelligent context showing hierarchy - from modules.shared.jsonUtils import _buildIncompleteContext - unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) - elif incompletePart: - # Fallback: use incomplete part directly - unifiedContext = incompletePart - else: - unifiedContext = "Unable to extract context - response was completely broken" - - # Use the SAME template structure as in initial prompt - # Get contentType and contentStructureExample exactly like in _buildSectionGenerationPrompt - contentType = section.get("content_type", "paragraph") - contentStructureExample = self._getContentStructureExample(contentType) - - # Build the exact same JSON structure template as in initial prompt - structureTemplate = f"""JSON Structure Template: -{{ - "elements": [ - {{ - "type": "{contentType}", - "content": {contentStructureExample} - }} - ] -}} - -""" - - continuationPrompt = f"""{basePrompt} - ---- CONTINUATION REQUEST --- -The previous JSON response was incomplete. Continue from where it stopped. - -{structureTemplate}Context showing structure hierarchy with cut point: -{unifiedContext} - -Overlap Requirement: -To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. - -Last ~100 characters from previous response (repeat these at the start): -{overlapContext if overlapContext else "No overlap context available"} - -TASK: -1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) -2. Complete the incomplete element shown in the context above (marked with CUT POINT) -3. Continue generating the remaining content following the JSON structure template above -4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects - -CRITICAL: -- Your response must be valid JSON matching the structure template above -- Start with overlap (~100 chars) then continue seamlessly -- Complete the incomplete element and continue with remaining elements""" - return continuationPrompt + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation options = AiCallOptions( operationType=operationType, @@ -932,7 +834,8 @@ CRITICAL: "allSections": all_sections_list, "sectionIndex": sectionIndex, "isAggregation": isAggregation, - "services": self.services + "templateStructure": templateStructure, + "basePrompt": generationPrompt }, operationId=sectionOperationId, userPrompt=userPrompt, @@ -1038,7 +941,7 @@ CRITICAL: if len(contentPartIds) == 0 and useAiCall and generationHint: # Generate content from scratch using only generationHint logger.debug(f"Processing section {sectionId}: No content parts, generating from generationHint only") - generationPrompt = self._buildSectionGenerationPrompt( + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( section=section, contentParts=[], userPrompt=userPrompt, @@ -1097,106 +1000,8 @@ CRITICAL: else: isAggregation = False - async def buildSectionPromptWithContinuation( - continuationContext: Dict[str, Any], - **kwargs - ) -> str: - """Build section prompt with continuation context. Extracts section-specific parameters from kwargs.""" - # Extract parameters from kwargs (for section_content use case) - section = kwargs.get("section") - contentParts = kwargs.get("contentParts", []) - userPrompt = kwargs.get("userPrompt", "") - generationHint = kwargs.get("generationHint", "") - allSections = kwargs.get("allSections", []) - sectionIndex = kwargs.get("sectionIndex", 0) - isAggregation = kwargs.get("isAggregation", False) - basePrompt = self._buildSectionGenerationPrompt( - section=section, - contentParts=contentParts, - userPrompt=userPrompt, - generationHint=generationHint, - allSections=allSections, - sectionIndex=sectionIndex, - isAggregation=isAggregation, - language=language - ) - - # Extract JSON structure context for continuation - incompletePart = continuationContext.get("incomplete_part", "") - lastRawJson = continuationContext.get("last_raw_json", "") - - # Build overlap context: extract last ~100 characters from the response for overlap - overlapContext = "" - if lastRawJson: - # Get last 100 characters for overlap - overlapContext = lastRawJson[-100:].strip() - - # Build unified context showing structure hierarchy with cut point - # This combines structure template, last complete part, and incomplete part in one view - unifiedContext = "" - if lastRawJson: - # Find break position in raw JSON - if incompletePart: - breakPos = lastRawJson.find(incompletePart) - if breakPos == -1: - # Try to find where JSON ends - breakPos = len(lastRawJson.rstrip()) - else: - # No incomplete part found - assume end of JSON - breakPos = len(lastRawJson.rstrip()) - - # Build intelligent context showing hierarchy - from modules.shared.jsonUtils import _buildIncompleteContext - unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) - elif incompletePart: - # Fallback: use incomplete part directly - unifiedContext = incompletePart - else: - unifiedContext = "Unable to extract context - response was completely broken" - - # Use the SAME template structure as in initial prompt - # Get contentType and contentStructureExample exactly like in _buildSectionGenerationPrompt - contentType = section.get("content_type", "paragraph") - contentStructureExample = self._getContentStructureExample(contentType) - - # Build the exact same JSON structure template as in initial prompt - structureTemplate = f"""JSON Structure Template: -{{ - "elements": [ - {{ - "type": "{contentType}", - "content": {contentStructureExample} - }} - ] -}} - -""" - - continuationPrompt = f"""{basePrompt} - ---- CONTINUATION REQUEST --- -The previous JSON response was incomplete. Continue from where it stopped. - -{structureTemplate}Context showing structure hierarchy with cut point: -{unifiedContext} - -Overlap Requirement: -To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. - -Last ~100 characters from previous response (repeat these at the start): -{overlapContext if overlapContext else "No overlap context available"} - -TASK: -1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) -2. Complete the incomplete element shown in the context above (marked with CUT POINT) -3. Continue generating the remaining content following the JSON structure template above -4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects - -CRITICAL: -- Your response must be valid JSON matching the structure template above -- Start with overlap (~100 chars) then continue seamlessly -- Complete the incomplete element and continue with remaining elements""" - return continuationPrompt + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation options = AiCallOptions( operationType=operationType, @@ -1208,7 +1013,7 @@ CRITICAL: prompt=generationPrompt, options=options, debugPrefix=f"{chapterId}_section_{sectionId}", - promptBuilder=buildSectionPromptWithContinuation, + promptBuilder=self.buildSectionPromptWithContinuation, promptArgs={ "section": section, "contentParts": [], @@ -1217,7 +1022,9 @@ CRITICAL: "allSections": all_sections_list, "sectionIndex": sectionIndex, "isAggregation": isAggregation, - "services": self.services + "templateStructure": templateStructure, + "basePrompt": generationPrompt, + "language": language }, operationId=sectionOperationId, userPrompt=userPrompt, @@ -1399,7 +1206,7 @@ CRITICAL: if useAiCall and generationHint: # AI-Call mit einzelnen ContentPart (now may be text part after Vision extraction) logger.debug(f"Processing section {sectionId}: Single extracted part with AI call") - generationPrompt = self._buildSectionGenerationPrompt( + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( section=section, contentParts=[part], userPrompt=userPrompt, @@ -1458,109 +1265,8 @@ CRITICAL: else: isAggregation = False - async def buildSectionPromptWithContinuation( - continuationContext: Dict[str, Any], - **kwargs - ) -> str: - """Build section prompt with continuation context. Extracts section-specific parameters from kwargs.""" - # Extract parameters from kwargs (for section_content use case) - section = kwargs.get("section") - contentParts = kwargs.get("contentParts", []) - userPrompt = kwargs.get("userPrompt", "") - generationHint = kwargs.get("generationHint", "") - allSections = kwargs.get("allSections", []) - sectionIndex = kwargs.get("sectionIndex", 0) - isAggregation = kwargs.get("isAggregation", False) - services = kwargs.get("services") - basePrompt = self._buildSectionGenerationPrompt( - section=section, - contentParts=contentParts, - userPrompt=userPrompt, - generationHint=generationHint, - allSections=allSections, - sectionIndex=sectionIndex, - isAggregation=isAggregation, - language=language - ) - - # Extract JSON structure context for continuation - templateStructure = continuationContext.get("template_structure", "") - lastCompletePart = continuationContext.get("last_complete_part", "") - incompletePart = continuationContext.get("incomplete_part", "") - structureContext = continuationContext.get("structure_context", "") - lastRawJson = continuationContext.get("last_raw_json", "") - - # Build overlap context: extract last ~100 characters from the response for overlap - overlapContext = "" - if lastRawJson: - # Get last 100 characters for overlap - overlapContext = lastRawJson[-100:].strip() - - # Build unified context showing structure hierarchy with cut point - unifiedContext = "" - if lastRawJson: - # Find break position in raw JSON - if incompletePart: - breakPos = lastRawJson.find(incompletePart) - if breakPos == -1: - # Try to find where JSON ends - breakPos = len(lastRawJson.rstrip()) - else: - # No incomplete part found - assume end of JSON - breakPos = len(lastRawJson.rstrip()) - - # Build intelligent context showing hierarchy - from modules.shared.jsonUtils import _buildIncompleteContext - unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) - elif incompletePart: - # Fallback: use incomplete part directly - unifiedContext = incompletePart - else: - unifiedContext = "Unable to extract context - response was completely broken" - - # Use the SAME template structure as in initial prompt - # Get contentType and contentStructureExample exactly like in _buildSectionGenerationPrompt - contentType = section.get("content_type", "paragraph") - contentStructureExample = self._getContentStructureExample(contentType) - - # Build the exact same JSON structure template as in initial prompt - structureTemplate = f"""JSON Structure Template: -{{ - "elements": [ - {{ - "type": "{contentType}", - "content": {contentStructureExample} - }} - ] -}} - -""" - - continuationPrompt = f"""{basePrompt} - ---- CONTINUATION REQUEST --- -The previous JSON response was incomplete. Continue from where it stopped. - -{structureTemplate}Context showing structure hierarchy with cut point: -{unifiedContext} - -Overlap Requirement: -To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. - -Last ~100 characters from previous response (repeat these at the start): -{overlapContext if overlapContext else "No overlap context available"} - -TASK: -1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) -2. Complete the incomplete element shown in the context above (marked with CUT POINT) -3. Continue generating the remaining content following the JSON structure template above -4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects - -CRITICAL: -- Your response must be valid JSON matching the structure template above -- Start with overlap (~100 chars) then continue seamlessly -- Complete the incomplete element and continue with remaining elements""" - return continuationPrompt + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation options = AiCallOptions( operationType=operationType, @@ -1572,7 +1278,7 @@ CRITICAL: prompt=generationPrompt, options=options, debugPrefix=f"{chapterId}_section_{sectionId}", - promptBuilder=buildSectionPromptWithContinuation, + promptBuilder=self.buildSectionPromptWithContinuation, promptArgs={ "section": section, "contentParts": [part], @@ -1581,7 +1287,10 @@ CRITICAL: "allSections": all_sections_list, "sectionIndex": sectionIndex, "isAggregation": isAggregation, - "services": self.services + "services": self.services, + "templateStructure": templateStructure, + "basePrompt": generationPrompt, + "language": language }, operationId=sectionOperationId, userPrompt=userPrompt, @@ -2203,7 +1912,7 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. sectionIndex: Optional[int] = None, isAggregation: bool = False, language: str = "en" - ) -> str: + ) -> tuple[str, str]: """Baue Prompt für Section-Generierung mit vollständigem Kontext.""" # Filtere None-Werte validParts = [p for p in contentParts if p is not None] @@ -2312,6 +2021,17 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. contentStructureExample = self._getContentStructureExample(contentType) + # Create template structure explicitly (not extracted from prompt) + # This ensures exact identity between initial and continuation prompts + templateStructure = f"""{{ + "elements": [ + {{ + "type": "{contentType}", + "content": {contentStructureExample} + }} + ] +}}""" + if isAggregation: prompt = f"""# TASK: Generate Section Content (Aggregation) @@ -2459,7 +2179,78 @@ Output requirements: ## CONTEXT {contextText if contextText else ""} """ - return prompt + return prompt, templateStructure + + async def buildSectionPromptWithContinuation( + self, + continuationContext: Any, + templateStructure: str, + basePrompt: str + ) -> str: + """Build section prompt with continuation context. Uses unified signature. + + Single unified implementation for all section content generation contexts. + + Note: All initial context (section, contentParts, userPrompt, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json + + # Build overlap context: extract last ~100 characters from the response for overlap + overlapContext = "" + if lastRawJson: + overlapContext = lastRawJson[-100:].strip() + + # Build unified context showing structure hierarchy with cut point + unifiedContext = "" + if lastRawJson: + # Find break position in raw JSON + if incompletePart: + breakPos = lastRawJson.find(incompletePart) + if breakPos == -1: + breakPos = len(lastRawJson.rstrip()) + else: + breakPos = len(lastRawJson.rstrip()) + + # Build intelligent context showing hierarchy + from modules.shared.jsonUtils import _buildIncompleteContext + unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) + elif incompletePart: + unifiedContext = incompletePart + else: + unifiedContext = "Unable to extract context - response was completely broken" + + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} + +--- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. + +JSON Structure Template: +{templateStructure} + +Context showing structure hierarchy with cut point: +{unifiedContext} + +Overlap Requirement: +To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. + +Last ~100 characters from previous response (repeat these at the start): +{overlapContext if overlapContext else "No overlap context available"} + +TASK: +1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) +2. Complete the incomplete element shown in the context above (marked with CUT POINT) +3. Continue generating the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response must be valid JSON matching the structure template above +- Start with overlap (~100 chars) then continue seamlessly +- Complete the incomplete element and continue with remaining elements""" + return continuationPrompt def _extractAndMergeMultipleJsonBlocks(self, responseText: str, contentType: str, sectionId: str) -> List[Dict[str, Any]]: """ diff --git a/modules/services/serviceAi/subStructureGeneration.py b/modules/services/serviceAi/subStructureGeneration.py index 085815fd..a8090009 100644 --- a/modules/services/serviceAi/subStructureGeneration.py +++ b/modules/services/serviceAi/subStructureGeneration.py @@ -107,52 +107,80 @@ class StructureGenerator: resultFormat="json" ) + structurePrompt, templateStructure = self._buildChapterStructurePrompt( + userPrompt=userPrompt, + contentParts=contentParts, + outputFormat=outputFormat + ) + # Create prompt builder for continuation support async def buildChapterStructurePromptWithContinuation( - continuationContext: Optional[Dict[str, Any]] = None, - **kwargs + continuationContext: Any, + templateStructure: str, + basePrompt: str ) -> str: - """Build chapter structure prompt with optional continuation context. Extracts chapter-specific parameters from kwargs.""" - # Extract parameters from kwargs (for chapter_structure use case) - userPrompt = kwargs.get("userPrompt", "") - contentParts = kwargs.get("contentParts", []) - outputFormat = kwargs.get("outputFormat", "txt") + """Build chapter structure prompt with continuation context. Uses unified signature. - basePrompt = self._buildChapterStructurePrompt( - userPrompt=userPrompt, - contentParts=contentParts, - outputFormat=outputFormat - ) + Note: All initial context (userPrompt, contentParts, outputFormat, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json - if continuationContext: - # Add continuation instructions - deliveredSummary = continuationContext.get("delivered_summary", "") - elementBeforeCutoff = continuationContext.get("element_before_cutoff", "") - cutOffElement = continuationContext.get("cut_off_element", "") + # Build overlap context: extract last ~100 characters from the response for overlap + overlapContext = "" + if lastRawJson: + overlapContext = lastRawJson[-100:].strip() + + # Build unified context showing structure hierarchy with cut point + unifiedContext = "" + if lastRawJson: + # Find break position in raw JSON + if incompletePart: + breakPos = lastRawJson.find(incompletePart) + if breakPos == -1: + breakPos = len(lastRawJson.rstrip()) + else: + breakPos = len(lastRawJson.rstrip()) - continuationText = f"{deliveredSummary}\n\n" - continuationText += "⚠️ CONTINUATION: Response was cut off. Generate ONLY the remaining content that comes AFTER the reference elements below.\n\n" - - if elementBeforeCutoff: - continuationText += "# REFERENCE: Last complete element (already delivered - DO NOT repeat):\n" - continuationText += f"{elementBeforeCutoff}\n\n" - - if cutOffElement: - continuationText += "# REFERENCE: Incomplete element (cut off here - DO NOT repeat):\n" - continuationText += f"{cutOffElement}\n\n" - - continuationText += "⚠️ CRITICAL: The elements above are REFERENCE ONLY. They are already delivered.\n" - continuationText += "Generate ONLY what comes AFTER these elements. DO NOT regenerate the entire JSON structure.\n" - continuationText += "Start directly with the next chapter that should follow.\n\n" - - return f"""{basePrompt} - -{continuationText} - -Continue generating the remaining chapters now. -""" + # Build intelligent context showing hierarchy + from modules.shared.jsonUtils import _buildIncompleteContext + unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) + elif incompletePart: + unifiedContext = incompletePart else: - return basePrompt + unifiedContext = "Unable to extract context - response was completely broken" + + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} + +--- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. + +JSON Structure Template: +{templateStructure} + +Context showing structure hierarchy with cut point: +{unifiedContext} + +Overlap Requirement: +To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. + +Last ~100 characters from previous response (repeat these at the start): +{overlapContext if overlapContext else "No overlap context available"} + +TASK: +1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) +2. Complete the incomplete element shown in the context above (marked with CUT POINT) +3. Continue generating the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response must be valid JSON matching the structure template above +- Start with overlap (~100 chars) then continue seamlessly +- Complete the incomplete element and continue with remaining elements""" + return continuationPrompt # Call AI with looping support # NOTE: Do NOT pass contentParts here - we only need metadata for structure generation @@ -167,7 +195,8 @@ Continue generating the remaining chapters now. promptArgs={ "userPrompt": userPrompt, "outputFormat": outputFormat, - "services": self.services + "templateStructure": templateStructure, + "basePrompt": structurePrompt }, useCaseId="chapter_structure", # REQUIRED: Explicit use case ID operationId=structureOperationId, @@ -280,7 +309,7 @@ Continue generating the remaining chapters now. userPrompt: str, contentParts: List[ContentPart], outputFormat: str - ) -> str: + ) -> tuple[str, str]: """Baue Prompt für Chapter-Struktur-Generierung.""" # Baue ContentParts-Index - filtere leere Parts heraus contentPartsIndex = "" @@ -336,6 +365,36 @@ Continue generating the remaining chapters now. language = self._getUserLanguage() logger.debug(f"Using language from services (user intention analysis) for structure generation: {language}") + # Create template structure explicitly (not extracted from prompt) + # This ensures exact identity between initial and continuation prompts + templateStructure = f"""{{ + "metadata": {{ + "title": "Document Title", + "language": "{language}" + }}, + "documents": [{{ + "id": "doc_1", + "title": "Document Title", + "filename": "document.{outputFormat}", + "outputFormat": "{outputFormat}", + "language": "{language}", + "chapters": [ + {{ + "id": "chapter_1", + "level": 1, + "title": "Chapter Title", + "contentParts": {{ + "extracted_part_id": {{ + "instruction": "Use extracted content with ALL relevant details from user request" + }} + }}, + "generationHint": "Detailed description including ALL relevant details from user request for this chapter", + "sections": [] + }} + ] + }}] +}}""" + prompt = f"""# TASK: Generate Chapter Structure This is a PLANNING task. Return EXACTLY ONE complete JSON object. Do not generate multiple JSON objects, alternatives, or variations. Do not use separators like "---" between JSON objects. @@ -463,5 +522,5 @@ For each chapter, verify: OUTPUT FORMAT: Start with {{ and end with }}. Do NOT use markdown code fences (```json). Do NOT add explanatory text before or after the JSON. Return ONLY the JSON object itself. """ - return prompt + return prompt, templateStructure diff --git a/modules/services/serviceGeneration/paths/codePath.py b/modules/services/serviceGeneration/paths/codePath.py index 336c30d8..0f3ffdad 100644 --- a/modules/services/serviceGeneration/paths/codePath.py +++ b/modules/services/serviceGeneration/paths/codePath.py @@ -233,6 +233,26 @@ class CodeGenerationPath: if not contentPartsIndex: contentPartsIndex = "\n(No content parts available)" + # Create template structure explicitly (not extracted from prompt) + templateStructure = f"""{{ + "metadata": {{ + "language": "{language}", + "projectType": "single_file|multi_file", + "projectName": "" + }}, + "files": [ + {{ + "id": "", + "filename": "", + "fileType": "", + "dependencies": [], + "imports": [], + "functions": [], + "classes": [] + }} + ] +}}""" + # Build structure generation prompt structurePrompt = f"""# TASK: Generate Code Project Structure @@ -302,6 +322,75 @@ For single-file projects, return one file. For multi-file projects, include ALL Return ONLY valid JSON matching the request above. """ + # Build continuation prompt builder + async def buildCodeStructurePromptWithContinuation( + continuationContext: Any, + templateStructure: str, + basePrompt: str + ) -> str: + """Build code structure prompt with continuation context. Uses unified signature. + + Note: All initial context (userPrompt, contentParts, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json + + # Build overlap context: extract last ~100 characters from the response for overlap + overlapContext = "" + if lastRawJson: + overlapContext = lastRawJson[-100:].strip() + + # Build unified context showing structure hierarchy with cut point + unifiedContext = "" + if lastRawJson: + # Find break position in raw JSON + if incompletePart: + breakPos = lastRawJson.find(incompletePart) + if breakPos == -1: + breakPos = len(lastRawJson.rstrip()) + else: + breakPos = len(lastRawJson.rstrip()) + + # Build intelligent context showing hierarchy + from modules.shared.jsonUtils import _buildIncompleteContext + unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) + elif incompletePart: + unifiedContext = incompletePart + else: + unifiedContext = "Unable to extract context - response was completely broken" + + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} + +--- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. + +JSON Structure Template: +{templateStructure} + +Context showing structure hierarchy with cut point: +{unifiedContext} + +Overlap Requirement: +To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. + +Last ~100 characters from previous response (repeat these at the start): +{overlapContext if overlapContext else "No overlap context available"} + +TASK: +1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) +2. Complete the incomplete element shown in the context above (marked with CUT POINT) +3. Continue generating the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response must be valid JSON matching the structure template above +- Start with overlap (~100 chars) then continue seamlessly +- Complete the incomplete element and continue with remaining elements""" + return continuationPrompt + # Use generic looping system with code_structure use case options = AiCallOptions( operationType=OperationTypeEnum.DATA_GENERATE, @@ -311,6 +400,13 @@ Return ONLY valid JSON matching the request above. structureJson = await self.services.ai.callAiWithLooping( prompt=structurePrompt, options=options, + promptBuilder=buildCodeStructurePromptWithContinuation, + promptArgs={ + "userPrompt": userPrompt, + "contentParts": contentParts, + "templateStructure": templateStructure, + "basePrompt": structurePrompt + }, useCaseId="code_structure", debugPrefix="code_structure_generation", contentParts=contentParts @@ -640,6 +736,18 @@ Return ONLY valid JSON matching the request above. ``` """ + # Create template structure explicitly (not extracted from prompt) + templateStructure = f"""{{ + "files": [ + {{ + "filename": "{filename}", + "content": "// Complete code here", + "functions": {json.dumps(functions, indent=2) if functions else '[]'}, + "classes": {json.dumps(classes, indent=2) if classes else '[]'} + }} + ] +}}""" + # Build base prompt contentPrompt = f"""# TASK: Generate Code File Content @@ -667,141 +775,77 @@ Generate complete, production-ready code with: 5. Type hints where appropriate Return ONLY valid JSON in this format: -{{ - "files": [ - {{ - "filename": "{filename}", - "content": "// Complete code here", - "functions": {json.dumps(functions, indent=2) if functions else '[]'}, - "classes": {json.dumps(classes, indent=2) if classes else '[]'} - }} - ] -}} +{templateStructure} """ # Build continuation prompt builder async def buildCodeContentPromptWithContinuation( - continuationContext: Optional[Dict[str, Any]] = None, - **kwargs + continuationContext: Any, + templateStructure: str, + basePrompt: str ) -> str: - """Build code content prompt with optional continuation context. Extracts code-specific parameters from kwargs.""" - # Extract parameters from kwargs (for code_content use case) - filename = kwargs.get("filename", "") - fileType = kwargs.get("fileType", "") - functions = kwargs.get("functions", []) - classes = kwargs.get("classes", []) - dependencies = kwargs.get("dependencies", []) - metadata = kwargs.get("metadata", {}) - userPrompt = kwargs.get("userPrompt", "") - contentParts = kwargs.get("contentParts", []) - contextInfo = kwargs.get("contextInfo", "") + """Build code content prompt with continuation context. Uses unified signature. - # Rebuild base prompt (same as initial prompt) - userRequestSection = "" - if userPrompt: - userRequestSection = f""" -## ORIGINAL USER REQUEST -``` -{userPrompt} -``` -""" + Note: All initial context (filename, fileType, functions, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json - contentPartsSection = "" - if contentParts: - relevantParts = [] - for part in contentParts: - usageHint = part.metadata.get('usageHint', '').lower() - originalFileName = part.metadata.get('originalFileName', '').lower() - filenameLower = filename.lower() - - if (filenameLower in usageHint or - filenameLower in originalFileName or - part.metadata.get('contentFormat') == 'reference' or - (part.data and len(str(part.data).strip()) > 0)): - relevantParts.append(part) - - if relevantParts: - contentPartsSection = "\n## AVAILABLE CONTENT PARTS\n" - for i, part in enumerate(relevantParts, 1): - contentFormat = part.metadata.get("contentFormat", "unknown") - originalFileName = part.metadata.get('originalFileName', 'N/A') - contentPartsSection += f"\n{i}. ContentPart ID: {part.id}\n" - contentPartsSection += f" Format: {contentFormat}\n" - contentPartsSection += f" Type: {part.typeGroup}\n" - contentPartsSection += f" Original file name: {originalFileName}\n" - contentPartsSection += f" Usage hint: {part.metadata.get('usageHint', 'N/A')}\n" - if part.data and isinstance(part.data, str) and len(part.data) < 2000: - contentPartsSection += f" Content preview: {part.data[:500]}...\n" + # Build overlap context: extract last ~100 characters from the response for overlap + overlapContext = "" + if lastRawJson: + overlapContext = lastRawJson[-100:].strip() - basePrompt = f"""# TASK: Generate Code File Content - -Generate complete, executable code for the file: {filename} -{userRequestSection}## FILE SPECIFICATIONS - -File Type: {fileType} -Language: {metadata.get('language', 'python') if metadata else 'python'} -{contentPartsSection} - -Required functions: -{json.dumps(functions, indent=2) if functions else 'None specified'} - -Required classes: -{json.dumps(classes, indent=2) if classes else 'None specified'} - -Dependencies on other files: {', '.join(dependencies) if dependencies else 'None'} -{contextInfo} - -Generate complete, production-ready code with: -1. Proper imports (including imports from other files in the project if dependencies exist) -2. All required functions and classes -3. Error handling -4. Documentation/docstrings -5. Type hints where appropriate - -Return ONLY valid JSON in this format: -{{ - "files": [ - {{ - "filename": "{filename}", - "content": "// Complete code here", - "functions": {json.dumps(functions, indent=2) if functions else '[]'}, - "classes": {json.dumps(classes, indent=2) if classes else '[]'} - }} - ] -}} -""" + # Build unified context showing structure hierarchy with cut point + unifiedContext = "" + if lastRawJson: + # Find break position in raw JSON + if incompletePart: + breakPos = lastRawJson.find(incompletePart) + if breakPos == -1: + breakPos = len(lastRawJson.rstrip()) + else: + breakPos = len(lastRawJson.rstrip()) + + # Build intelligent context showing hierarchy + from modules.shared.jsonUtils import _buildIncompleteContext + unifiedContext = _buildIncompleteContext(lastRawJson, breakPos) + elif incompletePart: + unifiedContext = incompletePart + else: + unifiedContext = "Unable to extract context - response was completely broken" - if continuationContext: - # Add continuation instructions - deliveredSummary = continuationContext.get("delivered_summary", "") - elementBeforeCutoff = continuationContext.get("element_before_cutoff", "") - cutOffElement = continuationContext.get("cut_off_element", "") - - continuationText = f"{deliveredSummary}\n\n" - continuationText += "⚠️ CONTINUATION: Response was cut off. Generate ONLY the remaining content that comes AFTER the reference elements below.\n\n" - - if elementBeforeCutoff: - continuationText += "# REFERENCE: Last complete element (already delivered - DO NOT repeat):\n" - continuationText += f"{elementBeforeCutoff}\n\n" - - if cutOffElement: - continuationText += "# REFERENCE: Incomplete element (cut off here - DO NOT repeat):\n" - continuationText += f"{cutOffElement}\n\n" - - continuationText += "⚠️ CRITICAL: The elements above are REFERENCE ONLY. They are already delivered.\n" - continuationText += "Generate ONLY what comes AFTER these elements. DO NOT regenerate the entire JSON structure.\n" - continuationText += "Continue generating the remaining code content now.\n\n" - - return f"""{basePrompt} + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} --- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. -{continuationText} +JSON Structure Template: +{templateStructure} -Continue generating the remaining code content now. -""" - else: - return basePrompt +Context showing structure hierarchy with cut point: +{unifiedContext} + +Overlap Requirement: +To ensure proper merging, your response MUST start by repeating approximately the last 100 characters from the previous response, then continue with new content. + +Last ~100 characters from previous response (repeat these at the start): +{overlapContext if overlapContext else "No overlap context available"} + +TASK: +1. Start your response by repeating the last ~100 characters shown above (for overlap/merging) +2. Complete the incomplete element shown in the context above (marked with CUT POINT) +3. Continue generating the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response must be valid JSON matching the structure template above +- Start with overlap (~100 chars) then continue seamlessly +- Complete the incomplete element and continue with remaining elements""" + return continuationPrompt # Use generic looping system with code_content use case options = AiCallOptions( @@ -823,7 +867,8 @@ Continue generating the remaining code content now. "userPrompt": userPrompt, "contentParts": contentParts, "contextInfo": contextInfo, - "services": self.services + "templateStructure": templateStructure, + "basePrompt": contentPrompt }, useCaseId="code_content", debugPrefix=f"code_content_{fileStructure.get('id', 'file')}", diff --git a/modules/shared/jsonUtils.py b/modules/shared/jsonUtils.py index 7769e0d9..c2ded569 100644 --- a/modules/shared/jsonUtils.py +++ b/modules/shared/jsonUtils.py @@ -5,6 +5,7 @@ import logging import re from typing import Any, Dict, List, Optional, Tuple, Union, Type, TypeVar from pydantic import BaseModel, ValidationError +from modules.datamodels.datamodelAi import ContinuationContext logger = logging.getLogger(__name__) @@ -843,8 +844,9 @@ def _extractOverlapFromElement(elem: Dict[str, Any], elemType: str) -> Optional[ def buildContinuationContext( allSections: List[Dict[str, Any]], lastRawResponse: Optional[str] = None, - useCaseId: Optional[str] = None -) -> Dict[str, Any]: + useCaseId: Optional[str] = None, + templateStructure: Optional[str] = None +) -> ContinuationContext: """ Build context information from accumulated sections for continuation prompt. @@ -854,14 +856,12 @@ def buildContinuationContext( allSections: List of ALL sections accumulated across ALL iterations lastRawResponse: Raw JSON response from last iteration (can be broken/incomplete) useCaseId: Optional use case ID to determine expected JSON structure + templateStructure: JSON structure template from initial prompt (MUST be identical) Returns: - Dict with delivered_summary, cut_off_element, element_before_cutoff, template_structure, - last_complete_part, incomplete_part, structure_context + ContinuationContext: Pydantic model with all continuation context information """ - context = { - "section_count": len(allSections), - } + section_count = len(allSections) # Build summary of delivered data (per-section counts) summary_lines = [] @@ -978,7 +978,7 @@ def buildContinuationContext( else: summary_lines.extend(summary_items) - context["delivered_summary"] = "\n".join(summary_lines) + delivered_summary = "\n".join(summary_lines) # Extract cut-off point using new algorithm # 1. Loop over all sections until finding incomplete section @@ -1029,9 +1029,6 @@ def buildContinuationContext( except Exception as e: logger.debug(f"Error extracting cut-off point: {e}") - context["element_before_cutoff"] = element_before_cutoff - context["cut_off_element"] = cut_off_element - # Extract overlap information for continuation prompt # GENERIC overlap extraction: handles elements of any size, including long strings # Strategy: Extract last N elements, but if an element is very large, extract only a portion @@ -1067,38 +1064,36 @@ def buildContinuationContext( if overlapStrings: overlapString = ",\n".join(overlapStrings) - context["overlap_elements"] = overlapElements - context["overlap_string"] = overlapString + # Store raw JSON response and extract structure context + last_raw_json = lastRawResponse or "" + last_complete_part = "" + incomplete_part = "" + structure_context = "" - # Store raw JSON response for prompt builder to check if lastRawResponse: - context["last_raw_json"] = lastRawResponse - # Extract JSON structure context for continuation prompt - # This provides: template structure, last complete part, incomplete part, structure context + # This provides: last complete part, incomplete part, structure context + # NOTE: template_structure is now passed as parameter, not extracted try: structureContext = extractJsonStructureContext(lastRawResponse, useCaseId) - context["template_structure"] = structureContext.get("template_structure", "") - context["last_complete_part"] = structureContext.get("last_complete_part", "") - context["incomplete_part"] = structureContext.get("incomplete_part", "") - context["structure_context"] = structureContext.get("structure_context", "") - # Log if extraction succeeded but returned empty values - if not context["template_structure"] and not context["structure_context"]: - logger.debug(f"JSON structure context extraction returned empty values for useCaseId={useCaseId}") + last_complete_part = structureContext.get("last_complete_part", "") + incomplete_part = structureContext.get("incomplete_part", "") + structure_context = structureContext.get("structure_context", "") except Exception as e: logger.warning(f"Error extracting JSON structure context: {e}", exc_info=True) - context["template_structure"] = "" - context["last_complete_part"] = "" - context["incomplete_part"] = "" - context["structure_context"] = "" - else: - context["last_raw_json"] = "" - context["template_structure"] = "" - context["last_complete_part"] = "" - context["incomplete_part"] = "" - context["structure_context"] = "" - return context + # Return ContinuationContext Pydantic model + return ContinuationContext( + section_count=section_count, + delivered_summary=delivered_summary, + cut_off_element=cut_off_element, + element_before_cutoff=element_before_cutoff, + template_structure=templateStructure, # Use passed parameter, not extracted + last_complete_part=last_complete_part, + incomplete_part=incomplete_part, + structure_context=structure_context, + last_raw_json=last_raw_json + ) def extractJsonStructureContext(