intelligent generic json merger for ai responses

modules/services/serviceAi/mainServiceAi.py

@@ -161,11 +161,12 @@ class AiService:
         prompt: str,
         placeholders: Optional[List[PromptPlaceholder]] = None,
         options: Optional[AiCallOptions] = None,
-        loopInstruction: Optional[str] = None
+        loopInstructionFormat: Optional[str] = None
         ) -> str:
         """Planning AI call for task planning, action planning, action selection, etc."""
         await self._ensureAiObjectsInitialized()
-        return await self.coreAi.callAiPlanning(prompt, placeholders, options, loopInstruction)
+        # Always use "json" for planning calls since they return JSON
+        return await self.coreAi.callAiPlanning(prompt, placeholders, options, "json")
 
     async def callAiDocuments(
         self,
@@ -177,7 +178,8 @@ class AiService:
         ) -> Union[str, Dict[str, Any]]:
         """Document generation AI call for all non-planning calls."""
         await self._ensureAiObjectsInitialized()
-        return await self.coreAi.callAiDocuments(prompt, documents, options, outputFormat, title)
+        # Use "json" for document generation calls since they return JSON
+        return await self.coreAi.callAiDocuments(prompt, documents, options, outputFormat, title, "json")
 
 
     def sanitizePromptContent(self, content: str, contentType: str = "text") -> str:

modules/services/serviceAi/subCoreAi.py

@@ -6,6 +6,14 @@ from modules.shared.debugLogger import writeDebugFile
 
 logger = logging.getLogger(__name__)
 
+# Loop instruction texts for different formats
+LoopInstructionTexts = {
+    "json": """CRITICAL: You MUST set the "continuation" field in your JSON response. If you cannot complete the full response, deliver the possible part and set "continuation" to a brief description of what still needs to be generated. If you can complete the full response, set "continuation" to null.""",
+    # Add more formats here as needed
+    # "xml": "...",
+    # "text": "...",
+}
+
 
 class SubCoreAi:
     """Core AI operations including image analysis, text generation, and planning calls."""
@@ -26,7 +34,7 @@ class SubCoreAi:
         prompt: str,
         options: AiCallOptions,
         debugPrefix: str = "ai_call",
-        loopInstruction: str = None
+        loopInstructionFormat: str = None
         ) -> str:
         """
         Shared core function for AI calls with looping system.
@@ -37,7 +45,7 @@ class SubCoreAi:
             prompt: The prompt to send to AI
             options: AI call configuration options
             debugPrefix: Prefix for debug file names
-            loopInstruction: If provided, replaces LOOP_INSTRUCTION placeholder and includes in continuation prompts
+            loopInstructionFormat: If provided, replaces LOOP_INSTRUCTION placeholder and includes in continuation prompts
             
         Returns:
             Complete AI response after all iterations
@@ -46,19 +54,17 @@ class SubCoreAi:
         iteration = 0
         accumulatedContent = []
         
-        logger.debug(f"Starting AI call with looping (debug prefix: {debugPrefix}, loopInstruction: {loopInstruction is not None})")
+        logger.debug(f"Starting AI call with looping (debug prefix: {debugPrefix}, loopInstructionFormat: {loopInstructionFormat is not None})")
         
-        # Import debug logger for use in iterations
        
-        # Store original prompt to preserve LOOP_INSTRUCTION placeholder
+        # Determine loopInstruction based on loopInstructionFormat (before iterations)
-        originalPrompt = prompt
+        if not loopInstructionFormat:
-        
+            loopInstruction = ""
-        # Handle LOOP_INSTRUCTION placeholder replacement for first iteration
+        elif loopInstructionFormat in LoopInstructionTexts:
-        if loopInstruction and iteration == 0:
+            loopInstruction = LoopInstructionTexts[loopInstructionFormat]
-            if "LOOP_INSTRUCTION" not in prompt:
+        else:
-                raise ValueError("LOOP_INSTRUCTION placeholder not found in prompt when loopInstruction provided")
+            logger.error(f"Unsupported loopInstructionFormat for prompt: {loopInstructionFormat}")
-            prompt = prompt.replace("LOOP_INSTRUCTION", loopInstruction)
+            loopInstruction = ""
-            logger.debug("Replaced LOOP_INSTRUCTION placeholder with provided instruction")
         
         while iteration < max_iterations:
             iteration += 1
@@ -66,12 +72,17 @@ class SubCoreAi:
             
             # Build iteration prompt
             if iteration == 1:
-                iterationPrompt = prompt
+                if "LOOP_INSTRUCTION" in prompt:
+                    iterationPrompt = prompt.replace("LOOP_INSTRUCTION", loopInstruction)
+                else:
+                    iterationPrompt = prompt
             elif loopInstruction and iteration > 1:
-                # Only use continuation logic if loopInstruction is provided
+                continuationContent = self._buildContinuationContent(accumulatedContent, iteration)
-                iterationPrompt = self._buildContinuationPrompt(originalPrompt, accumulatedContent, iteration, loopInstruction)
+                if "LOOP_INSTRUCTION" in prompt:
+                    iterationPrompt = prompt.replace("LOOP_INSTRUCTION", f"{loopInstruction}\n\n{continuationContent}")
+                else:
+                    iterationPrompt = prompt
             else:
-                # No looping - use original prompt
                 iterationPrompt = prompt
             
             # Make AI call
@@ -103,14 +114,27 @@ class SubCoreAi:
                     logger.warning(f"Iteration {iteration}: Empty response, stopping")
                     break
                 
-                # Check if this is a continuation response (only if loopInstruction is provided)
+                # Check if this is a continuation response (only for supported formats)
-                if loopInstruction and "[CONTINUE:" in result:
+                if loopInstructionFormat in LoopInstructionTexts:
-                    # Extract the content before the continuation marker
+                    try:
-                    contentPart = result.split("[CONTINUE:")[0].strip()
+                        import json
-                    if contentPart:
+                        # Try to parse as JSON to check for continuation attribute
-                        accumulatedContent.append(contentPart)
+                        parsed_result = json.loads(result)
-                    logger.debug(f"Iteration {iteration}: Continuation detected, continuing...")
+                        if isinstance(parsed_result, dict) and parsed_result.get("continuation") is not None:
-                    continue
+                            # This is a continuation response
+                            accumulatedContent.append(result)
+                            logger.debug(f"Iteration {iteration}: Continuation detected in JSON, continuing...")
+                            continue
+                        else:
+                            # This is the final response (continuation is null or missing)
+                            accumulatedContent.append(result)
+                            logger.debug(f"Iteration {iteration}: Final response received")
+                            break
+                    except json.JSONDecodeError:
+                        # Not JSON, treat as final response
+                        accumulatedContent.append(result)
+                        logger.warning(f"Iteration {iteration}: Non-JSON response received")
+                        break
                 else:
                     # This is the final response
                     accumulatedContent.append(result)
@@ -124,8 +148,8 @@ class SubCoreAi:
         if iteration >= max_iterations:
             logger.warning(f"AI call stopped after maximum iterations ({max_iterations})")
         
-        # Combine all accumulated content
+        # Intelligently merge JSON content from all iterations
-        final_result = "\n\n".join(accumulatedContent) if accumulatedContent else ""
+        final_result = self._mergeJsonContent(accumulatedContent) if accumulatedContent else ""
         
         # Write final result to debug file
         writeDebugFile(final_result, f"{debugPrefix}_final_result", None)
@@ -133,34 +157,92 @@ class SubCoreAi:
         logger.info(f"AI call completed: {len(accumulatedContent)} parts from {iteration} iterations")
         return final_result
 
-    def _buildContinuationPrompt(
+    def _buildContinuationContent(
         self,
-        base_prompt: str,
         accumulatedContent: List[str],
-        iteration: int,
+        iteration: int
-        loopInstruction: str = None
         ) -> str:
         """
-        Build a prompt for continuation iterations.
+        Build continuation content for follow-up iterations.
         """
-        continuation_instructions = f"""
+        # Extract continuation description from the last response if it exists
+        continuation_description = ""
+        if accumulatedContent:
+            try:
+                import json
+                last_response = accumulatedContent[-1]
+                parsed_response = json.loads(last_response)
+                if isinstance(parsed_response, dict) and parsed_response.get("continuation"):
+                    continuation_description = parsed_response["continuation"]
+            except (json.JSONDecodeError, KeyError):
+                pass
         
-CONTINUATION REQUEST (Iteration {iteration}):
+        continuation_content = f"""CONTINUATION REQUEST (Iteration {iteration}):
-You are continuing from a previous response. Please continue generating content from where you left off.
+Continue generating content from where you left off.
 
 IMPORTANT: 
+- Maintain the same JSON structure
 - Continue from the exact point where you stopped
-- Maintain the same format and structure
+- If you can complete the remaining content: set "continuation": null
-- {loopInstruction if loopInstruction else "If you cannot complete the full response, end with: [CONTINUE: brief description of what still needs to be generated]"}
+- If content is still too long, deliver next part and set "continuation": "description of remaining content"
-- Only stop when the response is completely generated
 
-Previous content generated:
+Previous content:
 {chr(10).join(accumulatedContent[-1:]) if accumulatedContent else "None"}
 
-Continue generating content now:
+{f"Continuation needed: {continuation_description}" if continuation_description else ""}
-"""
 
-        return f"{base_prompt}{continuation_instructions}"
+Continue generating content now."""
+        
+        return continuation_content
+
+    def _mergeJsonContent(self, accumulatedContent: List[str]) -> str:
+        """
+        Generic JSON merger that combines all lists from multiple iterations.
+        Structure: root attributes + 1..n lists that get merged together.
+        """
+        if not accumulatedContent:
+            return ""
+        
+        if len(accumulatedContent) == 1:
+            return accumulatedContent[0]
+        
+        try:
+            import json
+            
+            # Parse all JSON responses
+            parsed_responses = []
+            for content in accumulatedContent:
+                try:
+                    parsed = json.loads(content)
+                    parsed_responses.append(parsed)
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Failed to parse JSON content: {str(e)}")
+                    continue
+            
+            if not parsed_responses:
+                return accumulatedContent[0]  # Return first response if all parsing failed
+            
+            # Start with first response as base
+            merged = parsed_responses[0].copy()
+            
+            # Merge all lists from all responses
+            for response in parsed_responses[1:]:
+                for key, value in response.items():
+                    if isinstance(value, list) and key in merged and isinstance(merged[key], list):
+                        # Merge lists by extending
+                        merged[key].extend(value)
+                    elif key not in merged:
+                        # Add new fields
+                        merged[key] = value
+            
+            # Mark as complete
+            merged["continuation"] = None
+            
+            return json.dumps(merged, indent=2)
+                
+        except Exception as e:
+            logger.error(f"Error merging JSON content: {str(e)}")
+            return accumulatedContent[0]  # Return first response on error
 
     def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str:
         """
@@ -214,7 +296,7 @@ Continue generating content now:
         prompt: str,
         placeholders: Optional[List[PromptPlaceholder]] = None,
         options: Optional[AiCallOptions] = None,
-        loopInstruction: Optional[str] = None
+        loopInstructionFormat: Optional[str] = None
         ) -> str:
         """
         Planning AI call for task planning, action planning, action selection, etc.
@@ -238,7 +320,7 @@ Continue generating content now:
             full_prompt = prompt
         
         # Use shared core function with planning-specific debug prefix
-        return await self._callAiWithLooping(full_prompt, options, "planning", loopInstruction=loopInstruction)
+        return await self._callAiWithLooping(full_prompt, options, "planning", loopInstructionFormat=loopInstructionFormat)
 
     # Document Generation AI Call
     async def callAiDocuments(
@@ -247,7 +329,8 @@ Continue generating content now:
         documents: Optional[List[ChatDocument]] = None,
         options: Optional[AiCallOptions] = None,
         outputFormat: Optional[str] = None,
-        title: Optional[str] = None
+        title: Optional[str] = None,
+        loopInstructionFormat: Optional[str] = None
         ) -> Union[str, Dict[str, Any]]:
         """
         Document generation AI call for all non-planning calls.
@@ -276,7 +359,7 @@ Continue generating content now:
                 logger.info("No documents provided - using direct generation")
                 extracted_content = None
             generation_prompt = await self._buildGenerationPrompt(prompt, extracted_content, outputFormat, title)
-            generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation", loopInstruction="If you cannot complete the full response, end with: [CONTINUE: brief description of what still needs to be generated]")
+            generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation", loopInstructionFormat=loopInstructionFormat)
             
             # Parse the generated JSON
             try:
@@ -337,7 +420,7 @@ Continue generating content now:
             result = await self.services.ai.documentProcessor.callAiText(prompt, documents, options)
         else:
             # Use shared core function for direct text calls
-            result = await self._callAiWithLooping(prompt, options, "text", loopInstruction=None)
+            result = await self._callAiWithLooping(prompt, options, "text", loopInstructionFormat=None)
         
         return result
 

modules/services/serviceGeneration/subPromptBuilder.py

@@ -196,15 +196,13 @@ Consider the user's intent and the most logical way to organize the extracted co
             services.utils.debugLogToFile(f"Generic prompt analysis failed: {str(e)}", "PROMPT_BUILDER")
     
     # Always use the proper generation prompt template with LOOP_INSTRUCTION
-    result = f"""You are an AI assistant that generates structured JSON content for document creation.
+    result = f"""Generate structured JSON content for document creation.
 
 USER REQUEST: "{userPrompt}"
 DOCUMENT TITLE: "{title}"
 TARGET FORMAT: {outputFormat}
 
-TASK: Generate JSON content that fulfills the user's request.
+Return ONLY this JSON structure:
-
-CRITICAL: You MUST return ONLY valid JSON in this exact structure:
 {{
   "metadata": {{
     "title": "{title}",
@@ -241,16 +239,18 @@ CRITICAL: You MUST return ONLY valid JSON in this exact structure:
         }}
       ]
     }}
-  ]
+  ],
+  "continuation": null
 }}
 
-IMPORTANT: 
+RULES:
-- Return ONLY the JSON structure above
+- Fill sections with content based on the user request
-- Do NOT include any text before or after the JSON
+- Use appropriate content_type: "heading", "paragraph", "table", "list"
-- Fill in the actual content based on the user request: {userPrompt}
+- If content is too long: deliver partial result and set "continuation": "description of remaining content"
-- If the content is too large, you can split it into multiple sections
+- If content fits: deliver complete result and set "continuation": null
-- Each section should have a unique id and appropriate content_type
+- Split large content into multiple sections if needed
-- LOOP_INSTRUCTION
+
+LOOP_INSTRUCTION
 """
     
     # Debug output
@@ -440,15 +440,13 @@ async def buildGenerationPrompt(
         services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Using static template instead of AI call", "PROMPT_BUILDER")
         
         # Return static generation prompt template
-        result = f"""You are an AI assistant that generates structured JSON content for document creation.
+        result = f"""Generate structured JSON content for document creation.
 
 USER REQUEST: "{safeUserPrompt}"
 DOCUMENT TITLE: "{title}"
 TARGET FORMAT: {outputFormat}
 
-TASK: Generate JSON content that fulfills the user's request.
+Return ONLY this JSON structure:
-
-CRITICAL: You MUST return ONLY valid JSON in this exact structure:
 {{
   "metadata": {{
     "title": "{title}",
@@ -485,15 +483,14 @@ CRITICAL: You MUST return ONLY valid JSON in this exact structure:
         }}
       ]
     }}
-  ]
+  ],
+  "continuation": null
 }}
 
-IMPORTANT: 
+RULES:
-- Return ONLY the JSON structure above
+- Fill sections with content based on the user request
-- Do NOT include any text before or after the JSON
+- Use appropriate content_type: "heading", "paragraph", "table", "list"
-- Fill in the actual content based on the user request: {safeUserPrompt}
+- Split large content into multiple sections if needed
-- If the content is too large, you can split it into multiple sections
-- Each section should have a unique id and appropriate content_type
 
 LOOP_INSTRUCTION
 """
@@ -509,71 +506,6 @@ LOOP_INSTRUCTION
         return f"Generate a comprehensive {outputFormat} document titled '{title}' based on the extracted content. User requirements: {userPrompt}"
 
 
-def _getFormatRules(outputFormat: str) -> str:
-    """
-    Get format-specific rules for the generation prompt.
-    """
-    format_rules = {
-        "xlsx": """
-XLSX Format Rules:
-- Create tables with clear headers and organized data
-- Use appropriate column widths and formatting
-- Include summary information if relevant
-- Ensure data is properly structured for spreadsheet analysis
-""",
-        "pdf": """
-PDF Format Rules:
-- Create professional document layout
-- Use appropriate headings and sections
-- Include proper spacing and formatting
-- Ensure content is well-organized and readable
-""",
-        "docx": """
-DOCX Format Rules:
-- Create professional document layout
-- Use appropriate headings and sections
-- Include proper spacing and formatting
-- Ensure content is well-organized and readable
-""",
-        "html": """
-HTML Format Rules:
-- Create clean, semantic HTML structure
-- Use appropriate tags for content organization
-- Include proper styling classes
-- Ensure content is accessible and well-formatted
-""",
-        "json": """
-JSON Format Rules:
-- Create well-structured JSON data
-- Use appropriate nesting and organization
-- Include metadata and context information
-- Ensure data is properly formatted and valid
-""",
-        "csv": """
-CSV Format Rules:
-- Create clear, organized tabular data
-- Use appropriate headers and data types
-- Ensure proper CSV formatting
-- Include all relevant data in structured format
-""",
-        "txt": """
-TXT Format Rules:
-- Create clean, readable text format
-- Use appropriate spacing and organization
-- Include clear headings and sections
-- Ensure content is well-structured and easy to read
-"""
-    }
-    
-    return format_rules.get(outputFormat.lower(), f"""
-{outputFormat.upper()} Format Rules:
-- Create well-structured content appropriate for {outputFormat}
-- Use appropriate formatting and organization
-- Ensure content is clear and professional
-- Include all relevant information in proper format
-""")
-
-
 async def _parseExtractionIntent(userPrompt: str, outputFormat: str, aiService=None, services=None) -> str:
     """
     Parse user prompt to extract the core extraction intent.

modules/workflows/methods/methodOutlook.py

@@ -1181,7 +1181,8 @@ Return JSON:
 {{
   "subject": "subject line",
   "body": "email body (HTML allowed)",
-  "attachments": ["doc_ref1", "doc_ref2"]
+  "attachments": ["doc_ref1", "doc_ref2"],
+  "continuation": null
 }}
 
 LOOP_INSTRUCTION"""

modules/workflows/processing/core/taskPlanner.py

@@ -108,7 +108,8 @@ class TaskPlanner:
             prompt = await self.services.ai.callAiPlanning(
                 prompt=taskPlanningPromptTemplate,
                 placeholders=placeholders,
-                options=options
+                options=options,
+                loopInstructionFormat="json"
             )
             
             # Check if AI response is valid

modules/workflows/processing/shared/promptGenerationActionsActionplan.py

@@ -79,7 +79,8 @@ Generate the next action to advance toward completing the task objective.
             "description": "What this action accomplishes",
             "userMessage": "User-friendly message in language '{{KEY:USER_LANGUAGE}}'"
         }
-    ]
+    ],
+    "continuation": null
 }
 ```
 
@@ -95,7 +96,8 @@ Generate the next action to advance toward completing the task objective.
             "description": "Extract data from documents",
             "userMessage": "Extracting data from documents"
         }
-    ]
+    ],
+    "continuation": null
 }
 ```
 
@@ -123,7 +125,7 @@ Generate the next action to advance toward completing the task objective.
 - **Keep messages concise** but informative
 
 ## 🚀 Response Format
-Return ONLY the JSON object with complete action objects. If you cannot complete the full response, ensure each action object is complete and valid.
+Return ONLY the JSON object with complete action objects. If you cannot complete the full response, set "continuation" to a brief description of what still needs to be generated. If you can complete the response, keep "continuation" as null.
 LOOP_INSTRUCTION
 """
 
@@ -173,7 +175,8 @@ def generateResultReviewPrompt(context: Any) -> PromptBundle:
         "met_criteria": ["criteria1", "criteria2"],
         "unmet_criteria": ["criteria3", "criteria4"],
         "confidence": 0.85,
-        "userMessage": "User-friendly message explaining the validation result in language '{{KEY:USER_LANGUAGE}}'"
+        "userMessage": "User-friendly message explaining the validation result in language '{{KEY:USER_LANGUAGE}}'",
+        "continuation": null
     }
     ```
 
@@ -231,8 +234,9 @@ def generateResultReviewPrompt(context: Any) -> PromptBundle:
     - "Break down complex objective into smaller, focused actions"
     - "Verify document references before processing"
 
-    ## 🚀 Response Format
+
-    Return ONLY the JSON object. Do not include any explanatory text."""
+LOOP_INSTRUCTION
+"""
 
     return PromptBundle(prompt=template, placeholders=placeholders)
 

modules/workflows/processing/shared/promptGenerationTaskplan.py

@@ -85,7 +85,8 @@ Break down user requests into logical, executable task steps.
             "estimated_complexity": "low|medium|high",
             "userMessage": "What this task will accomplish in language '{{KEY:USER_LANGUAGE}}'"
         }
-    ]
+    ],
+    "continuation": null
 }
 ```
 
@@ -128,8 +129,6 @@ Break down user requests into logical, executable task steps.
 - **Medium**: Multi-action tasks for one topic (3-5 actions)
 - **High**: Complex strategic tasks (6+ actions)
 
-## 🚀 Response Format
-Return ONLY the JSON object with complete task objects. If you cannot complete the full response, ensure each task object is complete and valid.
 LOOP_INSTRUCTION
 """
     return PromptBundle(prompt=template, placeholders=placeholders)