fixing ai continuation loop context

modules/datamodels/datamodelChat.py

@@ -877,6 +877,16 @@ class ReviewResult(BaseModel):
     userMessage: Optional[str] = Field(
         None, description="User-friendly message in user's language"
     )
+    # NEW: Concrete next action guidance (when status is "continue")
+    nextAction: Optional[str] = Field(
+        None, description="Specific action to execute next (e.g., 'ai.convert', 'ai.process', 'ai.reformat')"
+    )
+    nextActionParameters: Optional[Dict[str, Any]] = Field(
+        None, description="Parameters for the next action (e.g., {'fromFormat': 'json', 'toFormat': 'csv'})"
+    )
+    nextActionObjective: Optional[str] = Field(
+        None, description="What this specific action will achieve"
+    )
 
 
 registerModelLabels(

modules/services/serviceAi/mainServiceAi.py

@@ -166,8 +166,10 @@ Respond with ONLY a JSON object in this exact format:
         debugPrefix: str = "ai_call",
         promptBuilder: Optional[callable] = None,
         promptArgs: Optional[Dict[str, Any]] = None,
-        operationId: Optional[str] = None
+        operationId: Optional[str] = None,
-        ) -> str:
+        userPrompt: Optional[str] = None,
+        workflowIntent: Optional[Dict[str, Any]] = None
+    ) -> str:
         """
         Shared core function for AI calls with repair-based looping system.
         Automatically repairs broken JSON and continues generation seamlessly.
@@ -216,8 +218,20 @@ Respond with ONLY a JSON object in this exact format:
                 if not lastRawResponse:
                     logger.warning(f"Iteration {iteration}: No previous response available for continuation!")
                 
+                # CRITICAL: Add workflowIntent (actionIntent) to continuationContext for DoD-based progress filtering
+                # This allows buildGenerationPrompt to filter progress stats based on Definition of Done KPIs
+                if workflowIntent:
+                    continuationContext['taskIntent'] = workflowIntent  # Keep key name 'taskIntent' for compatibility
+                
+                # Filter promptArgs to only include parameters that buildGenerationPrompt accepts
+                # buildGenerationPrompt accepts: outputFormat, userPrompt, title, extracted_content, continuationContext
+                filteredPromptArgs = {
+                    k: v for k, v in promptArgs.items() 
+                    if k in ['outputFormat', 'userPrompt', 'title', 'extracted_content']
+                }
+                
                 # Rebuild prompt with continuation context using the provided prompt builder
-                iterationPrompt = await promptBuilder(**promptArgs, continuationContext=continuationContext)
+                iterationPrompt = await promptBuilder(**filteredPromptArgs, continuationContext=continuationContext)
             else:
                 # First iteration - use original prompt
                 iterationPrompt = prompt
@@ -270,11 +284,6 @@ Respond with ONLY a JSON object in this exact format:
                 # Store raw response for continuation (even if broken)
                 lastRawResponse = result
                 
-                # Check for complete_response flag in raw response (before parsing)
-                import re
-                if re.search(r'"complete_response"\s*:\s*true', result, re.IGNORECASE):
-                    pass  # Flag detected, will stop in _shouldContinueGeneration
-                
                 # Extract sections from response (handles both valid and broken JSON)
                 extractedSections, wasJsonComplete, parsedResult = self._extractSectionsFromResponse(result, iteration, debugPrefix)
                 
@@ -288,12 +297,13 @@ Respond with ONLY a JSON object in this exact format:
                         self.services.chat.progressLogUpdate(iterationOperationId, 0.8, f"Extracted {len(extractedSections)} sections")
                 
                 if not extractedSections:
-                    # If we're in continuation mode and JSON was incomplete, don't stop - continue to allow retry
+                    # CRITICAL: If JSON was incomplete/broken, continue even if no sections extracted
-                    if iteration > 1 and not wasJsonComplete:
+                    # This allows the AI to retry and complete the broken JSON
-                        logger.warning(f"Iteration {iteration}: No sections extracted from continuation fragment, continuing for another attempt")
+                    if not wasJsonComplete:
+                        logger.warning(f"Iteration {iteration}: No sections extracted from broken JSON, continuing for another attempt")
                         continue
-                    # Otherwise, stop if no sections
+                    # If JSON was complete but no sections extracted - this is an error, stop
-                    logger.warning(f"Iteration {iteration}: No sections extracted, stopping")
+                    logger.warning(f"Iteration {iteration}: No sections extracted from complete JSON, stopping")
                     break
                 
                 # Merge new sections with existing sections intelligently
@@ -302,7 +312,28 @@ Respond with ONLY a JSON object in this exact format:
                 allSections = self._mergeSectionsIntelligently(allSections, extractedSections, iteration)
                 
                 # Check if we should continue (completion detection)
-                if self._shouldContinueGeneration(allSections, iteration, wasJsonComplete, result):
+                # Extract user prompt from promptArgs if available
+                extractedUserPrompt = userPrompt
+                if not extractedUserPrompt and promptArgs:
+                    extractedUserPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt")
+                if not extractedUserPrompt:
+                    # Try to extract from original prompt
+                    if "User request:" in prompt:
+                        try:
+                            extractedUserPrompt = prompt.split("User request:")[1].split("\n")[0].strip('"')
+                        except:
+                            pass
+                
+                shouldContinue = self._shouldContinueGeneration(
+                    allSections, 
+                    iteration, 
+                    wasJsonComplete, 
+                    result,
+                    userPrompt=extractedUserPrompt,
+                    workflowIntent=workflowIntent
+                )
+                
+                if shouldContinue:
                     # Finish iteration operation (will continue with next iteration)
                     if iterationOperationId:
                         self.services.chat.progressLogFinish(iterationOperationId, True)
@@ -416,16 +447,25 @@ Respond with ONLY a JSON object in this exact format:
                 continue
             
             # Strategy 4: Structural Analysis - detect continuation
-            # For code_block: if last section is code_block and new section is also code_block, merge
+            # For code_block and table: if last section matches new section type, merge them
             if mergedSections:
                 lastSection = mergedSections[-1]
-                if (lastSection.get("content_type") == "code_block" and 
+                lastContentType = lastSection.get("content_type")
-                    newSection.get("content_type") == "code_block"):
+                newContentType = newSection.get("content_type")
-                    # Both are code blocks - merge them
+                
+                # Both are code blocks - merge them
+                if lastContentType == "code_block" and newContentType == "code_block":
                     mergedSections[-1] = self._mergeSectionContent(lastSection, newSection, iteration)
                     merged = True
                     logger.debug(f"Iteration {iteration}: Merged code_block sections by structural analysis")
                     continue
+                
+                # Both are tables - merge them (common case for broken JSON iterations)
+                if lastContentType == "table" and newContentType == "table":
+                    mergedSections[-1] = self._mergeSectionContent(lastSection, newSection, iteration)
+                    merged = True
+                    logger.debug(f"Iteration {iteration}: Merged table sections by structural analysis")
+                    continue
             
             # No merge strategy matched - add as new section
             if not merged:
@@ -483,6 +523,26 @@ Respond with ONLY a JSON object in this exact format:
                             if parts and len(parts[-1]) < 5:  # Last part is very short - might be incomplete
                                 return True
         
+        # Check table for incomplete rows
+        if contentType == "table":
+            rows = lastElement.get("rows", [])
+            if rows:
+                # Check if last row is incomplete (ends with incomplete data)
+                lastRow = rows[-1] if isinstance(rows, list) else []
+                if isinstance(lastRow, list) and lastRow:
+                    # Check if last row ends with incomplete data (e.g., incomplete string)
+                    lastCell = lastRow[-1] if lastRow else ""
+                    if isinstance(lastCell, str):
+                        # If last cell is incomplete (ends with quote or is very short), section might be incomplete
+                        if lastCell.endswith('"') or (len(lastCell) < 3 and lastCell):
+                            return True
+                    # Also check if last row doesn't have expected number of columns (if headers exist)
+                    headers = lastElement.get("headers", [])
+                    if headers and isinstance(headers, list):
+                        expectedCols = len(headers)
+                        if len(lastRow) < expectedCols:
+                            return True
+        
         # Check paragraph/text for incomplete sentences
         if contentType in ["paragraph", "heading"]:
             text = lastElement.get("text", "")
@@ -495,6 +555,52 @@ Respond with ONLY a JSON object in this exact format:
                     if len(textStripped) < 20:
                         return True
         
+        # Check lists for incomplete items
+        if contentType in ["bullet_list", "numbered_list"]:
+            items = lastElement.get("items", [])
+            if items and isinstance(items, list):
+                # Check if last item is incomplete (very short or ends with incomplete string)
+                lastItem = items[-1] if items else None
+                if isinstance(lastItem, str) and len(lastItem) < 3:
+                    return True
+                # Check if items array seems incomplete (e.g., expected count not reached)
+                # This is harder to detect without context, so we rely on other heuristics
+        
+        # Check image for incomplete base64 data
+        if contentType == "image":
+            imageData = lastElement.get("base64Data", "")
+            if imageData:
+                # Base64 strings should end with padding ('=' or '==')
+                # If it doesn't, it might be incomplete
+                stripped = imageData.rstrip()
+                if stripped and not stripped.endswith(('=', '==')):
+                    # Check if it's a valid base64 character sequence that was cut off
+                    # Base64 uses A-Z, a-z, 0-9, +, /, and = for padding
+                    if len(stripped) > 0 and stripped[-1] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=':
+                        return True
+                    # If length is not a multiple of 4 (base64 requirement), it might be incomplete
+                    if len(stripped) % 4 != 0:
+                        return True
+        
+        # GENERIC CHECK: Look for incomplete structures in any element
+        # Check if element has arrays/lists that might be incomplete
+        for key, value in lastElement.items():
+            if isinstance(value, list) and len(value) > 0:
+                # Check last item in list
+                lastItem = value[-1]
+                if isinstance(lastItem, str):
+                    # If last string item is very short, might be incomplete
+                    if len(lastItem) < 3:
+                        return True
+                elif isinstance(lastItem, dict):
+                    # If last dict item has very few keys, might be incomplete
+                    if len(lastItem) < 2:
+                        return True
+            elif isinstance(value, str):
+                # Check if string ends abruptly (no punctuation, very short)
+                if len(value) > 0 and len(value) < 10 and not value[-1] in '.!?\n':
+                    return True
+        
         return False
     
     def _mergeSectionContent(
@@ -559,11 +665,34 @@ Respond with ONLY a JSON object in this exact format:
                 existingElem["text"] = mergedText
         
         elif contentType == "table":
-            # Merge table rows
+            # Merge table rows with overlap detection
             existingRows = existingElem.get("rows", [])
             newRows = newElem.get("rows", [])
             if existingRows and newRows:
+                # CRITICAL: Detect and remove overlaps before merging
+                # Check if last existing row matches first new row (exact overlap)
+                if len(existingRows) > 0 and len(newRows) > 0:
+                    lastExistingRow = existingRows[-1]
+                    firstNewRow = newRows[0]
+                    # Compare rows (handle both list and tuple formats)
+                    if isinstance(lastExistingRow, (list, tuple)) and isinstance(firstNewRow, (list, tuple)):
+                        if list(lastExistingRow) == list(firstNewRow):
+                            # Exact duplicate - remove first new row
+                            newRows = newRows[1:]
+                            logger.debug(f"Iteration {iteration}: Removed duplicate table row (exact match)")
+                
+                # Combine rows from both sections (after removing overlaps)
                 existingElem["rows"] = existingRows + newRows
+                logger.debug(f"Iteration {iteration}: Merged table rows - existing: {len(existingRows)}, new: {len(newRows)}, total: {len(existingRows) + len(newRows)}")
+            elif newRows:
+                # If existing has no rows but new does, use new rows
+                existingElem["rows"] = newRows
+            # Preserve headers from existing (or use new if existing has none)
+            if not existingElem.get("headers") and newElem.get("headers"):
+                existingElem["headers"] = newElem["headers"]
+            # Preserve caption from existing (or use new if existing has none)
+            if not existingElem.get("caption") and newElem.get("caption"):
+                existingElem["caption"] = newElem["caption"]
         
         elif contentType in ["bullet_list", "numbered_list"]:
             # Merge list items
@@ -572,9 +701,66 @@ Respond with ONLY a JSON object in this exact format:
             if existingItems and newItems:
                 existingElem["items"] = existingItems + newItems
         
+        elif contentType == "image":
+            # Images are typically complete - if new image is provided, replace existing
+            # But check if existing image data is incomplete (e.g., base64 string cut off)
+            existingImageData = existingElem.get("base64Data", "")
+            newImageData = newElem.get("base64Data", "")
+            if existingImageData and newImageData:
+                # If existing image data doesn't end with valid base64 padding, it might be incomplete
+                # Base64 padding is '=' or '==' at the end
+                if not existingImageData.rstrip().endswith(('=', '==')):
+                    # Existing image might be incomplete - merge by appending new data
+                    # This handles cases where base64 string was cut off
+                    existingElem["base64Data"] = existingImageData + newImageData
+                    logger.debug(f"Iteration {iteration}: Merged incomplete image base64 data")
+                else:
+                    # Existing image is complete - replace with new (or keep existing if new is empty)
+                    if newImageData:
+                        existingElem["base64Data"] = newImageData
+            elif newImageData:
+                existingElem["base64Data"] = newImageData
+            # Preserve other image metadata
+            if not existingElem.get("altText") and newElem.get("altText"):
+                existingElem["altText"] = newElem["altText"]
+            if not existingElem.get("caption") and newElem.get("caption"):
+                existingElem["caption"] = newElem["caption"]
+        
+        else:
+            # GENERIC FALLBACK: Handle any other content types or unknown structures
+            # Try to merge common array/list fields generically
+            for key in ["items", "rows", "columns", "cells", "elements", "data", "content"]:
+                if key in existingElem and key in newElem:
+                    existingValue = existingElem[key]
+                    newValue = newElem[key]
+                    if isinstance(existingValue, list) and isinstance(newValue, list):
+                        # Merge lists by concatenation
+                        existingElem[key] = existingValue + newValue
+                        logger.debug(f"Iteration {iteration}: Merged generic list field '{key}' - existing: {len(existingValue)}, new: {len(newValue)}")
+                        break
+            
+            # If no common list fields found, try to merge all fields from newElem into existingElem
+            # This handles cases where objects have different structures
+            for key, value in newElem.items():
+                if key not in existingElem:
+                    # New field - add it
+                    existingElem[key] = value
+                elif isinstance(existingElem[key], list) and isinstance(value, list):
+                    # Both are lists - merge them
+                    existingElem[key] = existingElem[key] + value
+                elif isinstance(existingElem[key], dict) and isinstance(value, dict):
+                    # Both are dicts - recursively merge (shallow merge)
+                    existingElem[key].update(value)
+                elif isinstance(existingElem[key], str) and isinstance(value, str):
+                    # Both are strings - append new to existing
+                    existingElem[key] = existingElem[key] + "\n" + value
+        
         # Update section with merged content
         mergedSection = existingSection.copy()
         if isinstance(existingElements, list):
+            # Update the last element in the list with merged content
+            if existingElements:
+                existingElements[-1] = existingElem
             mergedSection["elements"] = existingElements
         else:
             mergedSection["elements"] = existingElem
@@ -653,38 +839,48 @@ Respond with ONLY a JSON object in this exact format:
         """
         Extract sections from AI response, handling both valid and broken JSON.
         Uses repair mechanism for broken JSON.
-        Checks for "complete_response": true flag to determine completion.
+        Determines completion based on JSON structure (complete JSON = complete, broken/incomplete = incomplete).
         Returns (sections, wasJsonComplete, parsedResult)
         """
         # First, try to parse as valid JSON
         try:
             extracted = extractJsonString(result)
-            parsed_result = json.loads(extracted)
             
-            # Check if AI marked response as complete
+            # CRITICAL: Check if raw response suggests incomplete JSON BEFORE parsing
-            isComplete = parsed_result.get("complete_response", False) == True
+            # extractFirstBalancedJson can return partial but valid JSON if raw is incomplete
+            from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
+            raw_normalized = normalizeJsonText(stripCodeFences(result.strip())).strip()
+            extracted_stripped = extracted.strip()
+            
+            # If extracted is shorter than raw, or raw doesn't end properly, it's incomplete
+            is_raw_incomplete = False
+            if len(extracted_stripped) < len(raw_normalized):
+                is_raw_incomplete = True
+                logger.info(f"Iteration {iteration}: Extracted JSON ({len(extracted_stripped)} chars) shorter than raw ({len(raw_normalized)} chars) - raw is incomplete")
+            elif raw_normalized and not raw_normalized.endswith(('}', ']')):
+                is_raw_incomplete = True
+                logger.info(f"Iteration {iteration}: Raw response doesn't end with }} or ] - raw is incomplete")
+            
+            parsed_result = json.loads(extracted)
             
             # Extract sections from parsed JSON
             sections = extractSectionsFromDocument(parsed_result)
             
-            # If AI marked as complete, always return as complete
+            # CRITICAL: If raw response is incomplete, mark as incomplete
-            if isComplete:
+            # JSON structure determines completion, not any flag
-                return sections, True, parsed_result
+            if is_raw_incomplete:
+                logger.info(f"Iteration {iteration}: JSON parseable but raw response incomplete - marking as incomplete")
+                return sections, False, parsed_result
             
-            # If in continuation mode (iteration > 1), continuation responses are expected to be fragments
+            # JSON was parseable and has sections or complete structure
-            # A fragment with 0 extractable sections means JSON is incomplete - need another iteration
+            # Raw response ends properly = complete
-            if len(sections) == 0 and iteration > 1:
+            logger.info(f"Iteration {iteration}: JSON parseable and raw response complete - marking as complete")
-                return sections, False, parsed_result  # Mark as incomplete so loop continues
+            return sections, True, parsed_result
-            
-            # First iteration with 0 sections means empty response - stop
-            if len(sections) == 0:
-                return sections, True, parsed_result  # Complete but empty
-            
-            return sections, True, parsed_result  # JSON was complete with sections
             
         except json.JSONDecodeError as e:
             # Broken JSON - try repair mechanism (normal in iterative generation)
             self.services.utils.writeDebugFile(result, f"{debugPrefix}_broken_json_iteration_{iteration}")
+            logger.info(f"Iteration {iteration}: JSON parsing failed (broken JSON), attempting repair")
             
             # Try to repair
             repaired_json = repairBrokenJson(result)
@@ -692,11 +888,14 @@ Respond with ONLY a JSON object in this exact format:
             if repaired_json:
                 # Extract sections from repaired JSON
                 sections = extractSectionsFromDocument(repaired_json)
-                return sections, False, repaired_json  # JSON was broken but repaired
+                # CRITICAL: JSON was broken, so mark as incomplete (wasJsonComplete = False)
+                # This ensures the loop continues to get the rest of the content
+                logger.info(f"Iteration {iteration}: JSON repaired, extracted {len(sections)} sections, marking as incomplete to continue")
+                return sections, False, repaired_json  # JSON was broken but repaired - mark as incomplete
             else:
-                # Repair failed - log error
+                # Repair failed - but we should still continue to allow AI to retry
-                logger.error(f"Iteration {iteration}: All repair strategies failed")
+                logger.warning(f"Iteration {iteration}: All repair strategies failed, but continuing to allow retry")
-                return [], False, None
+                return [], False, None  # Mark as incomplete so loop continues
         
         except Exception as e:
             logger.error(f"Iteration {iteration}: Unexpected error during parsing: {str(e)}")
@@ -707,36 +906,229 @@ Respond with ONLY a JSON object in this exact format:
         allSections: List[Dict[str, Any]],
         iteration: int,
         wasJsonComplete: bool,
-        rawResponse: str = None
+        rawResponse: str = None,
+        userPrompt: Optional[str] = None,
+        workflowIntent: Optional[Dict[str, Any]] = None
     ) -> bool:
         """
-        Determine if generation should continue based on JSON completeness, complete_response flag, and task completion.
+        Determine if AI generation loop should continue.
-        Returns True if we should continue, False if done.
+        
+        CRITICAL: This is ONLY about AI Loop Completion, NOT Action DoD!
+        Action DoD is checked AFTER the AI Loop completes in _refineDecide.
+        
+        Simple logic:
+        - If JSON is incomplete/broken → continue (needs more content)
+        - If JSON is complete → stop (all content delivered)
+        - Loop detection prevents infinite loops
+        
+        Returns True if we should continue, False if AI Loop is done.
         """
         if len(allSections) == 0:
             return True  # No sections yet, continue
         
-        # Check for complete_response flag in raw response
+        # CRITERION 1: If JSON was incomplete/broken - continue to repair/complete
-        if rawResponse:
+        if not wasJsonComplete:
-            import re
+            logger.info(f"Iteration {iteration}: JSON incomplete/broken - continuing to complete")
-            if re.search(r'"complete_response"\s*:\s*true', rawResponse, re.IGNORECASE):
-                logger.info(f"Iteration {iteration}: AI marked response as complete (complete_response flag detected)")
-                return False
-        
-        # If JSON was complete, stop (AI should have set complete_response if task is done)
-        # For continuation iterations (iteration > 1), if JSON is complete but no flag was set,
-        # stop to prevent infinite loops - AI had a chance to set the flag
-        if wasJsonComplete:
-            if iteration > 1:
-                # Continuation mode: JSON complete without flag means we're likely done
-                # Stop to prevent infinite loops
-                logger.info(f"Iteration {iteration}: JSON complete without complete_response flag - stopping")
-                return False
-            # First iteration with complete JSON - done
-            return False
-        else:
-            # JSON was incomplete/broken - continue
             return True
+        
+        # CRITERION 2: JSON is complete - check for loop detection
+        # If JSON is complete, we're done (all content delivered)
+        # But check for infinite loops first
+        if self._isStuckInLoop(allSections, iteration):
+            logger.warning(f"Iteration {iteration}: Detected potential infinite loop - stopping AI loop")
+            return False
+        
+        # JSON is complete and not stuck in loop - done
+        logger.info(f"Iteration {iteration}: JSON complete - AI loop done")
+        return False
+    
+    def _analyzeTaskCompletion(
+        self,
+        allSections: List[Dict[str, Any]],
+        userPrompt: Optional[str],
+        iteration: int,
+        workflowIntent: Optional[Dict[str, Any]] = None
+    ) -> bool:
+        """
+        GENERIC task completion analysis using KPIs from Intent Analyzer.
+        
+        Uses definitionOfDone KPIs from workflowIntent to check completion.
+        Falls back to simple heuristics if workflowIntent not available.
+        
+        Returns True if task appears complete, False otherwise.
+        """
+        if not allSections:
+            return False
+        
+        # Calculate current metrics from JSON structure
+        totalSections = len(allSections)
+        totalContentSize = 0
+        totalRows = 0
+        totalItems = 0
+        totalParagraphs = 0
+        totalHeadings = 0
+        totalCodeLines = 0
+        contentTypes = set()
+        lastSectionComplete = True
+        
+        for section in allSections:
+            contentType = section.get("content_type", "")
+            contentTypes.add(contentType)
+            elements = section.get("elements", [])
+            
+            if isinstance(elements, list) and elements:
+                lastElem = elements[-1] if elements else {}
+            else:
+                lastElem = elements if isinstance(elements, dict) else {}
+            
+            if isinstance(lastElem, dict):
+                if contentType == "code_block":
+                    code = lastElem.get("code", "")
+                    if code:
+                        lines = [l for l in code.split('\n') if l.strip()]
+                        totalCodeLines += len(lines)
+                        totalContentSize += len(code)
+                        if code and not code.rstrip().endswith('\n'):
+                            lastSectionComplete = False
+                
+                elif contentType == "table":
+                    rows = lastElem.get("rows", [])
+                    if isinstance(rows, list):
+                        totalRows += len(rows)
+                        totalContentSize += len(str(rows))
+                        if not lastElem.get("headers"):
+                            lastSectionComplete = False
+                
+                elif contentType in ["bullet_list", "numbered_list"]:
+                    items = lastElem.get("items", [])
+                    if isinstance(items, list):
+                        totalItems += len(items)
+                        totalContentSize += len(str(items))
+                
+                elif contentType == "heading":
+                    totalHeadings += 1
+                    text = lastElem.get("text", "")
+                    if text:
+                        totalContentSize += len(text)
+                
+                elif contentType == "paragraph":
+                    totalParagraphs += 1
+                    text = lastElem.get("text", "")
+                    if text:
+                        totalContentSize += len(text)
+                        if text and not text.rstrip()[-1] in '.!?':
+                            lastSectionComplete = False
+        
+        # STRATEGY 1: Use KPIs from Intent Analyzer (preferred method)
+        if workflowIntent and isinstance(workflowIntent, dict):
+            definitionOfDone = workflowIntent.get("definitionOfDone", {})
+            if definitionOfDone:
+                # Check all KPI thresholds
+                allKPIsMet = True
+                kpiChecks = []
+                
+                minSections = definitionOfDone.get("minSections", 0)
+                if minSections > 0:
+                    met = totalSections >= minSections
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"sections: {totalSections}/{minSections}")
+                
+                minParagraphs = definitionOfDone.get("minParagraphs", 0)
+                if minParagraphs > 0:
+                    met = totalParagraphs >= minParagraphs
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"paragraphs: {totalParagraphs}/{minParagraphs}")
+                
+                minHeadings = definitionOfDone.get("minHeadings", 0)
+                if minHeadings > 0:
+                    met = totalHeadings >= minHeadings
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"headings: {totalHeadings}/{minHeadings}")
+                
+                minTableRows = definitionOfDone.get("minTableRows", 0)
+                if minTableRows > 0:
+                    met = totalRows >= minTableRows
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"tableRows: {totalRows}/{minTableRows}")
+                
+                minListItems = definitionOfDone.get("minListItems", 0)
+                if minListItems > 0:
+                    met = totalItems >= minListItems
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"listItems: {totalItems}/{minListItems}")
+                
+                minCodeLines = definitionOfDone.get("minCodeLines", 0)
+                if minCodeLines > 0:
+                    met = totalCodeLines >= minCodeLines
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"codeLines: {totalCodeLines}/{minCodeLines}")
+                
+                minContentSize = definitionOfDone.get("minContentSize", 0)
+                if minContentSize > 0:
+                    met = totalContentSize >= minContentSize
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"contentSize: {totalContentSize}/{minContentSize}")
+                
+                # Check required content types
+                requiredContentTypes = definitionOfDone.get("requiredContentTypes", [])
+                if requiredContentTypes:
+                    met = all(ct in contentTypes for ct in requiredContentTypes)
+                    allKPIsMet = allKPIsMet and met
+                    kpiChecks.append(f"contentTypes: {list(contentTypes)}/{requiredContentTypes}")
+                
+                # If all KPIs met and last section is complete, task is done
+                if allKPIsMet and lastSectionComplete:
+                    logger.info(f"Task completion (KPI-based): All KPIs met - {', '.join(kpiChecks)}")
+                    return True
+        
+        # STRATEGY 2: Fallback to simple heuristics if no workflowIntent
+        # Only use if substantial content and last section complete
+        if totalContentSize > 20000 and lastSectionComplete and iteration > 2:
+            logger.info(f"Task completion (fallback heuristic): Large content ({totalContentSize} chars) over {iteration} iterations, last section complete")
+            return True
+        
+        return False
+    
+    def _isStuckInLoop(
+        self,
+        allSections: List[Dict[str, Any]],
+        iteration: int
+    ) -> bool:
+        """
+        Detect if we're stuck in a loop (same content being repeated).
+        
+        Generic approach: Check if recent iterations are adding minimal or duplicate content.
+        """
+        if iteration < 3:
+            return False  # Need at least 3 iterations to detect a loop
+        
+        if len(allSections) == 0:
+            return False
+        
+        # Check if last section is very small (might be stuck)
+        lastSection = allSections[-1]
+        elements = lastSection.get("elements", [])
+        
+        if isinstance(elements, list) and elements:
+            lastElem = elements[-1] if elements else {}
+        else:
+            lastElem = elements if isinstance(elements, dict) else {}
+        
+        # Check content size of last section
+        lastSectionSize = 0
+        if isinstance(lastElem, dict):
+            for key, value in lastElem.items():
+                if isinstance(value, str):
+                    lastSectionSize += len(value)
+                elif isinstance(value, list):
+                    lastSectionSize += len(str(value))
+        
+        # If last section is very small and we've done many iterations, might be stuck
+        if lastSectionSize < 100 and iteration > 10:
+            logger.warning(f"Potential loop detected: iteration {iteration}, last section size {lastSectionSize}")
+            return True
+        
+        return False
     
     def _extractDocumentMetadata(
         self,
@@ -1039,13 +1431,41 @@ Respond with ONLY a JSON object in this exact format:
                 }
                 
                 self.services.chat.progressLogUpdate(aiOperationId, 0.4, "Calling AI for content generation")
+                # Extract user prompt from promptArgs for task completion analysis
+                userPrompt = None
+                if promptArgs:
+                    userPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt")
+                
+                # CRITICAL: Get actionIntent (not taskIntent or workflowIntent) for Definition of Done
+                # Action Intent contains Definition of Done for THIS specific action
+                # Each action needs its own DoD because actions have different completion criteria
+                # Example: Action 1 "Generate 2000 primes" → DoD: 200 rows, Action 2 "Convert to CSV" → DoD: 1 document
+                actionIntent = None
+                if hasattr(self.services, 'workflow') and self.services.workflow:
+                    # Priority 1: Use actionIntent (most specific - for THIS action)
+                    actionIntent = getattr(self.services.workflow, '_actionIntent', None)
+                    if not actionIntent:
+                        # Priority 2: Fallback to taskIntent (for THIS task)
+                        actionIntent = getattr(self.services.workflow, '_taskIntent', None)
+                        if actionIntent:
+                            logger.info("Action intent not found, using task intent as fallback")
+                    if not actionIntent:
+                        # Priority 3: Fallback to workflowIntent (for entire workflow)
+                        actionIntent = getattr(self.services.workflow, '_workflowIntent', None)
+                        logger.warning("Action and task intent not found, using workflow intent as fallback")
+                
+                # Store actionIntent separately (not in promptArgs - buildGenerationPrompt doesn't accept it)
+                # actionIntent is passed to _callAiWithLooping for completion detection, not for prompt building
+                
                 generated_json = await self._callAiWithLooping(
                     generation_prompt, 
                     options, 
                     "document_generation",
                     buildGenerationPrompt,
-                    promptArgs,
+                    promptArgs,  # Does NOT contain taskIntent - buildGenerationPrompt doesn't accept it
-                    aiOperationId
+                    aiOperationId,
+                    userPrompt=userPrompt,
+                    workflowIntent=actionIntent  # Use actionIntent (contains Definition of Done for THIS action)
                 )
                 
                 self.services.chat.progressLogUpdate(aiOperationId, 0.7, "Parsing generated JSON")

modules/services/serviceGeneration/subPromptBuilderGeneration.py

@@ -47,46 +47,107 @@ async def buildGenerationPrompt(
     
     if hasContinuation:
         # CONTINUATION PROMPT - user already received first part, continue from where it stopped
-        lastRawJson = continuationContext.get("last_raw_json", "")
+        lastItemObject = continuationContext.get("last_item_object", "")  # Last complete sub-element (row, item, line, etc.)
-        lastItemObject = continuationContext.get("last_item_object", "")  # Full object like {"text": "value"}
-        lastItemsFromFragment = continuationContext.get("last_items_from_fragment", "")
         totalItemsCount = continuationContext.get("total_items_count", 0)
         
-        # Show the last few items to indicate where to continue (limit fragment size)
+        # CRITICAL: Only use lastItemObject - it contains the last complete sub-element
-        # Extract just the ending portion of the JSON to show where it cut off
+        # If extraction failed and lastItemObject is empty, we'll show a message that extraction failed
-        fragmentSnippet = ""
+        # No need for fragmentSnippet - it's redundant and causes duplication
-        if lastRawJson:
-            # Show last 1500 chars or the whole thing if shorter - just enough to show the cut point
-            fragmentSnippet = lastRawJson[-1500:] if len(lastRawJson) > 1500 else lastRawJson
-            # Add ellipsis if truncated
-            if len(lastRawJson) > 1500:
-                fragmentSnippet = "..." + fragmentSnippet
         
-        # Build clear continuation guidance
+        # Build clear continuation guidance with PROGRESS STATISTICS from all accumulated sections
+        # This helps AI understand completion status without seeing entire content
+        # GENERIC approach: Works for all task types (books, reports, code, lists, tables, etc.)
         continuationGuidance = []
         
-        if totalItemsCount > 0:
+        progressStats = continuationContext.get("progress_stats", {})
-            continuationGuidance.append(f"You have already generated {totalItemsCount} items.")
+        totalRows = progressStats.get("total_rows", 0)
+        totalItems = progressStats.get("total_items", 0)
+        totalCodeLines = progressStats.get("total_code_lines", 0)
+        totalParagraphs = progressStats.get("total_paragraphs", 0)
+        totalHeadings = progressStats.get("total_headings", 0)
+        sectionCount = progressStats.get("section_count", 0)
+        contentTypeCount = progressStats.get("content_type_count", 0)
+        lastContentType = progressStats.get("last_content_type")
+        
+        # CRITICAL: Filter progress stats based on Definition of Done from taskIntent
+        # Only show KPIs that are relevant for this specific action/task
+        taskIntent = continuationContext.get("taskIntent", {})
+        definitionOfDone = taskIntent.get("definitionOfDone", {}) if isinstance(taskIntent, dict) else {}
+        
+        # Build comprehensive progress information (filtered by DoD if available)
+        progressParts = []
+        
+        # Only show progress metrics that are relevant based on DoD KPIs
+        # If DoD specifies minTableRows, show rows; if minListItems, show items; etc.
+        if definitionOfDone:
+            # Filter based on DoD KPIs - only show metrics that matter for this task
+            if definitionOfDone.get("minTableRows", 0) > 0 and totalRows > 0:
+                progressParts.append(f"{totalRows} row{'s' if totalRows > 1 else ''}")
+            if definitionOfDone.get("minListItems", 0) > 0 and totalItems > 0:
+                progressParts.append(f"{totalItems} item{'s' if totalItems > 1 else ''}")
+            if definitionOfDone.get("minCodeLines", 0) > 0 and totalCodeLines > 0:
+                progressParts.append(f"{totalCodeLines} line{'s' if totalCodeLines > 1 else ''} of code/data")
+            if definitionOfDone.get("minParagraphs", 0) > 0 and totalParagraphs > 0:
+                progressParts.append(f"{totalParagraphs} paragraph{'s' if totalParagraphs > 1 else ''}")
+            if definitionOfDone.get("minHeadings", 0) > 0 and totalHeadings > 0:
+                progressParts.append(f"{totalHeadings} heading{'s' if totalHeadings > 1 else ''}")
+            if definitionOfDone.get("minSections", 0) > 0 and sectionCount > 0:
+                progressParts.append(f"{sectionCount} section{'s' if sectionCount > 1 else ''}")
+            # Only show contentSize if no other metrics are available (it's less informative)
+            # Prefer showing rows/items/lines over characters
+            if not progressParts and definitionOfDone.get("minContentSize", 0) > 0:
+                totalContentSize = progressStats.get("total_content_size", 0)
+                if totalContentSize > 0:
+                    progressParts.append(f"{totalContentSize} characters")
+        else:
+            # No DoD available - show all progress metrics (fallback)
+            if sectionCount > 0:
+                progressParts.append(f"{sectionCount} section{'s' if sectionCount > 1 else ''}")
+            if totalHeadings > 0:
+                progressParts.append(f"{totalHeadings} heading{'s' if totalHeadings > 1 else ''}")
+            if totalParagraphs > 0:
+                progressParts.append(f"{totalParagraphs} paragraph{'s' if totalParagraphs > 1 else ''}")
+            if totalRows > 0:
+                progressParts.append(f"{totalRows} row{'s' if totalRows > 1 else ''}")
+            if totalItems > 0:
+                progressParts.append(f"{totalItems} item{'s' if totalItems > 1 else ''}")
+            if totalCodeLines > 0:
+                progressParts.append(f"{totalCodeLines} line{'s' if totalCodeLines > 1 else ''} of code/data")
+            if contentTypeCount > 1:
+                progressParts.append(f"{contentTypeCount} different content types")
+        
+        if progressParts:
+            continuationGuidance.append(f"PROGRESS: You have already generated: {', '.join(progressParts)}.")
+        elif totalItemsCount > 0:
+            # Fallback to old totalItemsCount if progress_stats not available
+            continuationGuidance.append(f"PROGRESS: You have already generated {totalItemsCount} items.")
+        
+        # Show the last complete item AND cut item for continuation point
+        # CRITICAL: AI needs both to know where to continue
+        cutItemObject = continuationContext.get("cut_item_object")
+        contentTypeForItems = continuationContext.get("content_type_for_items")
         
-        # Show the last complete item object (full object format)
         if lastItemObject:
-            continuationGuidance.append(f"Last item in previous response: {lastItemObject}. Continue with the NEXT item after this.")
+            if cutItemObject:
+                # Both complete and cut items available - show both
+                continuationGuidance.append(f"Last complete {contentTypeForItems or 'item'} in previous response: {lastItemObject}")
+                continuationGuidance.append(f"Incomplete/cut {contentTypeForItems or 'item'} at the end: {cutItemObject}")
+                continuationGuidance.append(f"Continue from the incomplete item above - complete it first, then add NEW items.")
+            else:
+                # Only complete item available
+                continuationGuidance.append(f"Last complete {contentTypeForItems or 'item'} in previous response: {lastItemObject}")
+                continuationGuidance.append(f"Continue with the NEXT item after this.")
         
         continuationText = "\n".join(continuationGuidance) if continuationGuidance else "Continue from where it stopped."
         
         # PROMPT FOR CONTINUATION
-
         generationPrompt = f"""User request: "{userPrompt}"
 
-The user already received part of the response. Continue generating the remaining content.
+NOTE: The user already received part of the response.
+TASK: Continue generating the remaining content.
 
 {continuationText}
 
-Previous response ended here (JSON was cut off at this point):
-```json
-{fragmentSnippet if fragmentSnippet else "(No fragment available)"}
-```
-
 JSON structure template:
 {jsonTemplate}
 
@@ -94,11 +155,10 @@ Instructions:
 - Return ONLY valid JSON (strict). No comments of any kind (no //, /* */, or #). No trailing commas. Strings must use double quotes.
 - Arrays must contain ONLY JSON values; do not include comments or ellipses.
 - Use ONLY the element structures shown in the template.
-- Continue from where it stopped — add NEW items only; do not repeat existing items.
+- Continue from where it stopped - add NEW items only; do not repeat existing items.
 - Generate remaining content to complete the user request. Do NOT just give an instruction or comments. Deliver the complete response.
 - Fill with actual content (no placeholders or instructional text such as "Add more...").
 - IMPORTANT: Ensure "filename" in each document has meaningful name with appropriate extension matching the content.
-- When the request is fully satisfied, add "complete_response": true at root level.
 - Output JSON only; no markdown fences or extra text.
 
 IMPORTANT: Before responding, analyse the remaining data to fully satisfy user request.
@@ -117,12 +177,11 @@ JSON structure template:
 {jsonTemplate}
 
 Instructions:
-- Start with {{"metadata": ...}} — return COMPLETE, STRICT JSON.
+- Start with {{"metadata": ...}} - return COMPLETE, STRICT JSON.
 - Return ONLY valid JSON (strict). No comments. No trailing commas. Use double quotes.
 - Do NOT reuse example section IDs; create your own.
 - Generate complete content based on the user request. Do NOT just give an instruction or comments. Deliver the complete response.
 - IMPORTANT: Set a meaningful "filename" in each document with appropriate file extension (e.g., "prime_numbers.txt", "report.docx", "data.json"). The filename should reflect the content and task objective.
-- When the request is fully satisfied, add "complete_response": true at root level.
 - Output JSON only; no markdown fences or extra text.
 
 Generate your complete response starting from {{"metadata": ...}}:

modules/shared/jsonUtils.py

@@ -764,128 +764,259 @@ def buildContinuationContext(allSections: List[Dict[str, Any]], lastRawResponse:
     Build context information from accumulated sections for continuation prompt.
     Extracts last items and provides clear continuation point.
     
+    CRITICAL: Analyzes ALL accumulated sections (not just last response) to provide
+    accurate progress information to AI. This allows AI to understand completion status
+    without seeing the entire content (which would exceed token limits).
+    
     Args:
-        allSections: List of sections already generated
+        allSections: List of ALL sections accumulated across ALL iterations
         lastRawResponse: Raw JSON response from last iteration (can be broken/incomplete)
         
     Returns:
-        Dict with section_count, last_raw_json, last_items, and continuation point
+        Dict with section_count, last_raw_json, last_items, continuation point, and
+        PROGRESS STATISTICS from all accumulated sections
     """
     context = {
         "section_count": len(allSections),
     }
     
-    # Extract last COMPLETE object directly from raw response (generic - works for any structure)
+    # CRITICAL: Analyze ALL accumulated sections to get accurate progress statistics
-    # This is extracted BEFORE any merging/accumulation happens
+    # This allows AI to understand completion status without seeing entire content
-    # Returns the full last complete object like {"text": "..."} or {"code": "...", "language": "..."} etc.
+    # GENERIC approach: Works for all task types (books, reports, code, lists, etc.)
-    # Logic: find the last complete {...} where there are no nested { inside (flat object)
+    totalRows = 0
-    last_complete_object = ""  # Full object as JSON string
+    totalItems = 0
+    totalCodeLines = 0
+    totalParagraphs = 0
+    totalHeadings = 0
+    totalContentSize = 0
+    contentTypes = set()
+    lastContentType = None
+    
+    for section in allSections:
+        contentType = section.get("content_type", "")
+        contentTypes.add(contentType)
+        elements = section.get("elements", [])
+        
+        # CRITICAL: Iterate through ALL elements, not just the last one
+        # This ensures we count all rows/items/lines from all elements in the section
+        if isinstance(elements, list):
+            # Multiple elements - iterate through all
+            for elem in elements:
+                if isinstance(elem, dict):
+                    if contentType == "code_block":
+                        code = elem.get("code", "")
+                        if code:
+                            lines = [l for l in code.split('\n') if l.strip()]
+                            totalCodeLines += len(lines)
+                            totalContentSize += len(code)
+                            lastContentType = "code_block"
+                    
+                    elif contentType == "table":
+                        rows = elem.get("rows", [])
+                        if isinstance(rows, list):
+                            totalRows += len(rows)  # Count ALL rows from ALL table elements
+                            totalContentSize += len(str(rows))
+                            lastContentType = "table"
+                    
+                    elif contentType in ["bullet_list", "numbered_list"]:
+                        items = elem.get("items", [])
+                        if isinstance(items, list):
+                            totalItems += len(items)  # Count ALL items from ALL list elements
+                            totalContentSize += len(str(items))
+                            lastContentType = "list"
+                    
+                    elif contentType == "heading":
+                        text = elem.get("text", "")
+                        if text:
+                            totalHeadings += 1
+                            totalContentSize += len(text)
+                            lastContentType = "heading"
+                    
+                    elif contentType == "paragraph":
+                        text = elem.get("text", "")
+                        if text:
+                            totalParagraphs += 1
+                            totalContentSize += len(text)
+                            lastContentType = "paragraph"
+        elif isinstance(elements, dict):
+            # Single element as dict
+            elem = elements
+            if contentType == "code_block":
+                code = elem.get("code", "")
+                if code:
+                    lines = [l for l in code.split('\n') if l.strip()]
+                    totalCodeLines += len(lines)
+                    totalContentSize += len(code)
+                    lastContentType = "code_block"
+            
+            elif contentType == "table":
+                rows = elem.get("rows", [])
+                if isinstance(rows, list):
+                    totalRows += len(rows)
+                    totalContentSize += len(str(rows))
+                    lastContentType = "table"
+            
+            elif contentType in ["bullet_list", "numbered_list"]:
+                items = elem.get("items", [])
+                if isinstance(items, list):
+                    totalItems += len(items)
+                    totalContentSize += len(str(items))
+                    lastContentType = "list"
+            
+            elif contentType == "heading":
+                text = elem.get("text", "")
+                if text:
+                    totalHeadings += 1
+                    totalContentSize += len(text)
+                    lastContentType = "heading"
+            
+            elif contentType == "paragraph":
+                text = elem.get("text", "")
+                if text:
+                    totalParagraphs += 1
+                    totalContentSize += len(text)
+                    lastContentType = "paragraph"
+    
+    # Store progress statistics (not full content - that would exceed token limits)
+    # These statistics help AI understand progress for ALL task types
+    context["progress_stats"] = {
+        "total_rows": totalRows,
+        "total_items": totalItems,
+        "total_code_lines": totalCodeLines,
+        "total_paragraphs": totalParagraphs,
+        "total_headings": totalHeadings,
+        "total_content_size": totalContentSize,
+        "section_count": len(allSections),
+        "content_type_count": len(contentTypes),
+        "content_types": list(contentTypes),
+        "last_content_type": lastContentType
+    }
+    
+    # Extract last complete sub-item from allSections (already merged, contains all delivered data)
+    # Extract cut/incomplete sub-item from raw JSON (what was cut off)
+    last_complete_subobject = None
+    cut_subobject = None
+    content_type_for_items = None
     total_items_count = 0
     
+    # STEP 1: Extract last complete sub-item from allSections (this is what was already delivered)
+    if allSections:
+        sorted_sections = sorted(allSections, key=lambda s: s.get("order", 0))
+        last_section = sorted_sections[-1]
+        content_type_for_items = last_section.get("content_type", "")
+        elements = last_section.get("elements", [])
+        
+        if elements and isinstance(elements, list) and len(elements) > 0:
+            last_element = elements[-1]
+            if isinstance(last_element, dict):
+                # TABLE: Extract last complete row
+                if content_type_for_items == "table" and "rows" in last_element:
+                    rows = last_element.get("rows", [])
+                    if rows and isinstance(rows, list) and len(rows) > 0:
+                        total_items_count = len(rows)
+                        last_complete_subobject = rows[-1]
+                
+                # LIST: Extract last complete item
+                elif content_type_for_items in ["bullet_list", "numbered_list"] and "items" in last_element:
+                    items = last_element.get("items", [])
+                    if items and isinstance(items, list) and len(items) > 0:
+                        total_items_count = len(items)
+                        last_complete_subobject = items[-1]
+                
+                # CODE_BLOCK: Extract last complete line
+                elif content_type_for_items == "code_block" and "code" in last_element:
+                    code = last_element.get("code", "")
+                    if code:
+                        lines = [l for l in code.split('\n') if l.strip()]
+                        total_items_count = len(lines)
+                        if lines:
+                            last_complete_subobject = lines[-1]
+                
+                # PARAGRAPH/HEADING: Extract last complete sentence
+                elif content_type_for_items in ["paragraph", "heading"] and "text" in last_element:
+                    text = last_element.get("text", "")
+                    if text:
+                        import re
+                        sentences = re.split(r'([.!?]+)', text)
+                        complete_sentences = []
+                        for i in range(0, len(sentences) - 1, 2):
+                            if i + 1 < len(sentences):
+                                complete_sentences.append(sentences[i] + sentences[i + 1])
+                        total_items_count = len(complete_sentences)
+                        if complete_sentences:
+                            last_complete_subobject = complete_sentences[-1]
+    
+    # STEP 2: Extract cut/incomplete sub-item from raw JSON (what was cut off)
     if lastRawResponse:
         raw_json = stripCodeFences(lastRawResponse.strip())
         if raw_json and raw_json.strip() != "{}":
-            # Find last complete flat object (no nested objects inside)
-            # Scan from the end backwards to find the last complete {...} object
-            # A flat object is complete if: starts with {, ends with }, and has no nested { inside
-            
-            # Work backwards from the end, find last }
-            for i in range(len(raw_json) - 1, -1, -1):
-                if raw_json[i] == '}':
-                    # Found a closing brace, work backwards to find its opening brace
-                    depth = 1
-                    opening_pos = -1
-                    
-                    for j in range(i - 1, -1, -1):
-                        if raw_json[j] == '}':
-                            depth += 1
-                        elif raw_json[j] == '{':
-                            depth -= 1
-                            if depth == 0:
-                                # Found matching opening brace
-                                opening_pos = j
-                                # Check if this is a flat object (no nested { inside)
-                                obj_content = raw_json[j + 1:i]
-                                if '{' not in obj_content:
-                                    # This is a flat object (no nested objects inside)
-                                    last_complete_object = raw_json[j:i + 1]
-                                    break
-                    
-                    if last_complete_object:
-                        break
-            
-            # Also try structure-based parsing for item count
             try:
-                parsed = repairBrokenJson(raw_json)
+                import re
-                if parsed:
+                if content_type_for_items == "code_block":
-                    sections = extractSectionsFromDocument(parsed)
+                    # Find incomplete code line at the end
-                    if sections:
+                    # Look for code string that doesn't end with closing quote
-                        sorted_sections = sorted(sections, key=lambda s: s.get("order", 0))
+                    code_match = re.search(r'"code"\s*:\s*"([^"]*?)(?:"|$)', raw_json)
-                        last_section = sorted_sections[-1]
+                    if code_match:
-                        elements = last_section.get("elements", [])
+                        code_content = code_match.group(1)
-                        
-                        if elements and isinstance(elements, list) and len(elements) > 0:
-                            if last_section.get("content_type") == "list":
-                                last_element = elements[-1]
-                                if isinstance(last_element, dict):
-                                    if "items" in last_element and isinstance(last_element["items"], list):
-                                        items_list = last_element["items"]
-                                        # Only count complete items (those successfully extracted)
-                                        total_items_count = len(items_list)
-            except Exception as e:
-                logger.debug(f"Could not extract item count from raw response structure: {e}")
-            
-            # Also extract last items for display (fragment extraction)
-            last_items_from_fragment = _extractLastItemsFromFragment(raw_json, max_items=10)
-            
-            context["last_raw_json"] = raw_json
-            context["last_item_object"] = last_complete_object  # Full last complete object (generic - any structure)
-            context["last_items_from_fragment"] = last_items_from_fragment
-            context["total_items_count"] = total_items_count  # Count from raw response
-            
-            logger.debug(f"Included previous JSON response in continuation context ({len(raw_json)} chars, {total_items_count} items in response, last complete object: {last_complete_object})")
-        else:
-            logger.warning("lastRawResponse was empty or just '{}' - continuation may not work correctly")
-    else:
-        # No raw response - fallback to extracting from accumulated sections
-        # Extract the last complete object from the last element
-        last_item_object_from_sections = ""
-        if allSections:
-            sorted_sections = sorted(allSections, key=lambda s: s.get("order", 0))
-            last_section = sorted_sections[-1]
-            elements = last_section.get("elements", [])
-            
-            if elements and isinstance(elements, list) and len(elements) > 0:
-                # Get the last element (could be any structure - generic)
-                last_element = elements[-1]
-                if isinstance(last_element, dict):
-                    # Try to get items if it's a list structure
-                    if "items" in last_element and isinstance(last_element["items"], list):
-                        items_list = last_element["items"]
-                        total_items_count = len(items_list)
-                        if items_list:
-                            # Get last item (any structure)
-                            last_item = items_list[-1]
-                            if isinstance(last_item, dict):
-                                # Convert to JSON string (generic - works for any object structure)
-                                import json
-                                try:
-                                    last_item_object_from_sections = json.dumps(last_item)
-                                except:
-                                    pass
-                    else:
-                        # Element itself is the object (no items array)
-                        total_items_count = len(elements)
-                        # Convert to JSON string (generic)
-                        import json
                         try:
-                            last_item_object_from_sections = json.dumps(last_element)
+                            code_content = json.loads('"' + code_content + '"')
                         except:
                             pass
-        
+                        lines = code_content.split('\n')
-        context["last_item_object"] = last_item_object_from_sections
+                        if lines and not raw_json.rstrip().endswith('"'):
-        context["total_items_count"] = total_items_count
+                            # Code string is incomplete - last line is cut
-        logger.debug(f"No previous raw response available for continuation context (but have {total_items_count} items accumulated, last item object: {last_item_object_from_sections})")
+                            cut_subobject = lines[-1] if lines else None
+                elif content_type_for_items == "table":
+                    # Find incomplete row at the end
+                    row_pattern = r'\["([^"]*)"(?:,\s*"([^"]*)")*'
+                    matches = list(re.finditer(row_pattern, raw_json))
+                    if matches:
+                        last_match = matches[-1]
+                        end_pos = last_match.end()
+                        if end_pos < len(raw_json):
+                            remaining = raw_json[end_pos:end_pos+20].strip()
+                            if not remaining.startswith(']'):
+                                # Row is incomplete - extract values
+                                cut_values = re.findall(r'"([^"]*)"', raw_json[last_match.start():last_match.end()])
+                                if cut_values:
+                                    cut_subobject = cut_values
+                elif content_type_for_items in ["bullet_list", "numbered_list"]:
+                    # Find incomplete item at the end
+                    item_pattern = r'"([^"]*)"'
+                    matches = list(re.finditer(item_pattern, raw_json))
+                    if matches:
+                        last_match = matches[-1]
+                        end_pos = last_match.end()
+                        if end_pos < len(raw_json):
+                            remaining = raw_json[end_pos:end_pos+10].strip()
+                            if remaining and remaining[0] not in [',', ']', '}', '"']:
+                                cut_subobject = last_match.group(1)
+            except Exception as e:
+                logger.debug(f"Could not extract cut sub-object from raw JSON: {e}")
+            
+            context["last_raw_json"] = raw_json
+        else:
+            context["last_raw_json"] = ""
+    else:
+        context["last_raw_json"] = ""
+    
+    # Convert to JSON strings
+    if last_complete_subobject is not None:
+        try:
+            last_complete_subobject = json.dumps(last_complete_subobject)
+        except:
+            last_complete_subobject = str(last_complete_subobject)
+    
+    if cut_subobject is not None:
+        try:
+            cut_subobject = json.dumps(cut_subobject)
+        except:
+            cut_subobject = str(cut_subobject)
+    
+    context["last_item_object"] = last_complete_subobject if last_complete_subobject else ""
+    context["cut_item_object"] = cut_subobject if cut_subobject else None
+    context["content_type_for_items"] = content_type_for_items
+    context["total_items_count"] = total_items_count
     
     return context
 

modules/workflows/processing/adaptive/contentValidator.py

@@ -338,9 +338,9 @@ OUTPUT FORMAT - JSON ONLY (no prose):
 }}
 
 Field explanations:
-- "improvementSuggestions": Overall actions to improve the entire result (general, high-level)
+- "improvementSuggestions": CONCRETE, EXECUTABLE actions to fix the issues. DO NOT just repeat the original task - suggest SPECIFIC, actionable steps that address the identified problems. Each suggestion should be a concrete action that can be executed, not a vague instruction to repeat the task.
-- "validationDetails[].suggestions": Specific fixes for each document's individual issues (document-specific, detailed)
+- "validationDetails[].suggestions": Specific fixes for each document's individual issues (document-specific, detailed, actionable)
-- Do NOT use prefixes like "NEXT STEP:" - describe actions directly
+- IMPORTANT: Improvement suggestions must be ACTIONABLE and SPECIFIC. Instead of saying "generate CSV again", suggest concrete steps like "convert existing JSON output to CSV format" or "regenerate with CSV format parameter". Focus on what needs to be done differently, not repeating the original request.
 
 DELIVERED DOCUMENTS ({len(documents)} items):
 """

modules/workflows/processing/adaptive/intentAnalyzer.py

@@ -56,6 +56,7 @@ Analyze the user's intent and determine:
 3. What quality requirements they have (accuracy, completeness)
 4. What specific success criteria define completion
 5. What language the user is communicating in (detect from the user request)
+6. DEFINITION OF DONE: Define measurable KPIs that can be checked against JSON structure metrics
 
 CRITICAL: Respond with ONLY the JSON object below. Do not include any explanatory text, analysis, or other content before or after the JSON.
 
@@ -69,8 +70,29 @@ CRITICAL: Respond with ONLY the JSON object below. Do not include any explanator
     }},
     "successCriteria": ["specific criterion 1", "specific criterion 2"],
     "languageUserDetected": "en",
-    "confidenceScore": 0.0-1.0
+    "confidenceScore": 0.0-1.0,
+    "definitionOfDone": {{
+        "minSections": 0,
+        "minParagraphs": 0,
+        "minHeadings": 0,
+        "minTableRows": 0,
+        "minListItems": 0,
+        "minCodeLines": 0,
+        "minContentSize": 0,
+        "requiredContentTypes": [],
+        "completionType": "quantitative|qualitative|structural"
+    }}
 }}
+
+DEFINITION OF DONE RULES:
+- Extract quantitative requirements from user prompt (e.g., "4000 prime numbers" -> minTableRows: 4000)
+- For qualitative tasks (books, reports), set structural requirements (minSections, minParagraphs, minHeadings)
+- For code tasks, set minCodeLines based on requirements
+- For lists, set minListItems based on requirements
+- Set minContentSize as minimum expected content size in characters
+- Set requiredContentTypes if specific content types are required (e.g., ["table"] for CSV, ["paragraph", "heading"] for books)
+- Set completionType: "quantitative" for tasks with specific counts, "qualitative" for content quality tasks, "structural" for structured documents
+- Use 0 for metrics that are not relevant for this task type
 """
             
             # Call AI service for analysis

modules/workflows/processing/modes/modeDynamic.py

@@ -66,12 +66,14 @@ class DynamicMode(BaseMode):
             self.workflowIntent = await self.intentAnalyzer.analyzeUserIntent(original_prompt, context)
             logger.warning(f"Workflow intent not found in workflow object, analyzed fresh")
         
-        # Task-level intent is NOT needed - use task.objective + task format fields (dataType, expectedFormats, qualityRequirements)
+        # CRITICAL: Task-level intent analysis - each task needs its own Definition of Done
-        # These format fields are populated from workflow intent during task planning
+        # Workflow intent is for overall planning, but each task has specific completion criteria
-        self.taskIntent = None  # Removed redundant task-level intent analysis
+        # This Definition of Done is needed for AI looping completion detection
-        logger.info(f"Workflow intent: {self.workflowIntent}")
+        self.taskIntent = await self.intentAnalyzer.analyzeUserIntent(taskStep.objective, context)
-        if taskStep.dataType or taskStep.expectedFormats or taskStep.qualityRequirements:
+        # Store taskIntent in workflow object so it's accessible from services
-            logger.info(f"Task format info: dataType={taskStep.dataType}, expectedFormats={taskStep.expectedFormats}")
+        workflow._taskIntent = self.taskIntent
+        logger.info(f"Task intent: {self.taskIntent}")
+        logger.info(f"Task format info: dataType={taskStep.dataType}, expectedFormats={taskStep.expectedFormats}")
         
         # NEW: Reset progress tracking for new task
         self.progressTracker.reset()
@@ -150,6 +152,16 @@ class DynamicMode(BaseMode):
                 if decision:  # Only append if decision is not None
                     context.previousReviewResult.append(decision)
                 
+                # Store next action guidance from decision for use in next iteration
+                if decision and decision.status == "continue" and decision.nextAction:
+                    # Use setattr for Pydantic models (TaskContext is a BaseModel)
+                    setattr(context, 'nextActionGuidance', {
+                        "action": decision.nextAction,
+                        "parameters": decision.nextActionParameters or {},
+                        "objective": decision.nextActionObjective or decision.reason or ""
+                    })
+                    logger.info(f"Stored next action guidance: {decision.nextAction} with parameters {decision.nextActionParameters}")
+                
                 # Update context with learnings from this step
                 if decision and decision.reason:
                     if not hasattr(context, 'improvements'):
@@ -205,6 +217,28 @@ class DynamicMode(BaseMode):
     
     async def _planSelect(self, context: TaskContext) -> Dict[str, Any]:
         """Plan: select exactly one action. Returns {"action": {method, name}}"""
+        # Check if we have concrete next action guidance from previous refinement decision
+        # Check for nextActionGuidance (stored as dynamic attribute via setattr)
+        nextActionGuidance = getattr(context, 'nextActionGuidance', None)
+        if nextActionGuidance:
+            guidance = nextActionGuidance
+            actionName = guidance.get("action")
+            parameters = guidance.get("parameters", {})
+            objective = guidance.get("objective", "")
+            
+            if actionName:
+                logger.info(f"Using guided next action: {actionName} (from refinement decision)")
+                # Create selection dict from guidance
+                selection = {
+                    "action": actionName,
+                    "actionObjective": objective,
+                    "parameters": parameters
+                }
+                # Clear guidance after use (one-time use)
+                setattr(context, 'nextActionGuidance', None)
+                return selection
+        
+        # Normal planning: use AI to select action
         bundle = generateDynamicPlanSelectionPrompt(self.services, context, self.adaptiveLearningEngine)
         promptTemplate = bundle.prompt
         placeholders = bundle.placeholders
@@ -315,6 +349,29 @@ class DynamicMode(BaseMode):
                          workflow: ChatWorkflow, stepIndex: int) -> ActionResult:
         """Act: request minimal parameters then execute selected action"""
         compoundActionName = selection.get('action', '')
+        actionObjective = selection.get('actionObjective', '')
+        
+        # CRITICAL: Create Action-level Intent with Definition of Done for THIS specific action
+        # Each action needs its own DoD because:
+        # - Action 1: "Generate first 2000 prime numbers" → DoD: 200 table rows
+        # - Action 2: "Generate remaining 2000 prime numbers" → DoD: 200 table rows  
+        # - Action 3: "Convert to CSV" → DoD: 1 document, CSV format
+        # Without action-specific DoD, AI loops never know when THIS action is complete
+        actionIntent = None
+        if actionObjective:
+            try:
+                actionIntent = await self.intentAnalyzer.analyzeUserIntent(actionObjective, context)
+                # Store actionIntent in workflow object so it's accessible from services
+                workflow._actionIntent = actionIntent
+                logger.info(f"Action intent created: {actionIntent.get('definitionOfDone', {}) if actionIntent else 'None'}")
+            except Exception as e:
+                logger.warning(f"Failed to create action intent: {e}, falling back to task intent")
+                # Fallback to task intent if action intent creation fails
+                actionIntent = getattr(workflow, '_taskIntent', None)
+        else:
+            # No actionObjective - fallback to task intent
+            actionIntent = getattr(workflow, '_taskIntent', None)
+            logger.warning("No actionObjective provided, using task intent as fallback")
         
         # Parse compound action name (e.g., "ai.webResearch" -> method="ai", action="webResearch")
         if '.' not in compoundActionName:

modules/workflows/processing/shared/promptGenerationActionsDynamic.py

@@ -293,16 +293,34 @@ def generateDynamicRefinementPrompt(services, context: Any, reviewContent: str)
 OBJECTIVE: '{{KEY:USER_PROMPT}}'
 
 DECISION RULES:
-1. "continue" = objective NOT fulfilled
+1. "continue" = objective NOT fulfilled - MUST specify concrete next action
 2. "success" = objective fulfilled
 3. Return ONLY JSON - no other text
 
 OUTPUT FORMAT (only JSON object to deliver):
 {{
   "status": "continue",
-  "reason": "Brief reason for decision"
+  "reason": "Brief reason for decision",
+  "nextAction": "ai.convert",
+  "nextActionParameters": {{
+    "fromFormat": "json",
+    "toFormat": "csv",
+    "targetDocument": "document.json"
+  }},
+  "nextActionObjective": "Convert the generated JSON document to CSV format with 10 columns per row"
 }}
 
+IMPORTANT RULES FOR NEXT ACTION:
+- If status is "continue", you MUST provide "nextAction" and "nextActionParameters"
+- "nextAction" must be a SPECIFIC, EXECUTABLE action (e.g., "ai.convert", "ai.process", "ai.reformat", "ai.generate")
+- "nextActionParameters" must contain concrete parameters for that action
+- "nextActionObjective" must describe what this specific action will achieve
+- DO NOT suggest repeating the same action that already failed - suggest a DIFFERENT approach
+- Use improvement suggestions from content validation to determine the next action
+- If format conversion is needed, use "ai.convert" action
+- If regeneration is needed with different parameters, use "ai.process" with specific format parameters
+- If reformatting is needed, use "ai.reformat" action
+
 OBSERVATION: {{KEY:REVIEW_CONTENT}}
 
 """

tests/functional/test05_workflow_with_documents.py

@@ -65,7 +65,7 @@ class WorkflowWithDocumentsTester:
         """Create a second text document with instructions."""
         docContent = """Anweisungen zur Primzahlgenerierung:
 
-1. Generiere die ersten 5000 Primzahlen
+1. Generiere Primzahlen
 2. Formatiere sie in einer Tabelle mit 10 Spalten pro Zeile
 3. Verwende das bereitgestellte CSV-Vorlagenformat
 4. Stelle sicher, dass alle Zahlen korrekt formatiert sind
@@ -158,10 +158,18 @@ class WorkflowWithDocumentsTester:
         print(f"   Mode: {self.workflow.workflowMode}")
         print(f"   Current Round: {self.workflow.currentRound}")
     
-    async def waitForWorkflowCompletion(self, maxWaitTime: int = 300) -> bool:
+    async def waitForWorkflowCompletion(self, maxWaitTime: Optional[int] = None) -> bool:
-        """Wait for workflow to complete, checking status periodically."""
+        """Wait for workflow to complete, checking status periodically.
+        
+        Args:
+            maxWaitTime: Maximum wait time in seconds. If None, wait indefinitely.
+        """
         print("\n" + "="*60)
         print("WAITING FOR WORKFLOW COMPLETION")
+        if maxWaitTime:
+            print(f"Maximum wait time: {maxWaitTime} seconds")
+        else:
+            print("Waiting indefinitely (no timeout)")
         print("="*60)
         
         if not self.workflow:
@@ -172,7 +180,15 @@ class WorkflowWithDocumentsTester:
         checkInterval = 2  # Check every 2 seconds
         lastStatus = None
         
-        while time.time() - startTime < maxWaitTime:
+        while True:
+            # Check timeout if maxWaitTime is set
+            if maxWaitTime is not None:
+                elapsed = time.time() - startTime
+                if elapsed >= maxWaitTime:
+                    print(f"\n⚠️  Workflow did not complete within {maxWaitTime} seconds")
+                    print(f"   Final status: {self.workflow.status}")
+                    return False
+            
             # Get current workflow status
             interfaceDbChat = interfaceDbChatObjects.getInterface(self.testUser)
             currentWorkflow = interfaceDbChat.getWorkflow(self.workflow.id)
@@ -182,24 +198,21 @@ class WorkflowWithDocumentsTester:
                 return False
             
             currentStatus = currentWorkflow.status
+            elapsed = int(time.time() - startTime)
             
             # Print status if it changed
             if currentStatus != lastStatus:
-                print(f"Workflow status: {currentStatus} (elapsed: {int(time.time() - startTime)}s)")
+                print(f"Workflow status: {currentStatus} (elapsed: {elapsed}s)")
                 lastStatus = currentStatus
             
             # Check if workflow is complete
             if currentStatus in ["completed", "stopped", "failed"]:
                 self.workflow = currentWorkflow
-                print(f"\n✅ Workflow finished with status: {currentStatus}")
+                print(f"\n✅ Workflow finished with status: {currentStatus} (elapsed: {elapsed}s)")
                 return currentStatus == "completed"
             
             # Wait before next check
             await asyncio.sleep(checkInterval)
-        
-        print(f"\n⚠️  Workflow did not complete within {maxWaitTime} seconds")
-        print(f"   Final status: {self.workflow.status}")
-        return False
     
     def analyzeWorkflowResults(self) -> Dict[str, Any]:
         """Analyze workflow results and extract information."""
@@ -298,11 +311,11 @@ class WorkflowWithDocumentsTester:
             fileIds = await self.uploadFiles()
             
             # Start workflow with prompt and files
-            prompt = "Generiere die ersten 5000 Primzahlen in einer Tabelle mit 10 Spalten pro Zeile."
+            prompt = "Generiere die ersten 4000 Primzahlen in einer Tabelle mit 10 Spalten pro Zeile."
             await self.startWorkflow(prompt, fileIds)
             
-            # Wait for completion
+            # Wait for completion (no timeout - wait indefinitely)
-            completed = await self.waitForWorkflowCompletion(maxWaitTime=300)
+            completed = await self.waitForWorkflowCompletion()
             
             # Analyze results
             results = self.analyzeWorkflowResults()