import json import logging import re import time from typing import Dict, Any, List, Optional, Tuple from modules.datamodels.datamodelChat import PromptPlaceholder from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.datamodels.datamodelExtraction import ContentPart from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData from modules.interfaces.interfaceAiObjects import AiObjects from modules.shared.jsonUtils import ( extractJsonString, repairBrokenJson, extractSectionsFromDocument, buildContinuationContext, parseJsonWithModel ) logger = logging.getLogger(__name__) # Rebuild the model to resolve forward references AiCallRequest.model_rebuild() class AiService: """AI service with core operations integrated.""" def __init__(self, serviceCenter=None) -> None: """Initialize AI service with service center access. Args: serviceCenter: Service center instance for accessing other services """ self.services = serviceCenter # Only depend on interfaces self.aiObjects = None # Will be initialized in create() or ensureAiObjectsInitialized() # Submodules initialized as None - will be set in _initializeSubmodules() after aiObjects is ready self.extractionService = None def _initializeSubmodules(self): """Initialize all submodules after aiObjects is ready.""" if self.aiObjects is None: raise RuntimeError("aiObjects must be initialized before initializing submodules") if self.extractionService is None: logger.info("Initializing ExtractionService...") self.extractionService = ExtractionService(self.services) async def ensureAiObjectsInitialized(self): """Ensure aiObjects is initialized and submodules are ready.""" if self.aiObjects is None: logger.info("Lazy initializing AiObjects...") self.aiObjects = await AiObjects.create() logger.info("AiObjects initialization completed") # Initialize submodules after aiObjects is ready self._initializeSubmodules() @classmethod async def create(cls, serviceCenter=None) -> "AiService": """Create AiService instance with all connectors and submodules initialized.""" logger.info("AiService.create() called") instance = cls(serviceCenter) logger.info("AiService created, about to call AiObjects.create()...") instance.aiObjects = await AiObjects.create() logger.info("AiObjects.create() completed") # Initialize all submodules after aiObjects is ready instance._initializeSubmodules() logger.info("AiService submodules initialized") return instance # Helper methods def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str: """ Build full prompt by replacing placeholders with their content. Uses the new {{KEY:placeholder}} format. Args: prompt: The base prompt template placeholders: Dictionary of placeholder key-value pairs Returns: Prompt with placeholders replaced """ if not placeholders: return prompt full_prompt = prompt for placeholder, content in placeholders.items(): # Skip if content is None or empty if content is None: continue # Replace {{KEY:placeholder}} full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", str(content)) return full_prompt async def _analyzePromptAndCreateOptions(self, prompt: str) -> AiCallOptions: """Analyze prompt to determine appropriate AiCallOptions parameters.""" try: # Get dynamic enum values from Pydantic models operationTypes = [e.value for e in OperationTypeEnum] priorities = [e.value for e in PriorityEnum] processingModes = [e.value for e in ProcessingModeEnum] # Create analysis prompt for AI to determine operation type and parameters analysisPrompt = f""" You are an AI operation analyzer. Analyze the following prompt and determine the most appropriate operation type and parameters. PROMPT TO ANALYZE: {self.services.utils.sanitizePromptContent(prompt, 'userinput')} Based on the prompt content, determine: 1. operationType: Choose the most appropriate from: {', '.join(operationTypes)} 2. priority: Choose from: {', '.join(priorities)} 3. processingMode: Choose from: {', '.join(processingModes)} 4. compressPrompt: true/false (true for story-like prompts, false for structured prompts with JSON/schemas) 5. compressContext: true/false (true to summarize context, false to process fully) Respond with ONLY a JSON object in this exact format: {{ "operationType": "dataAnalyse", "priority": "balanced", "processingMode": "basic", "compressPrompt": true, "compressContext": true }} """ # Use AI to analyze the prompt request = AiCallRequest( prompt=analysisPrompt, options=AiCallOptions( operationType=OperationTypeEnum.DATA_ANALYSE, priority=PriorityEnum.SPEED, processingMode=ProcessingModeEnum.BASIC, compressPrompt=True, compressContext=False ) ) response = await self.aiObjects.call(request) # Parse AI response using structured parsing with AiCallOptions model try: # Use parseJsonWithModel to parse response into AiCallOptions (handles enum conversion automatically) analysis = parseJsonWithModel(response.content, AiCallOptions) return analysis except Exception as e: logger.warning(f"Failed to parse AI analysis response: {e}") except Exception as e: logger.warning(f"Prompt analysis failed: {e}") # Fallback to default options return AiCallOptions( operationType=OperationTypeEnum.DATA_ANALYSE, priority=PriorityEnum.BALANCED, processingMode=ProcessingModeEnum.BASIC ) async def _callAiWithLooping( self, prompt: str, options: AiCallOptions, debugPrefix: str = "ai_call", promptBuilder: Optional[callable] = None, promptArgs: Optional[Dict[str, Any]] = None, operationId: Optional[str] = None, userPrompt: Optional[str] = None, workflowIntent: Optional[Dict[str, Any]] = None ) -> str: """ Shared core function for AI calls with repair-based looping system. Automatically repairs broken JSON and continues generation seamlessly. Args: prompt: The prompt to send to AI options: AI call configuration options debugPrefix: Prefix for debug file names promptBuilder: Optional function to rebuild prompts for continuation promptArgs: Optional arguments for prompt builder operationId: Optional operation ID for progress tracking Returns: Complete AI response after all iterations """ maxIterations = 50 # Prevent infinite loops iteration = 0 allSections = [] # Accumulate all sections across iterations lastRawResponse = None # Store last raw JSON response for continuation documentMetadata = None # Store document metadata (title, filename) from first iteration # Get parent log ID for iteration operations parentLogId = None if operationId: parentLogId = self.services.chat.getOperationLogId(operationId) while iteration < maxIterations: iteration += 1 # Create separate operation for each iteration with parent reference iterationOperationId = None if operationId: iterationOperationId = f"{operationId}_iter_{iteration}" self.services.chat.progressLogStart( iterationOperationId, "AI Call", f"Iteration {iteration}", "", parentId=parentLogId ) # Build iteration prompt if len(allSections) > 0 and promptBuilder and promptArgs: # This is a continuation - build continuation context with raw JSON and rebuild prompt continuationContext = buildContinuationContext(allSections, lastRawResponse) if not lastRawResponse: logger.warning(f"Iteration {iteration}: No previous response available for continuation!") # CRITICAL: Add workflowIntent (actionIntent) to continuationContext for DoD-based progress filtering # This allows buildGenerationPrompt to filter progress stats based on Definition of Done KPIs if workflowIntent: continuationContext['taskIntent'] = workflowIntent # Keep key name 'taskIntent' for compatibility # Filter promptArgs to only include parameters that buildGenerationPrompt accepts # buildGenerationPrompt accepts: outputFormat, userPrompt, title, extracted_content, continuationContext filteredPromptArgs = { k: v for k, v in promptArgs.items() if k in ['outputFormat', 'userPrompt', 'title', 'extracted_content'] } # Rebuild prompt with continuation context using the provided prompt builder iterationPrompt = await promptBuilder(**filteredPromptArgs, continuationContext=continuationContext) else: # First iteration - use original prompt iterationPrompt = prompt # Make AI call try: if iterationOperationId: self.services.chat.progressLogUpdate(iterationOperationId, 0.3, "Calling AI model") request = AiCallRequest( prompt=iterationPrompt, context="", options=options ) # Write the ACTUAL prompt sent to AI if iteration == 1: self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt") else: self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") response = await self.aiObjects.call(request) result = response.content # Update progress after AI call if iterationOperationId: self.services.chat.progressLogUpdate(iterationOperationId, 0.6, "AI response received") # Write raw AI response to debug file if iteration == 1: self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") else: self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}") # Emit stats for this iteration (only if workflow exists and has id) if self.services.workflow and hasattr(self.services.workflow, 'id') and self.services.workflow.id: try: self.services.chat.storeWorkflowStat( self.services.workflow, response, f"ai.call.{debugPrefix}.iteration_{iteration}" ) except Exception as statError: # Don't break the main loop if stat storage fails logger.warning(f"Failed to store workflow stat: {str(statError)}") if not result or not result.strip(): logger.warning(f"Iteration {iteration}: Empty response, stopping") break # Store raw response for continuation (even if broken) lastRawResponse = result # Extract sections from response (handles both valid and broken JSON) extractedSections, wasJsonComplete, parsedResult = self._extractSectionsFromResponse(result, iteration, debugPrefix) # Extract document metadata from first iteration if available if iteration == 1 and parsedResult and not documentMetadata: documentMetadata = self._extractDocumentMetadata(parsedResult) # Update progress after parsing if iterationOperationId: if extractedSections: self.services.chat.progressLogUpdate(iterationOperationId, 0.8, f"Extracted {len(extractedSections)} sections") if not extractedSections: # CRITICAL: If JSON was incomplete/broken, continue even if no sections extracted # This allows the AI to retry and complete the broken JSON if not wasJsonComplete: logger.warning(f"Iteration {iteration}: No sections extracted from broken JSON, continuing for another attempt") continue # If JSON was complete but no sections extracted - this is an error, stop logger.warning(f"Iteration {iteration}: No sections extracted from complete JSON, stopping") break # Merge new sections with existing sections intelligently # This handles the STANDARD CASE: broken JSON iterations must be merged together # The break can occur anywhere - in any section, at any depth allSections = self._mergeSectionsIntelligently(allSections, extractedSections, iteration) # Check if we should continue (completion detection) # Extract user prompt from promptArgs if available extractedUserPrompt = userPrompt if not extractedUserPrompt and promptArgs: extractedUserPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt") if not extractedUserPrompt: # Try to extract from original prompt if "User request:" in prompt: try: extractedUserPrompt = prompt.split("User request:")[1].split("\n")[0].strip('"') except: pass shouldContinue = self._shouldContinueGeneration( allSections, iteration, wasJsonComplete, result, userPrompt=extractedUserPrompt, workflowIntent=workflowIntent ) if shouldContinue: # Finish iteration operation (will continue with next iteration) if iterationOperationId: self.services.chat.progressLogFinish(iterationOperationId, True) continue else: # Done - finish iteration and update main operation if iterationOperationId: self.services.chat.progressLogFinish(iterationOperationId, True) if operationId: self.services.chat.progressLogUpdate(operationId, 0.95, f"Generation complete ({iteration} iterations, {len(allSections)} sections)") logger.info(f"Generation complete after {iteration} iterations: {len(allSections)} sections") break except Exception as e: logger.error(f"Error in AI call iteration {iteration}: {str(e)}") if iterationOperationId: self.services.chat.progressLogFinish(iterationOperationId, False) break if iteration >= maxIterations: logger.warning(f"AI call stopped after maximum iterations ({maxIterations})") # Build final result from accumulated sections final_result = self._buildFinalResultFromSections(allSections, documentMetadata) # Write final result to debug file self.services.utils.writeDebugFile(final_result, f"{debugPrefix}_final_result") return final_result def _mergeSectionsIntelligently( self, existingSections: List[Dict[str, Any]], newSections: List[Dict[str, Any]], iteration: int ) -> List[Dict[str, Any]]: """ Intelligently merge sections from multiple iterations. This is a GENERIC merging strategy that handles broken JSON iterations. The break can occur anywhere - in any section, at any depth. Merging strategies (in order of priority): 1. Same Section ID: Merge sections with identical IDs 2. Same Content-Type + Position: If last section is incomplete and new section continues it 3. Same Order: Merge sections with same order value 4. Structural Analysis: Detect continuation based on content structure Args: existingSections: Sections accumulated from previous iterations newSections: Sections extracted from current iteration iteration: Current iteration number Returns: Merged list of sections """ if not newSections: return existingSections if not existingSections: return newSections mergedSections = existingSections.copy() for newSection in newSections: merged = False # Strategy 1: Same Section ID - merge directly newSectionId = newSection.get("id") if newSectionId: for i, existingSection in enumerate(mergedSections): if existingSection.get("id") == newSectionId: # Merge sections with same ID mergedSections[i] = self._mergeSectionContent(existingSection, newSection, iteration) merged = True logger.debug(f"Iteration {iteration}: Merged section by ID '{newSectionId}'") break if merged: continue # Strategy 2: Same Content-Type + Position (continuation detection) # Check if last section is incomplete and new section continues it if mergedSections: lastSection = mergedSections[-1] lastContentType = lastSection.get("content_type") newContentType = newSection.get("content_type") if lastContentType == newContentType: # Same content type - check if last section is incomplete if self._isSectionIncomplete(lastSection): # Last section is incomplete, merge with new section mergedSections[-1] = self._mergeSectionContent(lastSection, newSection, iteration) merged = True logger.debug(f"Iteration {iteration}: Merged section by content-type continuation ({lastContentType})") continue # Strategy 3: Same Order value newOrder = newSection.get("order") if newOrder is not None: for i, existingSection in enumerate(mergedSections): existingOrder = existingSection.get("order") if existingOrder is not None and existingOrder == newOrder: # Merge sections with same order mergedSections[i] = self._mergeSectionContent(existingSection, newSection, iteration) merged = True logger.debug(f"Iteration {iteration}: Merged section by order {newOrder}") break if merged: continue # Strategy 4: Structural Analysis - detect continuation # For code_block and table: if last section matches new section type, merge them if mergedSections: lastSection = mergedSections[-1] lastContentType = lastSection.get("content_type") newContentType = newSection.get("content_type") # Both are code blocks - merge them if lastContentType == "code_block" and newContentType == "code_block": mergedSections[-1] = self._mergeSectionContent(lastSection, newSection, iteration) merged = True logger.debug(f"Iteration {iteration}: Merged code_block sections by structural analysis") continue # Both are tables - merge them (common case for broken JSON iterations) if lastContentType == "table" and newContentType == "table": mergedSections[-1] = self._mergeSectionContent(lastSection, newSection, iteration) merged = True logger.debug(f"Iteration {iteration}: Merged table sections by structural analysis") continue # No merge strategy matched - add as new section if not merged: mergedSections.append(newSection) logger.debug(f"Iteration {iteration}: Added new section '{newSection.get('id', 'no-id')}' ({newSection.get('content_type', 'unknown')})") return mergedSections def _isSectionIncomplete(self, section: Dict[str, Any]) -> bool: """ Check if a section is incomplete (broken at the end). This detects incomplete sections based on content analysis: - Code blocks: ends mid-line, ends with comma, ends with incomplete structure - Text sections: ends mid-sentence, ends with incomplete structure - Other types: check for incomplete elements """ contentType = section.get("content_type", "") elements = section.get("elements", []) if not elements: return False # Handle list of elements if isinstance(elements, list) and len(elements) > 0: lastElement = elements[-1] else: lastElement = elements if not isinstance(lastElement, dict): return False # Check code_block for incomplete code if contentType == "code_block": code = lastElement.get("code", "") if code: # Check if code ends incompletely: # - Ends with comma (incomplete CSV line) # - Ends with number but no newline (incomplete line) # - Ends mid-token (e.g., "23431,23" - incomplete number) codeStripped = code.rstrip() if codeStripped: # Check for incomplete patterns if codeStripped.endswith(',') or (',' in codeStripped and not codeStripped.endswith('\n')): # Ends with comma or has comma but no final newline - likely incomplete return True # Check if last line is incomplete (doesn't end with newline and has partial content) if not code.endswith('\n') and codeStripped: # No final newline - might be incomplete # More sophisticated: check if last number is complete lastLine = codeStripped.split('\n')[-1] if lastLine and ',' in lastLine: # Has commas but might be incomplete parts = lastLine.split(',') if parts and len(parts[-1]) < 5: # Last part is very short - might be incomplete return True # Check table for incomplete rows if contentType == "table": rows = lastElement.get("rows", []) if rows: # Check if last row is incomplete (ends with incomplete data) lastRow = rows[-1] if isinstance(rows, list) else [] if isinstance(lastRow, list) and lastRow: # Check if last row ends with incomplete data (e.g., incomplete string) lastCell = lastRow[-1] if lastRow else "" if isinstance(lastCell, str): # If last cell is incomplete (ends with quote or is very short), section might be incomplete if lastCell.endswith('"') or (len(lastCell) < 3 and lastCell): return True # Also check if last row doesn't have expected number of columns (if headers exist) headers = lastElement.get("headers", []) if headers and isinstance(headers, list): expectedCols = len(headers) if len(lastRow) < expectedCols: return True # Check paragraph/text for incomplete sentences if contentType in ["paragraph", "heading"]: text = lastElement.get("text", "") if text: # Simple heuristic: if doesn't end with sentence-ending punctuation textStripped = text.rstrip() if textStripped and not textStripped[-1] in '.!?': # Might be incomplete, but this is less reliable # Only mark as incomplete if very short (likely cut off) if len(textStripped) < 20: return True # Check lists for incomplete items if contentType in ["bullet_list", "numbered_list"]: items = lastElement.get("items", []) if items and isinstance(items, list): # Check if last item is incomplete (very short or ends with incomplete string) lastItem = items[-1] if items else None if isinstance(lastItem, str) and len(lastItem) < 3: return True # Check if items array seems incomplete (e.g., expected count not reached) # This is harder to detect without context, so we rely on other heuristics # Check image for incomplete base64 data if contentType == "image": imageData = lastElement.get("base64Data", "") if imageData: # Base64 strings should end with padding ('=' or '==') # If it doesn't, it might be incomplete stripped = imageData.rstrip() if stripped and not stripped.endswith(('=', '==')): # Check if it's a valid base64 character sequence that was cut off # Base64 uses A-Z, a-z, 0-9, +, /, and = for padding if len(stripped) > 0 and stripped[-1] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=': return True # If length is not a multiple of 4 (base64 requirement), it might be incomplete if len(stripped) % 4 != 0: return True # GENERIC CHECK: Look for incomplete structures in any element # Check if element has arrays/lists that might be incomplete for key, value in lastElement.items(): if isinstance(value, list) and len(value) > 0: # Check last item in list lastItem = value[-1] if isinstance(lastItem, str): # If last string item is very short, might be incomplete if len(lastItem) < 3: return True elif isinstance(lastItem, dict): # If last dict item has very few keys, might be incomplete if len(lastItem) < 2: return True elif isinstance(value, str): # Check if string ends abruptly (no punctuation, very short) if len(value) > 0 and len(value) < 10 and not value[-1] in '.!?\n': return True return False def _mergeSectionContent( self, existingSection: Dict[str, Any], newSection: Dict[str, Any], iteration: int ) -> Dict[str, Any]: """ Merge content from two sections. Handles different content types: - code_block: Append code, handle overlaps, merge incomplete lines - paragraph/heading: Append text - table: Merge rows - list: Merge items - Other: Merge elements """ contentType = existingSection.get("content_type", "") existingElements = existingSection.get("elements", []) newElements = newSection.get("elements", []) if not newElements: return existingSection # Handle list of elements if isinstance(existingElements, list): existingElem = existingElements[-1] if existingElements else {} else: existingElem = existingElements if isinstance(newElements, list): newElem = newElements[0] if newElements else {} else: newElem = newElements if not isinstance(existingElem, dict) or not isinstance(newElem, dict): return existingSection # Merge based on content type if contentType == "code_block": existingCode = existingElem.get("code", "") newCode = newElem.get("code", "") if existingCode and newCode: mergedCode = self._mergeCodeBlocks(existingCode, newCode, iteration) existingElem["code"] = mergedCode # Preserve language from existing or new if "language" not in existingElem and "language" in newElem: existingElem["language"] = newElem["language"] elif contentType in ["paragraph", "heading"]: existingText = existingElem.get("text", "") newText = newElem.get("text", "") if existingText and newText: # Append text with space if needed if existingText.rstrip() and not existingText.rstrip()[-1] in '.!?\n': mergedText = existingText.rstrip() + " " + newText.lstrip() else: mergedText = existingText.rstrip() + "\n" + newText.lstrip() existingElem["text"] = mergedText elif contentType == "table": # Merge table rows with overlap detection existingRows = existingElem.get("rows", []) newRows = newElem.get("rows", []) if existingRows and newRows: # CRITICAL: Detect and remove overlaps before merging # Check if last existing row matches first new row (exact overlap) if len(existingRows) > 0 and len(newRows) > 0: lastExistingRow = existingRows[-1] firstNewRow = newRows[0] # Compare rows (handle both list and tuple formats) if isinstance(lastExistingRow, (list, tuple)) and isinstance(firstNewRow, (list, tuple)): if list(lastExistingRow) == list(firstNewRow): # Exact duplicate - remove first new row newRows = newRows[1:] logger.debug(f"Iteration {iteration}: Removed duplicate table row (exact match)") # Combine rows from both sections (after removing overlaps) existingElem["rows"] = existingRows + newRows logger.debug(f"Iteration {iteration}: Merged table rows - existing: {len(existingRows)}, new: {len(newRows)}, total: {len(existingRows) + len(newRows)}") elif newRows: # If existing has no rows but new does, use new rows existingElem["rows"] = newRows # Preserve headers from existing (or use new if existing has none) if not existingElem.get("headers") and newElem.get("headers"): existingElem["headers"] = newElem["headers"] # Preserve caption from existing (or use new if existing has none) if not existingElem.get("caption") and newElem.get("caption"): existingElem["caption"] = newElem["caption"] elif contentType in ["bullet_list", "numbered_list"]: # Merge list items existingItems = existingElem.get("items", []) newItems = newElem.get("items", []) if existingItems and newItems: existingElem["items"] = existingItems + newItems elif contentType == "image": # Images are typically complete - if new image is provided, replace existing # But check if existing image data is incomplete (e.g., base64 string cut off) existingImageData = existingElem.get("base64Data", "") newImageData = newElem.get("base64Data", "") if existingImageData and newImageData: # If existing image data doesn't end with valid base64 padding, it might be incomplete # Base64 padding is '=' or '==' at the end if not existingImageData.rstrip().endswith(('=', '==')): # Existing image might be incomplete - merge by appending new data # This handles cases where base64 string was cut off existingElem["base64Data"] = existingImageData + newImageData logger.debug(f"Iteration {iteration}: Merged incomplete image base64 data") else: # Existing image is complete - replace with new (or keep existing if new is empty) if newImageData: existingElem["base64Data"] = newImageData elif newImageData: existingElem["base64Data"] = newImageData # Preserve other image metadata if not existingElem.get("altText") and newElem.get("altText"): existingElem["altText"] = newElem["altText"] if not existingElem.get("caption") and newElem.get("caption"): existingElem["caption"] = newElem["caption"] else: # GENERIC FALLBACK: Handle any other content types or unknown structures # Try to merge common array/list fields generically for key in ["items", "rows", "columns", "cells", "elements", "data", "content"]: if key in existingElem and key in newElem: existingValue = existingElem[key] newValue = newElem[key] if isinstance(existingValue, list) and isinstance(newValue, list): # Merge lists by concatenation existingElem[key] = existingValue + newValue logger.debug(f"Iteration {iteration}: Merged generic list field '{key}' - existing: {len(existingValue)}, new: {len(newValue)}") break # If no common list fields found, try to merge all fields from newElem into existingElem # This handles cases where objects have different structures for key, value in newElem.items(): if key not in existingElem: # New field - add it existingElem[key] = value elif isinstance(existingElem[key], list) and isinstance(value, list): # Both are lists - merge them existingElem[key] = existingElem[key] + value elif isinstance(existingElem[key], dict) and isinstance(value, dict): # Both are dicts - recursively merge (shallow merge) existingElem[key].update(value) elif isinstance(existingElem[key], str) and isinstance(value, str): # Both are strings - append new to existing existingElem[key] = existingElem[key] + "\n" + value # Update section with merged content mergedSection = existingSection.copy() if isinstance(existingElements, list): # Update the last element in the list with merged content if existingElements: existingElements[-1] = existingElem mergedSection["elements"] = existingElements else: mergedSection["elements"] = existingElem # Preserve metadata from new section if missing in existing if "order" not in mergedSection and "order" in newSection: mergedSection["order"] = newSection["order"] return mergedSection def _mergeCodeBlocks(self, existingCode: str, newCode: str, iteration: int) -> str: """ Merge two code blocks intelligently, handling overlaps and incomplete lines. """ if not existingCode: return newCode if not newCode: return existingCode existingLines = existingCode.rstrip().split('\n') newLines = newCode.strip().split('\n') if not existingLines or not newLines: return existingCode + "\n" + newCode lastExistingLine = existingLines[-1].strip() firstNewLine = newLines[0].strip() # Strategy 1: Exact overlap - remove duplicate line if lastExistingLine == firstNewLine: newLines = newLines[1:] logger.debug(f"Iteration {iteration}: Removed exact duplicate line in code merge") # Strategy 2: Incomplete line merge # If last existing line ends with comma or is incomplete, merge with first new line elif lastExistingLine.endswith(',') or (',' in lastExistingLine and len(lastExistingLine.split(',')[-1]) < 5): # Last line is incomplete - merge with first new line # Remove trailing comma from existing line mergedLine = lastExistingLine.rstrip(',') + ',' + firstNewLine.lstrip() existingLines[-1] = mergedLine newLines = newLines[1:] logger.debug(f"Iteration {iteration}: Merged incomplete line with continuation") # Strategy 3: Partial overlap detection # Check if first new line starts with the end of last existing line elif ',' in lastExistingLine and ',' in firstNewLine: lastExistingParts = lastExistingLine.split(',') firstNewParts = firstNewLine.split(',') # Check for overlap: if last part of existing matches first part of new if lastExistingParts and firstNewParts: lastExistingPart = lastExistingParts[-1].strip() firstNewPart = firstNewParts[0].strip() # If they match, there's overlap if lastExistingPart == firstNewPart and len(lastExistingParts) > 1: # Remove overlapping part from new line newLines[0] = ','.join(firstNewParts[1:]) logger.debug(f"Iteration {iteration}: Removed partial overlap in code merge") # Reconstruct merged code mergedCode = '\n'.join(existingLines) if newLines: if mergedCode and not mergedCode.endswith('\n'): mergedCode += '\n' mergedCode += '\n'.join(newLines) return mergedCode def _extractSectionsFromResponse( self, result: str, iteration: int, debugPrefix: str ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]]]: """ Extract sections from AI response, handling both valid and broken JSON. Uses repair mechanism for broken JSON. Determines completion based on JSON structure (complete JSON = complete, broken/incomplete = incomplete). Returns (sections, wasJsonComplete, parsedResult) """ # First, try to parse as valid JSON try: extracted = extractJsonString(result) # CRITICAL: Check if raw response suggests incomplete JSON BEFORE parsing # extractFirstBalancedJson can return partial but valid JSON if raw is incomplete from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText raw_normalized = normalizeJsonText(stripCodeFences(result.strip())).strip() extracted_stripped = extracted.strip() # If extracted is shorter than raw, or raw doesn't end properly, it's incomplete is_raw_incomplete = False if len(extracted_stripped) < len(raw_normalized): is_raw_incomplete = True logger.info(f"Iteration {iteration}: Extracted JSON ({len(extracted_stripped)} chars) shorter than raw ({len(raw_normalized)} chars) - raw is incomplete") elif raw_normalized and not raw_normalized.endswith(('}', ']')): is_raw_incomplete = True logger.info(f"Iteration {iteration}: Raw response doesn't end with }} or ] - raw is incomplete") parsed_result = json.loads(extracted) # Extract sections from parsed JSON sections = extractSectionsFromDocument(parsed_result) # CRITICAL: If raw response is incomplete, mark as incomplete # JSON structure determines completion, not any flag if is_raw_incomplete: logger.info(f"Iteration {iteration}: JSON parseable but raw response incomplete - marking as incomplete") return sections, False, parsed_result # JSON was parseable and has sections or complete structure # Raw response ends properly = complete logger.info(f"Iteration {iteration}: JSON parseable and raw response complete - marking as complete") return sections, True, parsed_result except json.JSONDecodeError as e: # Broken JSON - try repair mechanism (normal in iterative generation) self.services.utils.writeDebugFile(result, f"{debugPrefix}_broken_json_iteration_{iteration}") logger.info(f"Iteration {iteration}: JSON parsing failed (broken JSON), attempting repair") # Try to repair repaired_json = repairBrokenJson(result) if repaired_json: # Extract sections from repaired JSON sections = extractSectionsFromDocument(repaired_json) # CRITICAL: JSON was broken, so mark as incomplete (wasJsonComplete = False) # This ensures the loop continues to get the rest of the content logger.info(f"Iteration {iteration}: JSON repaired, extracted {len(sections)} sections, marking as incomplete to continue") return sections, False, repaired_json # JSON was broken but repaired - mark as incomplete else: # Repair failed - but we should still continue to allow AI to retry logger.warning(f"Iteration {iteration}: All repair strategies failed, but continuing to allow retry") return [], False, None # Mark as incomplete so loop continues except Exception as e: logger.error(f"Iteration {iteration}: Unexpected error during parsing: {str(e)}") return [], False, None def _shouldContinueGeneration( self, allSections: List[Dict[str, Any]], iteration: int, wasJsonComplete: bool, rawResponse: str = None, userPrompt: Optional[str] = None, workflowIntent: Optional[Dict[str, Any]] = None ) -> bool: """ Determine if AI generation loop should continue. CRITICAL: This is ONLY about AI Loop Completion, NOT Action DoD! Action DoD is checked AFTER the AI Loop completes in _refineDecide. Simple logic: - If JSON is incomplete/broken → continue (needs more content) - If JSON is complete → stop (all content delivered) - Loop detection prevents infinite loops Returns True if we should continue, False if AI Loop is done. """ if len(allSections) == 0: return True # No sections yet, continue # CRITERION 1: If JSON was incomplete/broken - continue to repair/complete if not wasJsonComplete: logger.info(f"Iteration {iteration}: JSON incomplete/broken - continuing to complete") return True # CRITERION 2: JSON is complete - check for loop detection # If JSON is complete, we're done (all content delivered) # But check for infinite loops first if self._isStuckInLoop(allSections, iteration): logger.warning(f"Iteration {iteration}: Detected potential infinite loop - stopping AI loop") return False # JSON is complete and not stuck in loop - done logger.info(f"Iteration {iteration}: JSON complete - AI loop done") return False def _analyzeTaskCompletion( self, allSections: List[Dict[str, Any]], userPrompt: Optional[str], iteration: int, workflowIntent: Optional[Dict[str, Any]] = None ) -> bool: """ GENERIC task completion analysis using KPIs from Intent Analyzer. Uses definitionOfDone KPIs from workflowIntent to check completion. Falls back to simple heuristics if workflowIntent not available. Returns True if task appears complete, False otherwise. """ if not allSections: return False # Calculate current metrics from JSON structure totalSections = len(allSections) totalContentSize = 0 totalRows = 0 totalItems = 0 totalParagraphs = 0 totalHeadings = 0 totalCodeLines = 0 contentTypes = set() lastSectionComplete = True for section in allSections: contentType = section.get("content_type", "") contentTypes.add(contentType) elements = section.get("elements", []) if isinstance(elements, list) and elements: lastElem = elements[-1] if elements else {} else: lastElem = elements if isinstance(elements, dict) else {} if isinstance(lastElem, dict): if contentType == "code_block": code = lastElem.get("code", "") if code: lines = [l for l in code.split('\n') if l.strip()] totalCodeLines += len(lines) totalContentSize += len(code) if code and not code.rstrip().endswith('\n'): lastSectionComplete = False elif contentType == "table": rows = lastElem.get("rows", []) if isinstance(rows, list): totalRows += len(rows) totalContentSize += len(str(rows)) if not lastElem.get("headers"): lastSectionComplete = False elif contentType in ["bullet_list", "numbered_list"]: items = lastElem.get("items", []) if isinstance(items, list): totalItems += len(items) totalContentSize += len(str(items)) elif contentType == "heading": totalHeadings += 1 text = lastElem.get("text", "") if text: totalContentSize += len(text) elif contentType == "paragraph": totalParagraphs += 1 text = lastElem.get("text", "") if text: totalContentSize += len(text) if text and not text.rstrip()[-1] in '.!?': lastSectionComplete = False # STRATEGY 1: Use KPIs from Intent Analyzer (preferred method) if workflowIntent and isinstance(workflowIntent, dict): definitionOfDone = workflowIntent.get("definitionOfDone", {}) if definitionOfDone: # Check all KPI thresholds allKPIsMet = True kpiChecks = [] minSections = definitionOfDone.get("minSections", 0) if minSections > 0: met = totalSections >= minSections allKPIsMet = allKPIsMet and met kpiChecks.append(f"sections: {totalSections}/{minSections}") minParagraphs = definitionOfDone.get("minParagraphs", 0) if minParagraphs > 0: met = totalParagraphs >= minParagraphs allKPIsMet = allKPIsMet and met kpiChecks.append(f"paragraphs: {totalParagraphs}/{minParagraphs}") minHeadings = definitionOfDone.get("minHeadings", 0) if minHeadings > 0: met = totalHeadings >= minHeadings allKPIsMet = allKPIsMet and met kpiChecks.append(f"headings: {totalHeadings}/{minHeadings}") minTableRows = definitionOfDone.get("minTableRows", 0) if minTableRows > 0: met = totalRows >= minTableRows allKPIsMet = allKPIsMet and met kpiChecks.append(f"tableRows: {totalRows}/{minTableRows}") minListItems = definitionOfDone.get("minListItems", 0) if minListItems > 0: met = totalItems >= minListItems allKPIsMet = allKPIsMet and met kpiChecks.append(f"listItems: {totalItems}/{minListItems}") minCodeLines = definitionOfDone.get("minCodeLines", 0) if minCodeLines > 0: met = totalCodeLines >= minCodeLines allKPIsMet = allKPIsMet and met kpiChecks.append(f"codeLines: {totalCodeLines}/{minCodeLines}") minContentSize = definitionOfDone.get("minContentSize", 0) if minContentSize > 0: met = totalContentSize >= minContentSize allKPIsMet = allKPIsMet and met kpiChecks.append(f"contentSize: {totalContentSize}/{minContentSize}") # Check required content types requiredContentTypes = definitionOfDone.get("requiredContentTypes", []) if requiredContentTypes: met = all(ct in contentTypes for ct in requiredContentTypes) allKPIsMet = allKPIsMet and met kpiChecks.append(f"contentTypes: {list(contentTypes)}/{requiredContentTypes}") # If all KPIs met and last section is complete, task is done if allKPIsMet and lastSectionComplete: logger.info(f"Task completion (KPI-based): All KPIs met - {', '.join(kpiChecks)}") return True # STRATEGY 2: Fallback to simple heuristics if no workflowIntent # Only use if substantial content and last section complete if totalContentSize > 20000 and lastSectionComplete and iteration > 2: logger.info(f"Task completion (fallback heuristic): Large content ({totalContentSize} chars) over {iteration} iterations, last section complete") return True return False def _isStuckInLoop( self, allSections: List[Dict[str, Any]], iteration: int ) -> bool: """ Detect if we're stuck in a loop (same content being repeated). Generic approach: Check if recent iterations are adding minimal or duplicate content. """ if iteration < 3: return False # Need at least 3 iterations to detect a loop if len(allSections) == 0: return False # Check if last section is very small (might be stuck) lastSection = allSections[-1] elements = lastSection.get("elements", []) if isinstance(elements, list) and elements: lastElem = elements[-1] if elements else {} else: lastElem = elements if isinstance(elements, dict) else {} # Check content size of last section lastSectionSize = 0 if isinstance(lastElem, dict): for key, value in lastElem.items(): if isinstance(value, str): lastSectionSize += len(value) elif isinstance(value, list): lastSectionSize += len(str(value)) # If last section is very small and we've done many iterations, might be stuck if lastSectionSize < 100 and iteration > 10: logger.warning(f"Potential loop detected: iteration {iteration}, last section size {lastSectionSize}") return True return False def _extractDocumentMetadata( self, parsedResult: Dict[str, Any] ) -> Optional[Dict[str, Any]]: """ Extract document metadata (title, filename) from parsed AI response. Returns dict with 'title' and 'filename' keys if found, None otherwise. """ if not isinstance(parsedResult, dict): return None # Try to get from documents array (preferred structure) if "documents" in parsedResult and isinstance(parsedResult["documents"], list) and len(parsedResult["documents"]) > 0: firstDoc = parsedResult["documents"][0] if isinstance(firstDoc, dict): title = firstDoc.get("title") filename = firstDoc.get("filename") if title or filename: return { "title": title, "filename": filename } return None def _buildFinalResultFromSections( self, allSections: List[Dict[str, Any]], documentMetadata: Optional[Dict[str, Any]] = None ) -> str: """ Build final JSON result from accumulated sections. Uses AI-provided metadata (title, filename) if available. """ if not allSections: return "" # Extract metadata from AI response if available title = "Generated Document" filename = "document.json" if documentMetadata: if documentMetadata.get("title"): title = documentMetadata["title"] if documentMetadata.get("filename"): filename = documentMetadata["filename"] # Build documents structure # Assuming single document for now documents = [{ "id": "doc_1", "title": title, "filename": filename, "sections": allSections }] result = { "metadata": { "split_strategy": "single_document", "source_documents": [], "extraction_method": "ai_generation" }, "documents": documents } return json.dumps(result, indent=2) # Public API Methods # Planning AI Call async def callAiPlanning( self, prompt: str, placeholders: Optional[List[PromptPlaceholder]] = None, debugType: Optional[str] = None ) -> str: """ Planning AI call for task planning, action planning, action selection, etc. Always uses static parameters optimized for planning tasks. Args: prompt: The planning prompt placeholders: Optional list of placeholder replacements debugType: Optional debug file type identifier (e.g., 'taskplan', 'dynamic', 'intentanalysis') If not provided, defaults to 'plan' Returns: Planning JSON response """ await self.ensureAiObjectsInitialized() # Planning calls always use static parameters options = AiCallOptions( operationType=OperationTypeEnum.PLAN, priority=PriorityEnum.QUALITY, processingMode=ProcessingModeEnum.DETAILED, compressPrompt=False, compressContext=False ) # Build full prompt with placeholders if placeholders: placeholdersDict = {p.label: p.content for p in placeholders} fullPrompt = self._buildPromptWithPlaceholders(prompt, placeholdersDict) else: fullPrompt = prompt # Root-cause fix: planning must return raw single-shot JSON, not section-based output request = AiCallRequest( prompt=fullPrompt, context="", options=options ) # Debug: persist prompt/response for analysis with context-specific naming debugPrefix = debugType if debugType else "plan" self.services.utils.writeDebugFile(fullPrompt, f"{debugPrefix}_prompt") response = await self.aiObjects.call(request) result = response.content or "" self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") return result async def callAiContent( self, prompt: str, options: AiCallOptions, contentParts: Optional[List[ContentPart]] = None, outputFormat: Optional[str] = None, title: Optional[str] = None, parentOperationId: Optional[str] = None # Parent operation ID for hierarchical logging ) -> AiResponse: """ Unified AI content processing method (replaces callAiDocuments and callAiText). Args: prompt: The main prompt for the AI call contentParts: Optional list of already-extracted content parts (preferred) options: AI call configuration options (REQUIRED - operationType must be set) outputFormat: Optional output format for document generation (e.g., 'pdf', 'docx', 'xlsx') title: Optional title for generated documents parentOperationId: Optional parent operation ID for hierarchical logging Returns: AiResponse with content, metadata, and optional documents """ await self.ensureAiObjectsInitialized() # Create separate operationId for detailed progress tracking workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" aiOperationId = f"ai_content_{workflowId}_{int(time.time())}" # Get parent log ID if parent operation exists parentLogId = None if parentOperationId: parentLogId = self.services.chat.getOperationLogId(parentOperationId) # Start progress tracking with parent reference self.services.chat.progressLogStart( aiOperationId, "AI content processing", "Content Processing", f"Format: {outputFormat or 'text'}", parentId=parentLogId ) try: # Extraction is now separate - contentParts must be extracted before calling # Require operationType to be set before calling opType = getattr(options, "operationType", None) if not opType: # If outputFormat is specified, default to DATA_GENERATE if outputFormat: options.operationType = OperationTypeEnum.DATA_GENERATE opType = OperationTypeEnum.DATA_GENERATE else: self.services.chat.progressLogUpdate(aiOperationId, 0.1, "Analyzing prompt parameters") analyzedOptions = await self._analyzePromptAndCreateOptions(prompt) if analyzedOptions and hasattr(analyzedOptions, "operationType") and analyzedOptions.operationType: options.operationType = analyzedOptions.operationType # Merge other analyzed options if hasattr(analyzedOptions, "priority"): options.priority = analyzedOptions.priority if hasattr(analyzedOptions, "processingMode"): options.processingMode = analyzedOptions.processingMode if hasattr(analyzedOptions, "compressPrompt"): options.compressPrompt = analyzedOptions.compressPrompt if hasattr(analyzedOptions, "compressContext"): options.compressContext = analyzedOptions.compressContext else: # Default to DATA_ANALYSE if analysis fails options.operationType = OperationTypeEnum.DATA_ANALYSE opType = options.operationType # Handle IMAGE_GENERATE operations if opType == OperationTypeEnum.IMAGE_GENERATE: self.services.chat.progressLogUpdate(aiOperationId, 0.4, "Calling AI for image generation") request = AiCallRequest( prompt=prompt, context="", options=options ) response = await self.aiObjects.call(request) if response.content: # Build document data for image imageDoc = DocumentData( documentName="generated_image.png", documentData=response.content, mimeType="image/png" ) metadata = AiResponseMetadata( title=title or "Generated Image", operationType=opType.value ) self.services.chat.storeWorkflowStat( self.services.workflow, response, "ai.generate.image" ) self.services.chat.progressLogUpdate(aiOperationId, 0.9, "Image generated") self.services.chat.progressLogFinish(aiOperationId, True) return AiResponse( content=response.content, metadata=metadata, documents=[imageDoc] ) else: errorMsg = f"No image data returned: {response.content}" logger.error(f"Error in AI image generation: {errorMsg}") self.services.chat.progressLogFinish(aiOperationId, False) raise ValueError(errorMsg) # Handle WEB_SEARCH and WEB_CRAWL operations if opType == OperationTypeEnum.WEB_SEARCH or opType == OperationTypeEnum.WEB_CRAWL: self.services.chat.progressLogUpdate(aiOperationId, 0.4, f"Calling AI for {opType.name}") request = AiCallRequest( prompt=prompt, # Raw JSON prompt - connector will parse it context="", options=options ) response = await self.aiObjects.call(request) if response.content: metadata = AiResponseMetadata( operationType=opType.value ) self.services.chat.storeWorkflowStat( self.services.workflow, response, f"ai.{opType.name.lower()}" ) self.services.chat.progressLogUpdate(aiOperationId, 0.9, f"{opType.name} completed") self.services.chat.progressLogFinish(aiOperationId, True) return AiResponse( content=response.content, metadata=metadata ) else: errorMsg = f"No content returned from {opType.name}: {response.content}" logger.error(f"Error in {opType.name}: {errorMsg}") self.services.chat.progressLogFinish(aiOperationId, False) raise ValueError(errorMsg) # Handle document generation (outputFormat specified) if outputFormat: # CRITICAL: For document generation with JSON templates, NEVER compress the prompt options.compressPrompt = False options.compressContext = False # Convert contentParts to text for generation prompt (if provided) if contentParts: # Convert contentParts to text for generation prompt content_for_generation = "\n\n".join([f"[{part.label}]\n{part.data}" for part in contentParts if part.data]) else: content_for_generation = None self.services.chat.progressLogUpdate(aiOperationId, 0.3, "Building generation prompt") from modules.services.serviceGeneration.subPromptBuilderGeneration import buildGenerationPrompt generation_prompt = await buildGenerationPrompt( outputFormat, prompt, title, content_for_generation, None ) promptArgs = { "outputFormat": outputFormat, "userPrompt": prompt, "title": title, "extracted_content": content_for_generation } self.services.chat.progressLogUpdate(aiOperationId, 0.4, "Calling AI for content generation") # Extract user prompt from promptArgs for task completion analysis userPrompt = None if promptArgs: userPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt") # CRITICAL: Get actionIntent (not taskIntent or workflowIntent) for Definition of Done # Action Intent contains Definition of Done for THIS specific action # Each action needs its own DoD because actions have different completion criteria # Example: Action 1 "Generate 2000 primes" → DoD: 200 rows, Action 2 "Convert to CSV" → DoD: 1 document actionIntent = None if hasattr(self.services, 'workflow') and self.services.workflow: # Priority 1: Use actionIntent (most specific - for THIS action) actionIntent = getattr(self.services.workflow, '_actionIntent', None) if not actionIntent: # Priority 2: Fallback to taskIntent (for THIS task) actionIntent = getattr(self.services.workflow, '_taskIntent', None) if actionIntent: logger.info("Action intent not found, using task intent as fallback") if not actionIntent: # Priority 3: Fallback to workflowIntent (for entire workflow) actionIntent = getattr(self.services.workflow, '_workflowIntent', None) logger.warning("Action and task intent not found, using workflow intent as fallback") # Store actionIntent separately (not in promptArgs - buildGenerationPrompt doesn't accept it) # actionIntent is passed to _callAiWithLooping for completion detection, not for prompt building generated_json = await self._callAiWithLooping( generation_prompt, options, "document_generation", buildGenerationPrompt, promptArgs, # Does NOT contain taskIntent - buildGenerationPrompt doesn't accept it aiOperationId, userPrompt=userPrompt, workflowIntent=actionIntent # Use actionIntent (contains Definition of Done for THIS action) ) self.services.chat.progressLogUpdate(aiOperationId, 0.7, "Parsing generated JSON") try: extracted_json = self.services.utils.jsonExtractString(generated_json) generated_data = json.loads(extracted_json) except json.JSONDecodeError as e: logger.error(f"Failed to parse generated JSON: {str(e)}") self.services.utils.writeDebugFile(generated_json, "failed_json_parsing") self.services.chat.progressLogFinish(aiOperationId, False) raise ValueError(f"Generated content is not valid JSON: {str(e)}") # Extract title and filename from generated document structure extractedTitle = title extractedFilename = None if isinstance(generated_data, dict) and "documents" in generated_data: docs = generated_data["documents"] if isinstance(docs, list) and len(docs) > 0: firstDoc = docs[0] if isinstance(firstDoc, dict): if firstDoc.get("title"): extractedTitle = firstDoc["title"] if firstDoc.get("filename"): extractedFilename = firstDoc["filename"] # Ensure metadata contains the extracted title if "metadata" not in generated_data: generated_data["metadata"] = {} if extractedTitle: generated_data["metadata"]["title"] = extractedTitle # Create separate operation for content rendering renderOperationId = f"{aiOperationId}_render" renderParentLogId = self.services.chat.getOperationLogId(aiOperationId) self.services.chat.progressLogStart( renderOperationId, "Content Rendering", "Rendering", f"Format: {outputFormat}", parentId=renderParentLogId ) try: from modules.services.serviceGeneration.mainServiceGeneration import GenerationService generationService = GenerationService(self.services) self.services.chat.progressLogUpdate(renderOperationId, 0.5, f"Rendering to {outputFormat} format") rendered_content, mime_type = await generationService.renderReport( generated_data, outputFormat, extractedTitle or "Generated Document", prompt, self ) self.services.chat.progressLogFinish(renderOperationId, True) # Determine document name if extractedFilename: documentName = extractedFilename elif extractedTitle and extractedTitle != "Generated Document": sanitized = re.sub(r"[^a-zA-Z0-9._-]", "_", extractedTitle) sanitized = re.sub(r"_+", "_", sanitized).strip("_") if sanitized: if not sanitized.lower().endswith(f".{outputFormat}"): documentName = f"{sanitized}.{outputFormat}" else: documentName = sanitized else: documentName = f"generated.{outputFormat}" else: documentName = f"generated.{outputFormat}" # Build document data docData = DocumentData( documentName=documentName, documentData=rendered_content, mimeType=mime_type ) metadata = AiResponseMetadata( title=extractedTitle or title or "Generated Document", filename=extractedFilename, operationType=opType.value if opType else None ) self.services.utils.writeDebugFile(str(generated_data), "document_generation_response") self.services.chat.progressLogFinish(aiOperationId, True) return AiResponse( content=json.dumps(generated_data), metadata=metadata, documents=[docData] ) except Exception as e: logger.error(f"Error rendering document: {str(e)}") if renderOperationId: self.services.chat.progressLogFinish(renderOperationId, False) self.services.chat.progressLogFinish(aiOperationId, False) raise ValueError(f"Rendering failed: {str(e)}") # Handle text processing (no outputFormat) self.services.chat.progressLogUpdate(aiOperationId, 0.5, "Processing text call") if contentParts: # Process contentParts through AI # Convert contentParts to text for prompt contentText = "\n\n".join([f"[{part.label}]\n{part.data}" for part in contentParts if part.data]) fullPrompt = f"{prompt}\n\n{contentText}" if contentText else prompt result_content = await self._callAiWithLooping( fullPrompt, options, "text", None, None, aiOperationId ) else: # Direct text call (no documents to process) result_content = await self._callAiWithLooping( prompt, options, "text", None, None, aiOperationId ) metadata = AiResponseMetadata( operationType=opType.value if opType else None ) self.services.chat.progressLogFinish(aiOperationId, True) return AiResponse( content=result_content, metadata=metadata ) except Exception as e: logger.error(f"Error in callAiContent: {str(e)}") self.services.chat.progressLogFinish(aiOperationId, False) raise # DEPRECATED METHODS REMOVED: # - callAiDocuments() - replaced by callAiContent() # - callAiText() - replaced by callAiContent() # All call sites have been updated to use callAiContent()