streamlined core ai system to planning and documentation agents

modules/interfaces/interfaceDbChatObjects.py

@@ -978,7 +978,7 @@ class ChatObjects:
     def _storeDebugMessageAndDocuments(self, message: ChatMessage) -> None:
         """
         Store message and documents (metadata and file bytes) for debugging purposes.
-        Structure: gateway/test-chat/messages/m_round_task_action_timestamp/documentlist_label/
+        Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/
         - message.json, message_text.txt
         - document_###_metadata.json
         - document_###_<original_filename> (actual file bytes)
@@ -992,7 +992,13 @@ class ChatObjects:
             from datetime import datetime, UTC
             
             # Create base debug directory
-            debug_root = "./test-chat/messages"
+            # Use configured log directory instead of hardcoded test-chat
+            from modules.shared.configuration import APP_CONFIG
+            logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
+            if not os.path.isabs(logDir):
+                gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+                logDir = os.path.join(gatewayDir, logDir)
+            debug_root = os.path.join(logDir, 'debug', 'messages')
             os.makedirs(debug_root, exist_ok=True)
             
             # Generate timestamp

modules/services/serviceAi/mainServiceAi.py

@@ -153,43 +153,6 @@ class AiService:
         await self._ensureAiObjectsInitialized()
         return await self.webResearchService.webResearch(request)
 
-    # Master AI Call (process user prompt with optional unlimited count of input documents delivering one or many output documents, no size limitations)
-    async def callAi(
-        self,
-        prompt: str,
-        documents: Optional[List[ChatDocument]] = None,
-        placeholders: Optional[List[PromptPlaceholder]] = None,
-        options: Optional[AiCallOptions] = None,
-        outputFormat: Optional[str] = None,
-        title: Optional[str] = None
-        ) -> Union[str, Dict[str, Any]]:
-        """
-        Unified AI call interface that automatically routes to appropriate handler.
-        
-        Args:
-            prompt: The main prompt for the AI call
-            documents: Optional list of documents to process
-            placeholders: Optional list of placeholder replacements for planning calls
-            options: AI call configuration options
-            outputFormat: Optional output format (html, pdf, docx, txt, md, json, csv, xlsx) for document generation
-            title: Optional title for generated documents
-        
-        Returns:
-            AI response as string, or dict with documents if outputFormat is specified
-            
-        Raises:
-            Exception: If all available models fail
-        """
-        await self._ensureAiObjectsInitialized()
-        
-        # Get document processor and generator
-        documentProcessor = self.documentProcessor
-        documentGenerator = self.documentGenerator
-        
-        return await self.coreAi.callAi(
-            prompt, documents, placeholders, options, outputFormat, title,
-            documentProcessor, documentGenerator
-        )
 
     def sanitizePromptContent(self, content: str, contentType: str = "text") -> str:
         """

modules/services/serviceAi/subCoreAi.py

@@ -20,12 +20,211 @@ class SubCoreAi:
         self.services = services
         self.aiObjects = aiObjects
     
-    # AI Processing Call
+    # Shared Core Function for AI Calls with Looping
-    async def callAi(
+    async def _callAiWithLooping(
+        self,
+        prompt: str,
+        options: AiCallOptions,
+        debug_prefix: str = "ai_call"
+        ) -> str:
+        """
+        Shared core function for AI calls with looping system.
+        Handles continuation logic when response needs multiple rounds.
+        Delivers prompt and response to debug file log.
+        
+        Args:
+            prompt: The prompt to send to AI
+            options: AI call configuration options
+            debug_prefix: Prefix for debug file names
+            
+        Returns:
+            Complete AI response after all iterations
+        """
+        max_iterations = 10  # Prevent infinite loops
+        iteration = 0
+        accumulated_content = []
+        
+        logger.info(f"Starting AI call with looping (debug prefix: {debug_prefix})")
+        
+        # Write initial prompt to debug file
+        from modules.shared.debugLogger import writeDebugFile
+        writeDebugFile(prompt, f"{debug_prefix}_prompt", None)
+        
+        while iteration < max_iterations:
+            iteration += 1
+            logger.info(f"AI call iteration {iteration}/{max_iterations}")
+            
+            # Build iteration prompt
+            if iteration == 1:
+                iteration_prompt = prompt
+            else:
+                iteration_prompt = self._buildContinuationPrompt(prompt, accumulated_content, iteration)
+            
+            # Make AI call
+            try:
+                from modules.datamodels.datamodelAi import AiCallRequest
+                request = AiCallRequest(
+                    prompt=iteration_prompt,
+                    context="",
+                    options=options
+                )
+                response = await self.aiObjects.call(request)
+                result = response.content
+                
+                # Write raw AI response to debug file
+                writeDebugFile(result, f"{debug_prefix}_response_iteration_{iteration}", None)
+                
+                # Emit stats for this iteration
+                self.services.workflow.storeWorkflowStat(
+                    self.services.currentWorkflow, 
+                    response, 
+                    f"ai.call.{debug_prefix}.iteration_{iteration}"
+                )
+                
+                if not result or not result.strip():
+                    logger.warning(f"Iteration {iteration}: Empty response, stopping")
+                    break
+                
+                # Check if this is a continuation response
+                if "[CONTINUE:" in result:
+                    # Extract the content before the continuation marker
+                    content_part = result.split("[CONTINUE:")[0].strip()
+                    if content_part:
+                        accumulated_content.append(content_part)
+                    logger.info(f"Iteration {iteration}: Continuation detected, continuing...")
+                    continue
+                else:
+                    # This is the final response
+                    accumulated_content.append(result)
+                    logger.info(f"Iteration {iteration}: Final response received")
+                    break
+                    
+            except Exception as e:
+                logger.error(f"Error in AI call iteration {iteration}: {str(e)}")
+                break
+        
+        if iteration >= max_iterations:
+            logger.warning(f"AI call stopped after maximum iterations ({max_iterations})")
+        
+        # Combine all accumulated content
+        final_result = "\n\n".join(accumulated_content) if accumulated_content else ""
+        
+        # Write final result to debug file
+        writeDebugFile(final_result, f"{debug_prefix}_final_result", None)
+        
+        logger.info(f"AI call completed: {len(accumulated_content)} parts from {iteration} iterations")
+        return final_result
+
+    def _buildContinuationPrompt(
+        self,
+        base_prompt: str,
+        accumulated_content: List[str],
+        iteration: int
+        ) -> str:
+        """
+        Build a prompt for continuation iterations.
+        """
+        continuation_instructions = f"""
+
+CONTINUATION REQUEST (Iteration {iteration}):
+You are continuing from a previous response. Please continue generating content from where you left off.
+
+IMPORTANT: 
+- Continue from the exact point where you stopped
+- Maintain the same format and structure
+- If you cannot complete the full response, end with: [CONTINUE: brief description of what still needs to be generated]
+- Only stop when the response is completely generated
+
+Previous content generated:
+{chr(10).join(accumulated_content[-1:]) if accumulated_content else "None"}
+
+Continue generating content now:
+"""
+        
+        return f"{base_prompt}{continuation_instructions}"
+
+    def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str:
+        """
+        Build full prompt by replacing placeholders with their content.
+        Uses the new {{KEY:placeholder}} format.
+        """
+        if not placeholders:
+            return prompt
+        
+        full_prompt = prompt
+        for placeholder, content in placeholders.items():
+            # Replace both old format {{placeholder}} and new format {{KEY:placeholder}}
+            full_prompt = full_prompt.replace(f"{{{{{placeholder}}}}}", content)
+            full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", content)
+        
+        return full_prompt
+
+    async def _buildGenerationPrompt(
+        self,
+        prompt: str,
+        extracted_content: Optional[str],
+        outputFormat: str,
+        title: str
+        ) -> str:
+        """
+        Build generation prompt for document generation.
+        """
+        from modules.services.serviceGeneration.subPromptBuilder import buildGenerationPrompt
+        
+        # Build the generation prompt using the existing system
+        generation_prompt = await buildGenerationPrompt(
+            outputFormat=outputFormat,
+            userPrompt=prompt,
+            title=title,
+            aiService=self,
+            services=self.services
+        )
+        
+        # If we have extracted content, prepend it to the prompt
+        if extracted_content:
+            generation_prompt = f"""EXTRACTED CONTENT FROM DOCUMENTS:
+{extracted_content}
+
+{generation_prompt}"""
+        
+        return generation_prompt
+
+    # Planning AI Call
+    async def callAiPlanning(
+        self,
+        prompt: str,
+        placeholders: Optional[List[PromptPlaceholder]] = None,
+        options: Optional[AiCallOptions] = None
+        ) -> str:
+        """
+        Planning AI call for task planning, action planning, action selection, etc.
+        
+        Args:
+            prompt: The planning prompt
+            placeholders: Optional list of placeholder replacements
+            options: AI call configuration options
+            
+        Returns:
+            Planning JSON response
+        """
+        if options is None:
+            options = AiCallOptions()
+        
+        # Build full prompt with placeholders
+        if placeholders:
+            placeholders_dict = {p.key: p.value for p in placeholders}
+            full_prompt = self._buildPromptWithPlaceholders(prompt, placeholders_dict)
+        else:
+            full_prompt = prompt
+        
+        # Use shared core function with planning-specific debug prefix
+        return await self._callAiWithLooping(full_prompt, options, "planning")
+
+    # Document Generation AI Call
+    async def callAiDocuments(
         self,
         prompt: str,
         documents: Optional[List[ChatDocument]] = None,
-        placeholders: Optional[List[PromptPlaceholder]] = None,
         options: Optional[AiCallOptions] = None,
         outputFormat: Optional[str] = None,
         title: Optional[str] = None,
@@ -33,94 +232,43 @@ class SubCoreAi:
         documentGenerator=None
         ) -> Union[str, Dict[str, Any]]:
         """
-        Unified AI call interface that automatically routes to appropriate handler.
+        Document generation AI call for all non-planning calls.
+        Uses the current unified path with extraction and generation.
         
         Args:
             prompt: The main prompt for the AI call
             documents: Optional list of documents to process
-            placeholders: Optional list of placeholder replacements for planning calls
             options: AI call configuration options
-            outputFormat: Optional output format (html, pdf, docx, txt, md, json, csv, xlsx) for document generation
+            outputFormat: Optional output format for document generation
             title: Optional title for generated documents
             documentProcessor: Document processing service instance
             documentGenerator: Document generation service instance
         
         Returns:
             AI response as string, or dict with documents if outputFormat is specified
-            
-        Raises:
-            Exception: If all available models fail
         """
         if options is None:
             options = AiCallOptions()
         
-        # Normalize placeholders from List[PromptPlaceholder]
-        placeholders_dict: Dict[str, str] = {}
-        placeholders_meta: Dict[str, bool] = {}
-        if placeholders:
-            placeholders_dict = {p.label: p.content for p in placeholders}
-            placeholders_meta = {p.label: bool(getattr(p, 'summaryAllowed', False)) for p in placeholders}
-
-        # Auto-determine call type based on documents and operation type
-        call_type = self._determineCallType(documents, options.operationType)
-        options.callType = call_type
-        
-        try:
-            # Build the full prompt that will be sent to AI
-            if placeholders:
-                full_prompt = prompt
-                for p in placeholders:
-                    placeholder = f"{{{{KEY:{p.label}}}}}"
-                    full_prompt = full_prompt.replace(placeholder, p.content)
-            else:
-                full_prompt = prompt
-            
-            # Check for unresolved placeholders and clean them up
-            try:
-                import re
-                # Find only {{KEY:...}} patterns that need to be removed
-                unresolved_placeholders = re.findall(r'\{\{KEY:[^}]+\}\}', full_prompt)
-                if unresolved_placeholders:
-                    logger.warning(f"Found unresolved KEY placeholders in prompt: {unresolved_placeholders}")
-                    # Remove only {{KEY:...}} patterns, leave other {{...}} content intact
-                    full_prompt = re.sub(r'\{\{KEY:[^}]+\}\}', '', full_prompt)
-                    # Clean up extra whitespace
-                    full_prompt = re.sub(r'\n\s*\n\s*\n', '\n\n', full_prompt)
-                    full_prompt = full_prompt.strip()
-                    logger.info("Cleaned up unresolved KEY placeholders from prompt")
-            except Exception as e:
-                logger.warning(f"Error cleaning up prompt placeholders: {str(e)}")
-            
-            # Log the final integrated prompt that AI will receive
-            try:
-                from modules.shared.debugLogger import writeDebugFile
-                # Determine the prompt type based on operation type
-                if options.operationType == OperationType.GENERATE_PLAN:
-                    prompt_type = "taskplanPrompt"
-                elif options.operationType == OperationType.ANALYSE_CONTENT:
-                    prompt_type = "analysisPrompt"
-                else:
-                    prompt_type = "aiPrompt"
-                
-                writeDebugFile(full_prompt, prompt_type, documents)
-            except Exception:
-                pass  # Don't fail on debug logging
-        except Exception:
-            pass
-        
         # Handle document generation with specific output format using unified approach
         if outputFormat and documentGenerator:
             # Use unified generation method for all document generation
             if documents and len(documents) > 0:
                 # Extract content from documents first
                 logger.info(f"Extracting content from {len(documents)} documents")
-                extracted_content = await documentProcessor.callAiText(full_prompt, documents, options)
+                extracted_content = await documentProcessor.callAiText(prompt, documents, options)
-                # Generate with extracted content
+                # Generate with extracted content using shared core function
-                generated_json = await self._callAiUnifiedGeneration(full_prompt, extracted_content, options, outputFormat, title)
+                generation_prompt = await self._buildGenerationPrompt(prompt, extracted_content, outputFormat, title)
+                generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation")
             else:
                 # Direct generation without documents
                 logger.info("No documents provided - using direct generation")
-                generated_json = await self._callAiUnifiedGeneration(full_prompt, None, options, outputFormat, title)
+                generation_prompt = await self._buildGenerationPrompt(prompt, None, outputFormat, title)
+                generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation")
+            
+            # Write the generated JSON to debug file
+            from modules.shared.debugLogger import writeDebugFile
+            writeDebugFile(generated_json, "unified_generation_response", documents)
             
             # Parse the generated JSON
             try:
@@ -128,6 +276,13 @@ class SubCoreAi:
                 generated_data = json.loads(generated_json)
             except json.JSONDecodeError as e:
                 logger.error(f"Failed to parse generated JSON: {str(e)}")
+                logger.error(f"JSON content length: {len(generated_json)}")
+                logger.error(f"JSON content preview (last 200 chars): ...{generated_json[-200:]}")
+                logger.error(f"JSON content around error position: {generated_json[max(0, e.pos-50):e.pos+50]}")
+                
+                # Write the problematic JSON to debug file
+                writeDebugFile(generated_json, "failed_json_parsing", None)
+                
                 return {"success": False, "error": f"Generated content is not valid JSON: {str(e)}"}
             
             # Render to final format using the existing renderer
@@ -135,7 +290,7 @@ class SubCoreAi:
                 from modules.services.serviceGeneration.mainServiceGeneration import GenerationService
                 generationService = GenerationService(self.services)
                 rendered_content, mime_type = await generationService.renderReport(
-                    generated_data, outputFormat, title or "Generated Document", full_prompt, self
+                    generated_data, outputFormat, title or "Generated Document", prompt, self
                 )
                 
                 # Build result in the expected format
@@ -162,47 +317,24 @@ class SubCoreAi:
                     writeDebugFile(str(result), "documentGenerationResponse", documents)
                 except Exception:
                     pass
-                return result
                 
+                return result
+        
             except Exception as e:
                 logger.error(f"Error rendering document: {str(e)}")
                 return {"success": False, "error": f"Rendering failed: {str(e)}"}
         
-        if call_type == "planning":
+        # Handle text calls (no output format specified)
-            result = await self._callAiPlanning(prompt, placeholders_dict, placeholders_meta, options)
+        if documents and documentProcessor:
-            # Log AI response for debugging
+            # Use document processing for text calls with documents
-            try:
+            result = await documentProcessor.callAiText(prompt, documents, options)
-                from modules.shared.debugLogger import writeDebugFile
-                writeDebugFile(str(result or ""), "taskplanResponse", documents)
-            except Exception:
-                pass
-            return result
         else:
-            # Set processDocumentsIndividually from the legacy parameter if not set in options
+            # Use shared core function for direct text calls
-            if options.processDocumentsIndividually is None and documents:
+            result = await self._callAiWithLooping(prompt, options, "text")
-                options.processDocumentsIndividually = False  # Default to batch processing
+        
-            
-            # For text calls, we need to build the full prompt with placeholders here
-            # since _callAiText doesn't handle placeholders directly
-            if placeholders_dict:
-                full_prompt = self._buildPromptWithPlaceholders(prompt, placeholders_dict)
-            else:
-                full_prompt = prompt
-            
-            if documentProcessor and documents:
-                result = await documentProcessor.callAiText(full_prompt, documents, options)
-            else:
-                # Enhanced direct AI call with partial results support
-                result = await self._callAiWithPartialResults(full_prompt, options)
-            
-            # Log AI response for debugging (additional logging for text calls)
-            try:
-                from modules.shared.debugLogger import writeDebugFile
-                writeDebugFile(str(result or ""), "aiTextResponse", documents)
-            except Exception:
-                pass
             return result
 
+
     # AI Image Analysis
     async def readImage(
         self,
@@ -312,382 +444,14 @@ class SubCoreAi:
         else:
             return "text"
 
-    async def _callAiPlanning(
-        self,
-        prompt: str,
-        placeholders: Optional[Dict[str, str]],
-        placeholdersMeta: Optional[Dict[str, bool]],
-        options: AiCallOptions
-        ) -> str:
-        """
-        Handle planning calls with placeholder system and selective summarization.
-        """
-        # Build full prompt with placeholders; if too large, summarize summaryAllowed placeholders proportionally
-        effective_placeholders = placeholders or {}
-        full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders)
 
-        if options.compressPrompt and placeholdersMeta:
-            # Determine model capacity
-            try:
-                caps = self._getModelCapabilitiesForContent(full_prompt, None, options)
-                max_bytes = caps.get("maxContextBytes", len(full_prompt.encode("utf-8")))
-            except Exception:
-                max_bytes = len(full_prompt.encode("utf-8"))
 
-            current_bytes = len(full_prompt.encode("utf-8"))
-            if current_bytes > max_bytes:
-                # Compute total bytes contributed by allowed placeholders (approximate by content length)
-                allowed_labels = [l for l, allow in placeholdersMeta.items() if allow]
-                allowed_sizes = {l: len((effective_placeholders.get(l) or "").encode("utf-8")) for l in allowed_labels}
-                total_allowed = sum(allowed_sizes.values())
 
-                overage = current_bytes - max_bytes
-                if total_allowed > 0 and overage > 0:
-                    # Target total for allowed after reduction
-                    target_allowed = max(total_allowed - overage, 0)
-                    # Global ratio to apply across allowed placeholders
-                    ratio = target_allowed / total_allowed if total_allowed > 0 else 1.0
-                    ratio = max(0.0, min(1.0, ratio))
 
-                    reduced: Dict[str, str] = {}
-                    for label, content in effective_placeholders.items():
-                        if label in allowed_labels and isinstance(content, str) and len(content) > 0:
-                            old_len = len(content)
-                            # Reduce by proportional ratio on characters (fallback if empty)
-                            reduction_factor = ratio if old_len > 0 else 1.0
-                            reduced[label] = self._reduceText(content, reduction_factor)
-                        else:
-                            reduced[label] = content
 
-                    effective_placeholders = reduced
-                    full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders)
 
-                    # If still slightly over, perform a second-pass fine adjustment with updated ratio
-                    current_bytes = len(full_prompt.encode("utf-8"))
-                    if current_bytes > max_bytes and total_allowed > 0:
-                        overage2 = current_bytes - max_bytes
-                        # Recompute allowed sizes after first reduction
-                        allowed_sizes2 = {l: len((effective_placeholders.get(l) or "").encode("utf-8")) for l in allowed_labels}
-                        total_allowed2 = sum(allowed_sizes2.values())
-                        if total_allowed2 > 0 and overage2 > 0:
-                            target_allowed2 = max(total_allowed2 - overage2, 0)
-                            ratio2 = target_allowed2 / total_allowed2
-                            ratio2 = max(0.0, min(1.0, ratio2))
-                            reduced2: Dict[str, str] = {}
-                            for label, content in effective_placeholders.items():
-                                if label in allowed_labels and isinstance(content, str) and len(content) > 0:
-                                    old_len = len(content)
-                                    reduction_factor = ratio2 if old_len > 0 else 1.0
-                                    reduced2[label] = self._reduceText(content, reduction_factor)
-                                else:
-                                    reduced2[label] = content
-                            effective_placeholders = reduced2
-                            full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders)
-        
-        
-        # Make AI call using AiObjects (let it handle model selection)
-        request = AiCallRequest(
-            prompt=full_prompt,
-            context="",  # Context is already included in the prompt
-            options=options
-        )
-        response = await self.aiObjects.call(request)
-        try:
-            logger.debug(f"AI model selected (planning): {getattr(response, 'modelName', 'unknown')}")
-        except Exception:
-            pass
-        return response.content
 
-    async def _callAiWithPartialResults(
-        self,
-        prompt: str,
-        options: AiCallOptions
-        ) -> str:
-        """
-        Call AI with partial results continuation logic for direct calls.
-        Handles cases where AI needs to generate large responses in chunks.
-        """
-        logger.info("Starting direct AI call with partial results support")
-        
-        # Build enhanced prompt with continuation instructions
-        enhanced_prompt = self._buildDirectContinuationPrompt(prompt)
-        
-        # Process with continuation logic
-        return await self._processDirectWithContinuationLoop(enhanced_prompt, options)
 
-    def _buildDirectContinuationPrompt(self, base_prompt: str) -> str:
-        """
-        Build a prompt for direct AI calls that includes partial results instructions.
-        """
-        continuation_instructions = """
-
-IMPORTANT: If your response is too large to generate completely in one response, you can deliver partial results and continue.
-
-CONTINUATION LOGIC:
-- If you cannot complete the full response, end your response with:
-  [CONTINUE: brief description of what still needs to be generated]
-- The system will call you again to continue from where you left off
-- Continue generating from the exact point where you stopped
-- Maintain consistency with your previous partial response
-- Only stop when you have generated the complete response
-
-Examples:
-
-Example - Code Generation:
-If generating a large code file and you can only generate part of it:
-- Generate the first part (imports, classes, functions)
-- End with: [CONTINUE: Generate the remaining methods and main execution code]
-- In the next call, continue from where you left off
-
-Example - Documentation:
-If writing comprehensive documentation and you can only generate sections 1-3:
-- Generate sections 1-3 with full content
-- End with: [CONTINUE: Generate sections 4-8 covering advanced topics and examples]
-- In the next call, continue with sections 4-8
-
-This allows you to handle very large responses that exceed normal limits.
-"""
-        
-        return f"{base_prompt}{continuation_instructions}"
-
-    async def _processDirectWithContinuationLoop(
-        self,
-        enhanced_prompt: str,
-        options: AiCallOptions
-        ) -> str:
-        """
-        Process direct AI call with continuation loop until complete.
-        """
-        max_iterations = 10  # Prevent infinite loops
-        iteration = 0
-        accumulated_content = []
-        continuation_hint = None
-        
-        while iteration < max_iterations:
-            iteration += 1
-            logger.info(f"Direct AI continuation iteration {iteration}/{max_iterations}")
-            
-            # Build prompt for this iteration
-            if continuation_hint:
-                iteration_prompt = self._buildDirectContinuationIterationPrompt(
-                    enhanced_prompt, continuation_hint, accumulated_content
-                )
-            else:
-                iteration_prompt = enhanced_prompt
-            
-            # Make AI call for this iteration
-            try:
-                request = AiCallRequest(
-                    prompt=iteration_prompt,
-                    context="",
-                    options=options
-                )
-                response = await self.aiObjects.call(request)
-                result = response.content
-                
-                # Emit stats for this iteration
-                self.services.workflow.storeWorkflowStat(
-                    self.services.currentWorkflow, 
-                    response, 
-                    f"ai.call.{options.operationType}.iteration_{iteration}"
-                )
-                
-                if not result or not result.strip():
-                    logger.warning(f"Iteration {iteration}: Empty response, stopping")
-                    break
-                
-                # Check for continuation marker
-                if "[CONTINUE:" in result:
-                    # Extract the continuation hint
-                    import re
-                    continue_match = re.search(r'\[CONTINUE:\s*([^\]]+)\]', result)
-                    if continue_match:
-                        continuation_hint = continue_match.group(1).strip()
-                        # Remove the continuation marker from the result
-                        result = re.sub(r'\s*\[CONTINUE:[^\]]+\]', '', result).strip()
-                    else:
-                        continuation_hint = "Continue from where you left off"
-                    
-                    # Add this partial result to accumulated content
-                    if result.strip():
-                        accumulated_content.append(result.strip())
-                    
-                    logger.info(f"Iteration {iteration}: Partial result added, continue hint: {continuation_hint}")
-                else:
-                    # No continuation marker - this is the final result
-                    if result.strip():
-                        accumulated_content.append(result.strip())
-                    
-                    logger.info(f"Direct AI continuation complete after {iteration} iterations")
-                    break
-                    
-            except Exception as e:
-                logger.error(f"Direct AI iteration {iteration} failed: {str(e)}")
-                break
-        
-        if iteration >= max_iterations:
-            logger.warning(f"Direct AI continuation stopped after maximum iterations ({max_iterations})")
-        
-        # For JSON responses, we need to merge them properly instead of concatenating
-        if accumulated_content:
-            import json
-            # Parse each part as JSON and merge them
-            merged_documents = []
-            merged_metadata = None
-            
-            for content in accumulated_content:
-                parsed = json.loads(content)
-                if isinstance(parsed, dict):
-                    # Extract metadata from first valid JSON
-                    if merged_metadata is None and "metadata" in parsed:
-                        merged_metadata = parsed["metadata"]
-                    
-                    # Extract documents from this part
-                    if "documents" in parsed and isinstance(parsed["documents"], list):
-                        merged_documents.extend(parsed["documents"])
-            
-            # Create final merged JSON - NO FALLBACK
-            final_result = json.dumps({
-                "metadata": merged_metadata or {
-                    "title": "Generated Document",
-                    "splitStrategy": "single_document",
-                    "source_documents": [],
-                    "extraction_method": "ai_generation"
-                },
-                "documents": merged_documents
-            }, indent=2)
-        else:
-            # Return empty JSON structure if no content
-            final_result = json.dumps({
-                "metadata": {
-                    "title": "Generated Document",
-                    "splitStrategy": "single_document",
-                    "source_documents": [],
-                    "extraction_method": "ai_generation"
-                },
-                "documents": []
-            }, indent=2)
-        
-        logger.info(f"Final direct AI result: {len(accumulated_content)} parts from {iteration} iterations")
-        return final_result
-
-    def _buildDirectContinuationIterationPrompt(
-        self,
-        base_prompt: str,
-        continuation_hint: str,
-        accumulated_content: List[str]
-    ) -> str:
-        """
-        Build a prompt for continuation iteration with context.
-        """
-        # Build context of what's already been generated
-        context_summary = "PREVIOUSLY GENERATED CONTENT:\n"
-        for i, content in enumerate(accumulated_content[-2:]):  # Show last 2 parts for context
-            preview = content[:200] + "..." if len(content) > 200 else content
-            context_summary += f"Part {i+1}: {preview}\n"
-        
-        continuation_prompt = f"""
-{base_prompt}
-
-{context_summary}
-
-CONTINUATION INSTRUCTIONS:
-- Continue from where you left off
-- Continuation hint: {continuation_hint}
-- Generate the next part of the content
-- Maintain consistency with previously generated content
-- End with [CONTINUE: description] if more content is needed
-- End without [CONTINUE] if the response is complete
-"""
-        
-        return continuation_prompt
-
-    async def _callAiUnifiedGeneration(
-        self,
-        prompt: str,
-        extracted_content: Optional[str] = None,
-        options: Optional[AiCallOptions] = None,
-        outputFormat: str = "json",
-        title: str = "Generated Document"
-        ) -> str:
-        """
-        Unified generation method that handles both scenarios:
-        - With extracted content (from documents)
-        - Without extracted content (direct generation)
-        
-        Always uses continuation logic for long responses.
-        Always returns standardized JSON format using the multi-document schema.
-        """
-        if options is None:
-            options = AiCallOptions()
-        
-        logger.info("Starting unified AI generation with continuation logic")
-        
-        # Use the existing buildGenerationPrompt to get the proper canonical format instructions
-        from modules.services.serviceGeneration.subPromptBuilder import buildGenerationPrompt
-        
-        # Build the generation prompt using the existing system
-        generation_prompt = await buildGenerationPrompt(
-            outputFormat=outputFormat,
-            userPrompt=prompt,
-            title=title,
-            aiService=self,
-            services=self.services
-        )
-        
-        # If we have extracted content, prepend it to the prompt
-        if extracted_content:
-            generation_prompt = f"""EXTRACTED CONTENT FROM DOCUMENTS:
-{extracted_content}
-
-{generation_prompt}"""
-        
-        # Use continuation logic for long responses
-        return await self._processDirectWithContinuationLoop(generation_prompt, options)
-
-    async def _callAiDirect(
-        self,
-        prompt: str,
-        documents: Optional[List[ChatDocument]],
-        options: AiCallOptions,
-        documentProcessor=None
-        ) -> Dict[str, Any]:
-        """
-        Call AI directly with prompt and documents for JSON output.
-        Used for multi-file generation - uses the existing generation pipeline.
-        """
-        # Use the existing generation pipeline that already works
-        # This ensures proper document processing and content extraction
-        logger.info(f"Using existing generation pipeline for {len(documents) if documents else 0} documents")
-        
-        if documentProcessor:
-            # Process documents with JSON merging using the existing pipeline
-            result = await documentProcessor.processDocumentsPerChunkJson(documents, prompt, options)
-        else:
-            # Fallback to simple AI call
-            request = AiCallRequest(
-                prompt=prompt,
-                context="",
-                options=options
-            )
-            response = await self.aiObjects.call(request)
-            result = {"metadata": {"title": "AI Response"}, "sections": [{"id": "section_1", "content_type": "paragraph", "elements": [{"text": response.content}]}]}
-        
-        # Convert single-file result to multi-file format if needed
-        if "sections" in result and "documents" not in result:
-            logger.info("Converting single-file result to multi-file format")
-            # This is a single-file result, convert it to multi-file format
-            return {
-                "metadata": result.get("metadata", {"title": "Converted Document"}),
-                "documents": [{
-                    "id": "doc_1",
-                    "title": result.get("metadata", {}).get("title", "Document"),
-                    "filename": "document.txt",
-                    "sections": result.get("sections", [])
-                }]
-            }
-        
-        return result
 
     def _getModelCapabilitiesForContent(self, prompt: str, documents: Optional[List[ChatDocument]], options: AiCallOptions) -> Dict[str, int]:
         """

modules/services/serviceAi/subDocumentGeneration.py

@@ -100,6 +100,10 @@ class SubDocumentGeneration:
             
             # Update progress - generating extraction prompt
             progressLogger.updateProgress(operationId, 0.1, "Generating prompt")
+                     
+            # Write prompt to debug file
+            from modules.shared.debugLogger import writeDebugFile
+            writeDebugFile(extractionPrompt, "extraction_prompt", documents)
             
             # Process with unified JSON pipeline using continuation logic
             aiResponse = await self.documentProcessor.processDocumentsWithContinuation(
@@ -109,11 +113,13 @@ class SubDocumentGeneration:
             # Update progress - AI processing completed
             progressLogger.updateProgress(operationId, 0.6, "Processing done")
 
-            # Log the AI response for debugging
+
-            logger.info(f"AI response received for validation:")
+            
-            logger.info(f"  - Type: {type(aiResponse)}")
+            # Write AI response to debug file
-            logger.info(f"  - Keys: {list(aiResponse.keys()) if isinstance(aiResponse, dict) else 'Not a dict'}")
+            from modules.shared.debugLogger import writeDebugFile
-            logger.info(f"  - Content: {aiResponse}")
+            import json
+            response_json = json.dumps(aiResponse, indent=2, ensure_ascii=False) if isinstance(aiResponse, dict) else str(aiResponse)
+            writeDebugFile(response_json, "ai_response", documents)
 
             # Validate response structure
             if not self._validateUnifiedResponseStructure(aiResponse):

modules/services/serviceAi/subDocumentProcessing.py

@@ -605,7 +605,13 @@ CONTINUATION INSTRUCTIONS:
                                     import os
                                     from datetime import datetime, UTC
                                     ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
-                                    debug_root = "./test-chat/ai"
+                                    # Use configured log directory instead of hardcoded test-chat
+                                    from modules.shared.configuration import APP_CONFIG
+                                    logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
+                                    if not os.path.isabs(logDir):
+                                        gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+                                        logDir = os.path.join(gatewayDir, logDir)
+                                    debug_root = os.path.join(logDir, 'debug')
                                     os.makedirs(debug_root, exist_ok=True)
                                     with open(os.path.join(debug_root, f"{ts}_extraction_image_chunk_{chunk_index}.txt"), "w", encoding="utf-8") as f:
                                         f.write(f"EXTRACTION IMAGE RESPONSE:\n{ai_result if ai_result else 'No response'}\n")

modules/services/serviceAi/subUtilities.py

@@ -61,7 +61,7 @@ class SubUtilities:
             pass
 
     def _writeAiResponseDebug(self, label: str, content: str, partIndex: int = 1, modelName: str = None, continuation: bool = None) -> None:
-        """Persist raw AI response parts for debugging under test-chat/ai - only if debug enabled."""
+        """Persist raw AI response parts for debugging under configured log directory - only if debug enabled."""
         try:
             # Check if debug logging is enabled
             debug_enabled = self.services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False)
@@ -70,10 +70,13 @@ class SubUtilities:
                 
             import os
             from datetime import datetime, UTC
-            # Base dir: gateway/test-chat/ai (go up 4 levels from this file)
+            # Use configured log directory instead of hardcoded test-chat
-            # .../gateway/modules/services/serviceAi/subUtilities.py -> up to gateway root
+            from modules.shared.configuration import APP_CONFIG
-            gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+            logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
-            outDir = os.path.join(gatewayDir, 'test-chat', 'ai')
+            if not os.path.isabs(logDir):
+                gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+                logDir = os.path.join(gatewayDir, logDir)
+            outDir = os.path.join(logDir, 'debug')
             os.makedirs(outDir, exist_ok=True)
             ts = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3]
             suffix = []

modules/services/serviceGeneration/subPromptBuilder.py

@@ -403,7 +403,13 @@ DO NOT return a schema description - return actual extracted content in the JSON
             import os
             from datetime import datetime, UTC
             ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
-            debug_root = "./test-chat/ai"
+            # Use configured log directory instead of hardcoded test-chat
+            from modules.shared.configuration import APP_CONFIG
+            logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
+            if not os.path.isabs(logDir):
+                gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+                logDir = os.path.join(gatewayDir, logDir)
+            debug_root = os.path.join(logDir, 'debug')
             os.makedirs(debug_root, exist_ok=True)
             with open(os.path.join(debug_root, f"{ts}_extraction_prompt.txt"), "w", encoding="utf-8") as f:
                 f.write(finalPrompt)
@@ -435,118 +441,70 @@ async def buildGenerationPrompt(
         # Debug output
         services.utils.debugLogToFile(f"GENERATION PROMPT REQUEST: buildGenerationPrompt called with outputFormat='{outputFormat}', title='{title}'", "PROMPT_BUILDER")
         
-        # AI call to generate the appropriate generation prompt
+        # Return static generation prompt template instead of calling AI
-        generationPromptRequest = f"""
+        services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Using static template instead of AI call", "PROMPT_BUILDER")
-You are creating instructions for an AI to generate JSON content in the CANONICAL FORMAT that will be converted to a {outputFormat} document.
+        
+        # Return static generation prompt template
+        result = f"""You are an AI assistant that generates structured JSON content for document creation.
 
-User request: "{safeUserPrompt}"
+USER REQUEST: "{safeUserPrompt}"
-Document title: "{title}"
+DOCUMENT TITLE: "{title}"
-Target format: {outputFormat}
+TARGET FORMAT: {outputFormat}
 
-Write clear, detailed instructions that tell the AI how to generate JSON content using the CANONICAL JSON FORMAT. Focus on:
+TASK: Generate JSON content that fulfills the user's request.
 
-1. What content is most important for the user
+CRITICAL: You MUST return ONLY valid JSON in this exact structure:
-2. How to structure and organize the content using the canonical JSON format with 'sections'
-3. Specific formatting requirements for the target format
-4. Language requirements to preserve
-5. How to ensure the JSON content meets the user's needs
-
-CRITICAL: The AI MUST generate content using the CANONICAL JSON FORMAT with this exact structure:
 {{
   "metadata": {{
-    "title": "Document Title"
+    "title": "{title}",
+    "splitStrategy": "single_document",
+    "source_documents": [],
+    "extraction_method": "ai_generation"
   }},
-  "sections": [
+  "documents": [
     {{
-      "id": "section_1",
+      "id": "doc_1",
-      "content_type": "heading",
+      "title": "{title}",
-      "elements": [
+      "filename": "document.{outputFormat}",
+      "sections": [
         {{
-          "level": 1,
+          "id": "section_1",
-          "text": "1. SECTION TITLE"
+          "content_type": "heading",
-        }}
+          "elements": [
-      ],
+            {{
-      "order": 1
+              "level": 1,
-    }},
+              "text": "1. SECTION TITLE"
-    {{
+            }}
-      "id": "section_2",
+          ],
-      "content_type": "paragraph",
+          "order": 1
-      "elements": [
+        }},
         {{
-          "text": "This is the actual content that should be extracted from the document."
+          "id": "section_2",
+          "content_type": "paragraph",
+          "elements": [
+            {{
+              "text": "This is the actual content that should be generated."
+            }}
+          ],
+          "order": 2
         }}
-      ],
+      ]
-      "order": 2
-    }},
-    {{
-      "id": "section_3",
-      "content_type": "table",
-      "elements": [
-        {{
-          "headers": ["Column 1", "Column 2", "Column 3"],
-          "rows": [
-            ["Value 1", "Value 2", "Value 3"],
-            ["Value 4", "Value 5", "Value 6"]
-          ]
-        }}
-      ],
-      "order": 3
     }}
-  ],
+  ]
-  "continue": false
 }}
 
-IMPORTANT CHUNKING LOGIC:
+IMPORTANT: 
-- If the document is too large to generate completely in one response, set "continue": true
+- Return ONLY the JSON structure above
-- When "continue": true, include a "continuation_context" field with:
+- Do NOT include any text before or after the JSON
-  - "last_section_id": "id of the last completed section"
+- Fill in the actual content based on the user request: {safeUserPrompt}
-  - "last_element_index": "index of the last completed element in that section"
+- If the content is too large, you can split it into multiple sections
-  - "remaining_requirements": "brief description of what still needs to be generated"
+- Each section should have a unique id and appropriate content_type
-- The AI will be called again with this context to continue generation
-- Only set "continue": false when the document is completely generated
-
-The AI should NOT create format-specific structures like "sheets" or "columns" - only use the canonical format with "sections" and "elements".
-
-Write the instructions as plain text, not JSON. Start with "Generate JSON content that..." and provide clear, actionable instructions for creating structured JSON data in the canonical format.
 """
         
-        # Call AI service to generate the prompt
-        services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Calling AI for generation prompt...", "PROMPT_BUILDER")
-        
-        # Import and set proper options for AI call
-        from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType
-        request_options = AiCallOptions()
-        request_options.operationType = OperationType.GENERAL
-        
-        request = AiCallRequest(prompt=generationPromptRequest, context="", options=request_options)
-        response = await aiService.aiObjects.call(request)
-        result = response.content if response else ""
-        
-        # Replace the placeholder that the AI created with actual format rules
-        if result:
-            formatRules = _getFormatRules(outputFormat)
-            result = result.replace("PLACEHOLDER_FOR_FORMAT_RULES", formatRules)
-        
         # Debug output
         services.utils.debugLogToFile(f"GENERATION PROMPT: Generated successfully", "PROMPT_BUILDER")
         
-        # Save full generation prompt and AI response to debug file - only if debug enabled
+        return result.strip()
-        try:
-            debug_enabled = services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False)
-            if debug_enabled:
-                import os
-                from datetime import datetime, UTC
-                ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
-                debug_root = "./test-chat/ai"
-                os.makedirs(debug_root, exist_ok=True)
-                with open(os.path.join(debug_root, f"{ts}_generation_prompt.txt"), "w", encoding="utf-8") as f:
-                    f.write(f"GENERATION PROMPT REQUEST:\n{generationPromptRequest}\n\n")
-                    f.write(f"GENERATION PROMPT AI RESPONSE:\n{response.content if response else 'No response'}\n\n")
-                    f.write(f"GENERATION PROMPT FINAL:\n{result if result else 'None'}\n")
-        except Exception:
-            pass
-        
-        return result if result else f"Generate a comprehensive {outputFormat} document titled '{title}' based on the extracted content."
         
     except Exception as e:
         # Fallback on any error - preserve user prompt for language instructions

modules/services/serviceNormalization/mainServiceNormalization.py

@@ -90,7 +90,7 @@ class NormalizationService:
             "    \"Date\": {\"formats\": [\"DD.MM.YYYY\",\"YYYY-MM-DD\"]}\n  }\n}\n"
         )
 
-        response = await self.services.ai.callAi(prompt=prompt)
+        response = await self.services.ai.coreAi.callAiPlanning(prompt=prompt, placeholders=None, options=None)
         if not response:
             return {"mapping": {}, "normalizationPolicy": {}}
         
@@ -244,7 +244,13 @@ class NormalizationService:
             debugEnabled = self.services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False)
             if not debugEnabled:
                 return
-            root = "./test-chat/ai"
+            # Use configured log directory instead of hardcoded test-chat
+            from modules.shared.configuration import APP_CONFIG
+            logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
+            if not os.path.isabs(logDir):
+                gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+                logDir = os.path.join(gatewayDir, logDir)
+            root = os.path.join(logDir, 'debug')
             os.makedirs(root, exist_ok=True)
             # Prefix timestamp for files that are frequently overwritten
             ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")

modules/services/serviceUtils/mainServiceUtils.py

@@ -157,7 +157,12 @@ class UtilsService:
                 return
             
             # Get debug directory
-            debug_dir = self.configGet("APP_DEBUG_CHAT_WORKFLOW_DIR", "./test-chat")
+            # Use configured log directory instead of hardcoded test-chat
+            logDir = self.configGet("APP_LOGGING_LOG_DIR", "./")
+            if not os.path.isabs(logDir):
+                gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+                logDir = os.path.join(gatewayDir, logDir)
+            debug_dir = os.path.join(logDir, 'debug')
             if not os.path.isabs(debug_dir):
                 # If relative path, make it relative to the gateway directory
                 gateway_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))

modules/services/serviceWorkflow/mainServiceWorkflow.py

@@ -59,7 +59,7 @@ class WorkflowService:
             
             # Get summary using AI service directly (avoiding circular dependency)
             ai_service = AiService(self)
-            return await ai_service.callAi(
+            return await ai_service.coreAi.callAiDocuments(
                 prompt=prompt,
                 documents=None,
                 options={
@@ -69,7 +69,9 @@ class WorkflowService:
                     "compress_prompt": True,
                     "compress_documents": False,
                     "max_cost": 0.01
-                }
+                },
+                documentProcessor=ai_service.documentProcessor,
+                documentGenerator=ai_service.documentGenerator
             )
             
         except Exception as e:

modules/shared/debugLogger.py

@@ -1,16 +1,25 @@
 """
 Simple debug logger for AI prompts and responses.
-Writes files chronologically to gateway/test-chat/ai/ with sequential numbering.
+Writes files chronologically to the configured log directory with sequential numbering.
 """
 import os
 from datetime import datetime, UTC
 from typing import List, Optional
+from modules.shared.configuration import APP_CONFIG
 
 
 def _getDebugDir() -> str:
-    """Get the debug directory path."""
+    """Get the debug directory path from configuration."""
-    gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    # Get log directory from config (same as used by main logging system)
-    return os.path.join(gatewayDir, 'test-chat', 'ai')
+    logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
+    if not os.path.isabs(logDir):
+        # If relative path, make it relative to the gateway directory
+        gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        logDir = os.path.join(gatewayDir, logDir)
+    
+    # Create debug subdirectory within the log directory
+    debugDir = os.path.join(logDir, 'debug')
+    return debugDir
 
 
 def _getNextSequenceNumber() -> int:

modules/workflows/methods/methodAi.py

@@ -106,23 +106,8 @@ class MethodAi(MethodBase):
                 if chatDocuments:
                     logger.info(f"Prepared {len(chatDocuments)} documents for AI processing")
             
-            # Update progress - building prompt
+            # Update progress - preparing AI call
-            progressLogger.updateProgress(operationId, 0.4, "Building prompt")
+            progressLogger.updateProgress(operationId, 0.4, "Preparing AI call")
-            
-            # Build enhanced prompt
-            enhanced_prompt = aiPrompt
-            
-            # Add processing mode instructions if specified (generic, not analysis-specific)
-            if processingMode == "detailed":
-                enhanced_prompt += "\n\nPlease provide a detailed response with comprehensive information."
-            elif processingMode == "advanced":
-                enhanced_prompt += "\n\nPlease provide an advanced response with deep insights."
-            
-            # Note: customInstructions parameter was removed as it's not defined in the method signature
-            
-            # Add format guidance to prompt
-            if normalized_result_type != "txt":
-                enhanced_prompt += f"\n\nPlease deliver the result in {normalized_result_type.upper()} format. Ensure the output follows the proper {normalized_result_type.upper()} syntax and structure."
             
             # Build options and delegate document handling to AI/Extraction/Generation services
             output_format = output_extension.replace('.', '') or 'txt'
@@ -139,17 +124,16 @@ class MethodAi(MethodBase):
                 requiredTags=requiredTags
             )
 
-            supported_generation_formats = {"html", "pdf", "docx", "txt", "md", "json", "csv", "xlsx"}
-            output_format_arg = output_format if output_format in supported_generation_formats else None
-
             # Update progress - calling AI
             progressLogger.updateProgress(operationId, 0.6, "Calling AI")
 
-            result = await self.services.ai.callAi(
+            result = await self.services.ai.coreAi.callAiDocuments(
-                prompt=enhanced_prompt,
+                prompt=aiPrompt,  # Use original prompt, let unified generation handle prompt building
                 documents=chatDocuments if chatDocuments else None,
                 options=options,
-                outputFormat=output_format_arg
+                outputFormat=output_format,
+                documentProcessor=self.services.ai.documentProcessor,
+                documentGenerator=self.services.ai.documentGenerator
             )
             
             # Update progress - processing result

modules/workflows/methods/methodOutlook.py

@@ -1186,7 +1186,7 @@ Return JSON:
             
             # Call AI service to generate email content
             try:
-                ai_response = await self.services.ai.callAi(
+                ai_response = await self.services.ai.coreAi.callAiDocuments(
                     prompt=ai_prompt,
                     documents=chatDocuments,
                     options=AiCallOptions(
@@ -1199,7 +1199,9 @@ Return JSON:
                         resultFormat="json",
                         maxCost=0.50,
                         maxProcessingTime=30
-                    )
+                    ),
+                    documentProcessor=self.services.ai.documentProcessor,
+                    documentGenerator=self.services.ai.documentGenerator
                 )
                 
                 # Parse AI response

modules/workflows/processing/adaptive/contentValidator.py

@@ -120,9 +120,9 @@ DELIVERED CONTENT TO CHECK:
             request_options = AiCallOptions()
             request_options.operationType = OperationType.GENERAL
             
-            response = await self.services.ai.callAi(
+            response = await self.services.ai.coreAi.callAiPlanning(
                 prompt=validationPrompt,
-                documents=None,
+                placeholders=None,
                 options=request_options
             )
             

modules/workflows/processing/adaptive/intentAnalyzer.py

@@ -63,9 +63,9 @@ CRITICAL: Respond with ONLY the JSON object below. Do not include any explanator
             request_options = AiCallOptions()
             request_options.operationType = OperationType.GENERAL
             
-            response = await self.services.ai.callAi(
+            response = await self.services.ai.coreAi.callAiPlanning(
                 prompt=analysisPrompt,
-                documents=None,
+                placeholders=None,
                 options=request_options
             )
             

modules/workflows/processing/core/taskPlanner.py

@@ -105,7 +105,7 @@ class TaskPlanner:
                 maxProcessingTime=30
             )
             
-            prompt = await self.services.ai.callAi(
+            prompt = await self.services.ai.coreAi.callAiPlanning(
                 prompt=taskPlanningPromptTemplate,
                 placeholders=placeholders,
                 options=options

modules/workflows/processing/modes/modeActionplan.py

@@ -137,7 +137,7 @@ class ActionplanMode(BaseMode):
                 maxProcessingTime=30
             )
             
-            prompt = await self.services.ai.callAi(prompt=actionPromptTemplate, placeholders=placeholders, options=options)
+            prompt = await self.services.ai.coreAi.callAiPlanning(prompt=actionPromptTemplate, placeholders=placeholders, options=options)
             
             # Check if AI response is valid
             if not prompt:
@@ -476,7 +476,7 @@ class ActionplanMode(BaseMode):
                 maxProcessingTime=30
             )
             
-            response = await self.services.ai.callAi(prompt=promptTemplate, placeholders=placeholders, options=options)
+            response = await self.services.ai.coreAi.callAiPlanning(prompt=promptTemplate, placeholders=placeholders, options=options)
             
             # Log result review response received
             logger.info("=== RESULT REVIEW AI RESPONSE RECEIVED ===")

modules/workflows/processing/modes/modeReact.py

@@ -201,7 +201,7 @@ class ReactMode(BaseMode):
             maxProcessingTime=30
         )
         
-        response = await self.services.ai.callAi(
+        response = await self.services.ai.coreAi.callAiPlanning(
             prompt=promptTemplate,
             placeholders=placeholders,
             options=options
@@ -313,7 +313,7 @@ class ReactMode(BaseMode):
             resultFormat="json"  # Explicitly request JSON format
         )
         
-        paramsResp = await self.services.ai.callAi(
+        paramsResp = await self.services.ai.coreAi.callAiPlanning(
             prompt=promptTemplate,
             placeholders=placeholders,
             options=options
@@ -625,7 +625,7 @@ class ReactMode(BaseMode):
             maxProcessingTime=30
         )
         
-        resp = await self.services.ai.callAi(
+        resp = await self.services.ai.coreAi.callAiPlanning(
             prompt=promptTemplate,
             placeholders=placeholders,
             options=options
@@ -719,8 +719,9 @@ User language: {userLanguage}
 Return only the user-friendly message, no technical details."""
 
             # Call AI to generate user-friendly message
-            response = await self.services.ai.callAi(
+            response = await self.services.ai.coreAi.callAiPlanning(
                 prompt=prompt,
+                placeholders=None,
                 options=AiCallOptions(
                     operationType=OperationType.GENERATE_CONTENT,
                     priority=Priority.SPEED,
@@ -759,8 +760,9 @@ Result context: {resultContext}
 Return only the user-friendly message, no technical details."""
 
             # Call AI to generate user-friendly result message
-            response = await self.services.ai.callAi(
+            response = await self.services.ai.coreAi.callAiPlanning(
                 prompt=prompt,
+                placeholders=None,
                 options=AiCallOptions(
                     operationType=OperationType.GENERATE_CONTENT,
                     priority=Priority.SPEED,

modules/workflows/workflowManager.py

@@ -220,7 +220,7 @@ class WorkflowManager:
                 )
 
                 # Call AI analyzer
-                aiResponse = await self.services.ai.callAi(prompt=analyzerPrompt)
+                aiResponse = await self.services.ai.coreAi.callAiPlanning(prompt=analyzerPrompt, placeholders=None, options=None)
 
                 detectedLanguage = None
                 normalizedRequest = None

test_unified_architecture.py

@@ -1,258 +0,0 @@
-import asyncio
-import sys
-import os
-from unittest.mock import AsyncMock, MagicMock
-
-# Add the project root to the sys.path
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
-
-from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType
-from modules.datamodels.datamodelChat import ChatDocument
-from modules.services.serviceAi.subCoreAi import SubCoreAi
-
-class MockAiObjects:
-    def __init__(self, responses):
-        self.responses = responses
-        self.call_count = 0
-
-    async def call(self, request: AiCallRequest):
-        if self.call_count < len(self.responses):
-            response_content = self.responses[self.call_count]
-            self.call_count += 1
-            mock_response = MagicMock()
-            mock_response.content = response_content
-            mock_response.modelName = "mock-model"
-            mock_response.priceUsd = 0.001
-            mock_response.processingTime = 0.1
-            print(f"  Mock AI Call {self.call_count}: Responding with partial result (length: {len(response_content)})")
-            return mock_response
-        else:
-            print("  Mock AI Call: No more mock responses, returning empty.")
-            mock_response = MagicMock()
-            mock_response.content = ""
-            return mock_response
-
-class MockServices:
-    def __init__(self):
-        self.currentWorkflow = MagicMock()
-        self.currentWorkflow.id = "test_workflow_123"
-        self.workflow = MagicMock()
-        self.workflow.createProgressLogger.return_value = MagicMock()
-        self.workflow.storeWorkflowStat = AsyncMock()
-        self.ai = MagicMock()
-        self.ai.sanitizePromptContent.side_effect = lambda content, type: content
-        self.utils = MagicMock()
-        self.utils.debugLogToFile.side_effect = lambda msg, tag: print(f"  DEBUG ({tag}): {msg}")
-        self.utils.configGet.return_value = False # Disable debug files for tests
-
-class MockDocumentProcessor:
-    async def callAiText(self, prompt, documents, options):
-        return "Extracted content from documents: Sample text content"
-
-async def test_unified_architecture():
-    print("\n=== Testing Unified Architecture ===")
-    
-    # Mock responses: 1 for generation prompt building + 2 for actual generation
-    mock_responses = [
-        # Response 1: Generation prompt building
-        "Generate JSON content that creates a structured document with prime numbers in a table format. Use the canonical JSON format with sections and elements.",
-        
-        # Response 2: First part of generation
-        """{
-  "metadata": {
-    "title": "Prime Numbers List",
-    "splitStrategy": "single_document",
-    "source_documents": [],
-    "extraction_method": "ai_generation"
-  },
-  "documents": [
-    {
-      "id": "doc_primes_1_500",
-      "title": "Prime Numbers 1-500",
-      "filename": "primes_1_500.docx",
-      "sections": [
-        {
-          "id": "section_1",
-          "content_type": "table",
-          "elements": [
-            {
-              "headers": ["Number", "Prime"],
-              "rows": [
-                ["1", "2"], ["2", "3"], ["3", "5"], ["4", "7"], ["5", "11"]
-              ]
-            }
-          ],
-          "order": 1
-        }
-      ]
-    }
-  ]
-} [CONTINUE: Generate remaining prime numbers from 501 to 1000]""",
-        
-        # Response 3: Second part of generation
-        """{
-  "metadata": {
-    "title": "Prime Numbers List",
-    "splitStrategy": "single_document",
-    "source_documents": [],
-    "extraction_method": "ai_generation"
-  },
-  "documents": [
-    {
-      "id": "doc_primes_501_1000",
-      "title": "Prime Numbers 501-1000",
-      "filename": "primes_501_1000.docx",
-      "sections": [
-        {
-          "id": "section_2",
-          "content_type": "table",
-          "elements": [
-            {
-              "headers": ["Number", "Prime"],
-              "rows": [
-                ["501", "3571"], ["502", "3572"], ["503", "3581"]
-              ]
-            }
-          ],
-          "order": 2
-        }
-      ]
-    }
-  ]
-}"""
-    ]
-    
-    mock_ai_objects = MockAiObjects(mock_responses)
-    mock_services = MockServices()
-    mock_document_processor = MockDocumentProcessor()
-    
-    core_ai_service = SubCoreAi(mock_services, mock_ai_objects)
-    
-    prompt = "Generate the first 1000 prime numbers and arrange them in a structured table format."
-    options = AiCallOptions(operationType=OperationType.GENERATE_CONTENT)
-    output_format = "docx"
-    title = "Prime Numbers List"
-    
-    print(f"User Prompt: '{prompt}'")
-    print("Testing unified architecture with direct generation (no documents)...")
-    
-    # Test the unified generation method directly
-    result = await core_ai_service._callAiUnifiedGeneration(prompt, None, options, output_format, title)
-    
-    print("\n--- Generated JSON Result ---")
-    print(f"Result length: {len(result)} characters")
-    print(f"Result preview: {result[:300]}...")
-    
-    # Verify it's valid JSON
-    import json
-    try:
-        parsed_result = json.loads(result)
-        print(f"✅ Valid JSON with {len(parsed_result.get('documents', []))} documents")
-        
-        # Verify it's using the multi-document format
-        if "documents" in parsed_result and "metadata" in parsed_result:
-            print("✅ Using unified multi-document format")
-            print("✅ Architecture is properly unified!")
-            return True
-        else:
-            print("❌ Not using multi-document format")
-            return False
-    except json.JSONDecodeError as e:
-        print(f"❌ Invalid JSON: {str(e)}")
-        return False
-
-async def test_with_documents():
-    print("\n=== Testing Unified Architecture WITH Documents ===")
-    
-    # Mock responses: 1 for generation prompt building + 1 for actual generation
-    mock_responses = [
-        # Response 1: Generation prompt building
-        "Generate JSON content that creates a comprehensive fruit analysis report based on the extracted content. Use the canonical JSON format with sections and elements.",
-        
-        # Response 2: Generation with extracted content
-        """{
-  "metadata": {
-    "title": "Fruit Analysis Report",
-    "splitStrategy": "single_document",
-    "source_documents": ["doc1"],
-    "extraction_method": "ai_generation"
-  },
-  "documents": [
-    {
-      "id": "doc_fruit_analysis",
-      "title": "Fruit Analysis Report",
-      "filename": "fruit_analysis.docx",
-      "sections": [
-        {
-          "id": "section_1",
-          "content_type": "paragraph",
-          "elements": [
-            {
-              "text": "Based on the extracted content, here is a comprehensive fruit analysis..."
-            }
-          ],
-          "order": 1
-        }
-      ]
-    }
-  ]
-}"""
-    ]
-    
-    mock_ai_objects = MockAiObjects(mock_responses)
-    mock_services = MockServices()
-    mock_document_processor = MockDocumentProcessor()
-    
-    core_ai_service = SubCoreAi(mock_services, mock_ai_objects)
-    
-    prompt = "Extract all fruit information and create a comprehensive analysis report."
-    options = AiCallOptions(operationType=OperationType.GENERATE_CONTENT)
-    output_format = "docx"
-    title = "Fruit Analysis Report"
-    
-    print(f"User Prompt: '{prompt}'")
-    print("Testing unified architecture with document extraction...")
-    
-    # Test the unified generation method with extracted content
-    result = await core_ai_service._callAiUnifiedGeneration(prompt, "Sample fruit data: apples, oranges, bananas", options, output_format, title)
-    
-    print("\n--- Generated JSON Result ---")
-    print(f"Result length: {len(result)} characters")
-    print(f"Result preview: {result[:300]}...")
-    
-    # Verify it's valid JSON
-    import json
-    try:
-        parsed_result = json.loads(result)
-        print(f"✅ Valid JSON with {len(parsed_result.get('documents', []))} documents")
-        
-        # Verify it's using the multi-document format
-        if "documents" in parsed_result and "metadata" in parsed_result:
-            print("✅ Using unified multi-document format")
-            print("✅ Architecture is properly unified!")
-            return True
-        else:
-            print("❌ Not using multi-document format")
-            return False
-    except json.JSONDecodeError as e:
-        print(f"❌ Invalid JSON: {str(e)}")
-        return False
-
-async def main():
-    print("🚀 Testing Unified Architecture Implementation")
-    print("=" * 60)
-    
-    success1 = await test_unified_architecture()
-    success2 = await test_with_documents()
-    
-    if success1 and success2:
-        print("\n🎉 ALL TESTS PASSED! Unified architecture is properly implemented.")
-        print("✅ Single document = multi-document with n=1")
-        print("✅ Always uses multi-document JSON format")
-        print("✅ Continuation logic works for long responses")
-        print("✅ Both scenarios (with/without documents) work")
-    else:
-        print("\n❌ Some tests failed. Please check the implementation.")
-
-if __name__ == "__main__":
-    asyncio.run(main())