From e368819b1b9f5695e69ef508256ea2791bffe485 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 19 Oct 2025 23:27:45 +0200 Subject: [PATCH] streamlined core ai system to planning and documentation agents --- modules/interfaces/interfaceDbChatObjects.py | 10 +- modules/services/serviceAi/mainServiceAi.py | 37 - modules/services/serviceAi/subCoreAi.py | 702 ++++++------------ .../serviceAi/subDocumentGeneration.py | 16 +- .../serviceAi/subDocumentProcessing.py | 8 +- modules/services/serviceAi/subUtilities.py | 13 +- .../serviceGeneration/subPromptBuilder.py | 148 ++-- .../mainServiceNormalization.py | 10 +- .../services/serviceUtils/mainServiceUtils.py | 7 +- .../serviceWorkflow/mainServiceWorkflow.py | 6 +- modules/shared/debugLogger.py | 17 +- modules/workflows/methods/methodAi.py | 30 +- modules/workflows/methods/methodOutlook.py | 6 +- .../processing/adaptive/contentValidator.py | 4 +- .../processing/adaptive/intentAnalyzer.py | 4 +- .../workflows/processing/core/taskPlanner.py | 2 +- .../processing/modes/modeActionplan.py | 4 +- .../workflows/processing/modes/modeReact.py | 12 +- modules/workflows/workflowManager.py | 2 +- test_unified_architecture.py | 258 ------- 20 files changed, 377 insertions(+), 919 deletions(-) delete mode 100644 test_unified_architecture.py diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatObjects.py index ce39a1e5..d29c0f14 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatObjects.py @@ -978,7 +978,7 @@ class ChatObjects: def _storeDebugMessageAndDocuments(self, message: ChatMessage) -> None: """ Store message and documents (metadata and file bytes) for debugging purposes. - Structure: gateway/test-chat/messages/m_round_task_action_timestamp/documentlist_label/ + Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ - message.json, message_text.txt - document_###_metadata.json - document_###_ (actual file bytes) @@ -992,7 +992,13 @@ class ChatObjects: from datetime import datetime, UTC # Create base debug directory - debug_root = "./test-chat/messages" + # Use configured log directory instead of hardcoded test-chat + from modules.shared.configuration import APP_CONFIG + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + debug_root = os.path.join(logDir, 'debug', 'messages') os.makedirs(debug_root, exist_ok=True) # Generate timestamp diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index d2086c57..0c6293c0 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -153,43 +153,6 @@ class AiService: await self._ensureAiObjectsInitialized() return await self.webResearchService.webResearch(request) - # Master AI Call (process user prompt with optional unlimited count of input documents delivering one or many output documents, no size limitations) - async def callAi( - self, - prompt: str, - documents: Optional[List[ChatDocument]] = None, - placeholders: Optional[List[PromptPlaceholder]] = None, - options: Optional[AiCallOptions] = None, - outputFormat: Optional[str] = None, - title: Optional[str] = None - ) -> Union[str, Dict[str, Any]]: - """ - Unified AI call interface that automatically routes to appropriate handler. - - Args: - prompt: The main prompt for the AI call - documents: Optional list of documents to process - placeholders: Optional list of placeholder replacements for planning calls - options: AI call configuration options - outputFormat: Optional output format (html, pdf, docx, txt, md, json, csv, xlsx) for document generation - title: Optional title for generated documents - - Returns: - AI response as string, or dict with documents if outputFormat is specified - - Raises: - Exception: If all available models fail - """ - await self._ensureAiObjectsInitialized() - - # Get document processor and generator - documentProcessor = self.documentProcessor - documentGenerator = self.documentGenerator - - return await self.coreAi.callAi( - prompt, documents, placeholders, options, outputFormat, title, - documentProcessor, documentGenerator - ) def sanitizePromptContent(self, content: str, contentType: str = "text") -> str: """ diff --git a/modules/services/serviceAi/subCoreAi.py b/modules/services/serviceAi/subCoreAi.py index 84ef012f..506f66a1 100644 --- a/modules/services/serviceAi/subCoreAi.py +++ b/modules/services/serviceAi/subCoreAi.py @@ -20,12 +20,211 @@ class SubCoreAi: self.services = services self.aiObjects = aiObjects - # AI Processing Call - async def callAi( + # Shared Core Function for AI Calls with Looping + async def _callAiWithLooping( + self, + prompt: str, + options: AiCallOptions, + debug_prefix: str = "ai_call" + ) -> str: + """ + Shared core function for AI calls with looping system. + Handles continuation logic when response needs multiple rounds. + Delivers prompt and response to debug file log. + + Args: + prompt: The prompt to send to AI + options: AI call configuration options + debug_prefix: Prefix for debug file names + + Returns: + Complete AI response after all iterations + """ + max_iterations = 10 # Prevent infinite loops + iteration = 0 + accumulated_content = [] + + logger.info(f"Starting AI call with looping (debug prefix: {debug_prefix})") + + # Write initial prompt to debug file + from modules.shared.debugLogger import writeDebugFile + writeDebugFile(prompt, f"{debug_prefix}_prompt", None) + + while iteration < max_iterations: + iteration += 1 + logger.info(f"AI call iteration {iteration}/{max_iterations}") + + # Build iteration prompt + if iteration == 1: + iteration_prompt = prompt + else: + iteration_prompt = self._buildContinuationPrompt(prompt, accumulated_content, iteration) + + # Make AI call + try: + from modules.datamodels.datamodelAi import AiCallRequest + request = AiCallRequest( + prompt=iteration_prompt, + context="", + options=options + ) + response = await self.aiObjects.call(request) + result = response.content + + # Write raw AI response to debug file + writeDebugFile(result, f"{debug_prefix}_response_iteration_{iteration}", None) + + # Emit stats for this iteration + self.services.workflow.storeWorkflowStat( + self.services.currentWorkflow, + response, + f"ai.call.{debug_prefix}.iteration_{iteration}" + ) + + if not result or not result.strip(): + logger.warning(f"Iteration {iteration}: Empty response, stopping") + break + + # Check if this is a continuation response + if "[CONTINUE:" in result: + # Extract the content before the continuation marker + content_part = result.split("[CONTINUE:")[0].strip() + if content_part: + accumulated_content.append(content_part) + logger.info(f"Iteration {iteration}: Continuation detected, continuing...") + continue + else: + # This is the final response + accumulated_content.append(result) + logger.info(f"Iteration {iteration}: Final response received") + break + + except Exception as e: + logger.error(f"Error in AI call iteration {iteration}: {str(e)}") + break + + if iteration >= max_iterations: + logger.warning(f"AI call stopped after maximum iterations ({max_iterations})") + + # Combine all accumulated content + final_result = "\n\n".join(accumulated_content) if accumulated_content else "" + + # Write final result to debug file + writeDebugFile(final_result, f"{debug_prefix}_final_result", None) + + logger.info(f"AI call completed: {len(accumulated_content)} parts from {iteration} iterations") + return final_result + + def _buildContinuationPrompt( + self, + base_prompt: str, + accumulated_content: List[str], + iteration: int + ) -> str: + """ + Build a prompt for continuation iterations. + """ + continuation_instructions = f""" + +CONTINUATION REQUEST (Iteration {iteration}): +You are continuing from a previous response. Please continue generating content from where you left off. + +IMPORTANT: +- Continue from the exact point where you stopped +- Maintain the same format and structure +- If you cannot complete the full response, end with: [CONTINUE: brief description of what still needs to be generated] +- Only stop when the response is completely generated + +Previous content generated: +{chr(10).join(accumulated_content[-1:]) if accumulated_content else "None"} + +Continue generating content now: +""" + + return f"{base_prompt}{continuation_instructions}" + + def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str: + """ + Build full prompt by replacing placeholders with their content. + Uses the new {{KEY:placeholder}} format. + """ + if not placeholders: + return prompt + + full_prompt = prompt + for placeholder, content in placeholders.items(): + # Replace both old format {{placeholder}} and new format {{KEY:placeholder}} + full_prompt = full_prompt.replace(f"{{{{{placeholder}}}}}", content) + full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", content) + + return full_prompt + + async def _buildGenerationPrompt( + self, + prompt: str, + extracted_content: Optional[str], + outputFormat: str, + title: str + ) -> str: + """ + Build generation prompt for document generation. + """ + from modules.services.serviceGeneration.subPromptBuilder import buildGenerationPrompt + + # Build the generation prompt using the existing system + generation_prompt = await buildGenerationPrompt( + outputFormat=outputFormat, + userPrompt=prompt, + title=title, + aiService=self, + services=self.services + ) + + # If we have extracted content, prepend it to the prompt + if extracted_content: + generation_prompt = f"""EXTRACTED CONTENT FROM DOCUMENTS: +{extracted_content} + +{generation_prompt}""" + + return generation_prompt + + # Planning AI Call + async def callAiPlanning( + self, + prompt: str, + placeholders: Optional[List[PromptPlaceholder]] = None, + options: Optional[AiCallOptions] = None + ) -> str: + """ + Planning AI call for task planning, action planning, action selection, etc. + + Args: + prompt: The planning prompt + placeholders: Optional list of placeholder replacements + options: AI call configuration options + + Returns: + Planning JSON response + """ + if options is None: + options = AiCallOptions() + + # Build full prompt with placeholders + if placeholders: + placeholders_dict = {p.key: p.value for p in placeholders} + full_prompt = self._buildPromptWithPlaceholders(prompt, placeholders_dict) + else: + full_prompt = prompt + + # Use shared core function with planning-specific debug prefix + return await self._callAiWithLooping(full_prompt, options, "planning") + + # Document Generation AI Call + async def callAiDocuments( self, prompt: str, documents: Optional[List[ChatDocument]] = None, - placeholders: Optional[List[PromptPlaceholder]] = None, options: Optional[AiCallOptions] = None, outputFormat: Optional[str] = None, title: Optional[str] = None, @@ -33,94 +232,43 @@ class SubCoreAi: documentGenerator=None ) -> Union[str, Dict[str, Any]]: """ - Unified AI call interface that automatically routes to appropriate handler. + Document generation AI call for all non-planning calls. + Uses the current unified path with extraction and generation. Args: prompt: The main prompt for the AI call documents: Optional list of documents to process - placeholders: Optional list of placeholder replacements for planning calls options: AI call configuration options - outputFormat: Optional output format (html, pdf, docx, txt, md, json, csv, xlsx) for document generation + outputFormat: Optional output format for document generation title: Optional title for generated documents documentProcessor: Document processing service instance documentGenerator: Document generation service instance Returns: AI response as string, or dict with documents if outputFormat is specified - - Raises: - Exception: If all available models fail """ if options is None: options = AiCallOptions() - # Normalize placeholders from List[PromptPlaceholder] - placeholders_dict: Dict[str, str] = {} - placeholders_meta: Dict[str, bool] = {} - if placeholders: - placeholders_dict = {p.label: p.content for p in placeholders} - placeholders_meta = {p.label: bool(getattr(p, 'summaryAllowed', False)) for p in placeholders} - - # Auto-determine call type based on documents and operation type - call_type = self._determineCallType(documents, options.operationType) - options.callType = call_type - - try: - # Build the full prompt that will be sent to AI - if placeholders: - full_prompt = prompt - for p in placeholders: - placeholder = f"{{{{KEY:{p.label}}}}}" - full_prompt = full_prompt.replace(placeholder, p.content) - else: - full_prompt = prompt - - # Check for unresolved placeholders and clean them up - try: - import re - # Find only {{KEY:...}} patterns that need to be removed - unresolved_placeholders = re.findall(r'\{\{KEY:[^}]+\}\}', full_prompt) - if unresolved_placeholders: - logger.warning(f"Found unresolved KEY placeholders in prompt: {unresolved_placeholders}") - # Remove only {{KEY:...}} patterns, leave other {{...}} content intact - full_prompt = re.sub(r'\{\{KEY:[^}]+\}\}', '', full_prompt) - # Clean up extra whitespace - full_prompt = re.sub(r'\n\s*\n\s*\n', '\n\n', full_prompt) - full_prompt = full_prompt.strip() - logger.info("Cleaned up unresolved KEY placeholders from prompt") - except Exception as e: - logger.warning(f"Error cleaning up prompt placeholders: {str(e)}") - - # Log the final integrated prompt that AI will receive - try: - from modules.shared.debugLogger import writeDebugFile - # Determine the prompt type based on operation type - if options.operationType == OperationType.GENERATE_PLAN: - prompt_type = "taskplanPrompt" - elif options.operationType == OperationType.ANALYSE_CONTENT: - prompt_type = "analysisPrompt" - else: - prompt_type = "aiPrompt" - - writeDebugFile(full_prompt, prompt_type, documents) - except Exception: - pass # Don't fail on debug logging - except Exception: - pass - # Handle document generation with specific output format using unified approach if outputFormat and documentGenerator: # Use unified generation method for all document generation if documents and len(documents) > 0: # Extract content from documents first logger.info(f"Extracting content from {len(documents)} documents") - extracted_content = await documentProcessor.callAiText(full_prompt, documents, options) - # Generate with extracted content - generated_json = await self._callAiUnifiedGeneration(full_prompt, extracted_content, options, outputFormat, title) + extracted_content = await documentProcessor.callAiText(prompt, documents, options) + # Generate with extracted content using shared core function + generation_prompt = await self._buildGenerationPrompt(prompt, extracted_content, outputFormat, title) + generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation") else: # Direct generation without documents logger.info("No documents provided - using direct generation") - generated_json = await self._callAiUnifiedGeneration(full_prompt, None, options, outputFormat, title) + generation_prompt = await self._buildGenerationPrompt(prompt, None, outputFormat, title) + generated_json = await self._callAiWithLooping(generation_prompt, options, "document_generation") + + # Write the generated JSON to debug file + from modules.shared.debugLogger import writeDebugFile + writeDebugFile(generated_json, "unified_generation_response", documents) # Parse the generated JSON try: @@ -128,6 +276,13 @@ class SubCoreAi: generated_data = json.loads(generated_json) except json.JSONDecodeError as e: logger.error(f"Failed to parse generated JSON: {str(e)}") + logger.error(f"JSON content length: {len(generated_json)}") + logger.error(f"JSON content preview (last 200 chars): ...{generated_json[-200:]}") + logger.error(f"JSON content around error position: {generated_json[max(0, e.pos-50):e.pos+50]}") + + # Write the problematic JSON to debug file + writeDebugFile(generated_json, "failed_json_parsing", None) + return {"success": False, "error": f"Generated content is not valid JSON: {str(e)}"} # Render to final format using the existing renderer @@ -135,7 +290,7 @@ class SubCoreAi: from modules.services.serviceGeneration.mainServiceGeneration import GenerationService generationService = GenerationService(self.services) rendered_content, mime_type = await generationService.renderReport( - generated_data, outputFormat, title or "Generated Document", full_prompt, self + generated_data, outputFormat, title or "Generated Document", prompt, self ) # Build result in the expected format @@ -162,47 +317,24 @@ class SubCoreAi: writeDebugFile(str(result), "documentGenerationResponse", documents) except Exception: pass - return result + return result + except Exception as e: logger.error(f"Error rendering document: {str(e)}") return {"success": False, "error": f"Rendering failed: {str(e)}"} - if call_type == "planning": - result = await self._callAiPlanning(prompt, placeholders_dict, placeholders_meta, options) - # Log AI response for debugging - try: - from modules.shared.debugLogger import writeDebugFile - writeDebugFile(str(result or ""), "taskplanResponse", documents) - except Exception: - pass - return result + # Handle text calls (no output format specified) + if documents and documentProcessor: + # Use document processing for text calls with documents + result = await documentProcessor.callAiText(prompt, documents, options) else: - # Set processDocumentsIndividually from the legacy parameter if not set in options - if options.processDocumentsIndividually is None and documents: - options.processDocumentsIndividually = False # Default to batch processing - - # For text calls, we need to build the full prompt with placeholders here - # since _callAiText doesn't handle placeholders directly - if placeholders_dict: - full_prompt = self._buildPromptWithPlaceholders(prompt, placeholders_dict) - else: - full_prompt = prompt - - if documentProcessor and documents: - result = await documentProcessor.callAiText(full_prompt, documents, options) - else: - # Enhanced direct AI call with partial results support - result = await self._callAiWithPartialResults(full_prompt, options) - - # Log AI response for debugging (additional logging for text calls) - try: - from modules.shared.debugLogger import writeDebugFile - writeDebugFile(str(result or ""), "aiTextResponse", documents) - except Exception: - pass + # Use shared core function for direct text calls + result = await self._callAiWithLooping(prompt, options, "text") + return result + # AI Image Analysis async def readImage( self, @@ -312,382 +444,14 @@ class SubCoreAi: else: return "text" - async def _callAiPlanning( - self, - prompt: str, - placeholders: Optional[Dict[str, str]], - placeholdersMeta: Optional[Dict[str, bool]], - options: AiCallOptions - ) -> str: - """ - Handle planning calls with placeholder system and selective summarization. - """ - # Build full prompt with placeholders; if too large, summarize summaryAllowed placeholders proportionally - effective_placeholders = placeholders or {} - full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders) - if options.compressPrompt and placeholdersMeta: - # Determine model capacity - try: - caps = self._getModelCapabilitiesForContent(full_prompt, None, options) - max_bytes = caps.get("maxContextBytes", len(full_prompt.encode("utf-8"))) - except Exception: - max_bytes = len(full_prompt.encode("utf-8")) - current_bytes = len(full_prompt.encode("utf-8")) - if current_bytes > max_bytes: - # Compute total bytes contributed by allowed placeholders (approximate by content length) - allowed_labels = [l for l, allow in placeholdersMeta.items() if allow] - allowed_sizes = {l: len((effective_placeholders.get(l) or "").encode("utf-8")) for l in allowed_labels} - total_allowed = sum(allowed_sizes.values()) - overage = current_bytes - max_bytes - if total_allowed > 0 and overage > 0: - # Target total for allowed after reduction - target_allowed = max(total_allowed - overage, 0) - # Global ratio to apply across allowed placeholders - ratio = target_allowed / total_allowed if total_allowed > 0 else 1.0 - ratio = max(0.0, min(1.0, ratio)) - reduced: Dict[str, str] = {} - for label, content in effective_placeholders.items(): - if label in allowed_labels and isinstance(content, str) and len(content) > 0: - old_len = len(content) - # Reduce by proportional ratio on characters (fallback if empty) - reduction_factor = ratio if old_len > 0 else 1.0 - reduced[label] = self._reduceText(content, reduction_factor) - else: - reduced[label] = content - effective_placeholders = reduced - full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders) - # If still slightly over, perform a second-pass fine adjustment with updated ratio - current_bytes = len(full_prompt.encode("utf-8")) - if current_bytes > max_bytes and total_allowed > 0: - overage2 = current_bytes - max_bytes - # Recompute allowed sizes after first reduction - allowed_sizes2 = {l: len((effective_placeholders.get(l) or "").encode("utf-8")) for l in allowed_labels} - total_allowed2 = sum(allowed_sizes2.values()) - if total_allowed2 > 0 and overage2 > 0: - target_allowed2 = max(total_allowed2 - overage2, 0) - ratio2 = target_allowed2 / total_allowed2 - ratio2 = max(0.0, min(1.0, ratio2)) - reduced2: Dict[str, str] = {} - for label, content in effective_placeholders.items(): - if label in allowed_labels and isinstance(content, str) and len(content) > 0: - old_len = len(content) - reduction_factor = ratio2 if old_len > 0 else 1.0 - reduced2[label] = self._reduceText(content, reduction_factor) - else: - reduced2[label] = content - effective_placeholders = reduced2 - full_prompt = self._buildPromptWithPlaceholders(prompt, effective_placeholders) - - - # Make AI call using AiObjects (let it handle model selection) - request = AiCallRequest( - prompt=full_prompt, - context="", # Context is already included in the prompt - options=options - ) - response = await self.aiObjects.call(request) - try: - logger.debug(f"AI model selected (planning): {getattr(response, 'modelName', 'unknown')}") - except Exception: - pass - return response.content - async def _callAiWithPartialResults( - self, - prompt: str, - options: AiCallOptions - ) -> str: - """ - Call AI with partial results continuation logic for direct calls. - Handles cases where AI needs to generate large responses in chunks. - """ - logger.info("Starting direct AI call with partial results support") - - # Build enhanced prompt with continuation instructions - enhanced_prompt = self._buildDirectContinuationPrompt(prompt) - - # Process with continuation logic - return await self._processDirectWithContinuationLoop(enhanced_prompt, options) - def _buildDirectContinuationPrompt(self, base_prompt: str) -> str: - """ - Build a prompt for direct AI calls that includes partial results instructions. - """ - continuation_instructions = """ - -IMPORTANT: If your response is too large to generate completely in one response, you can deliver partial results and continue. - -CONTINUATION LOGIC: -- If you cannot complete the full response, end your response with: - [CONTINUE: brief description of what still needs to be generated] -- The system will call you again to continue from where you left off -- Continue generating from the exact point where you stopped -- Maintain consistency with your previous partial response -- Only stop when you have generated the complete response - -Examples: - -Example - Code Generation: -If generating a large code file and you can only generate part of it: -- Generate the first part (imports, classes, functions) -- End with: [CONTINUE: Generate the remaining methods and main execution code] -- In the next call, continue from where you left off - -Example - Documentation: -If writing comprehensive documentation and you can only generate sections 1-3: -- Generate sections 1-3 with full content -- End with: [CONTINUE: Generate sections 4-8 covering advanced topics and examples] -- In the next call, continue with sections 4-8 - -This allows you to handle very large responses that exceed normal limits. -""" - - return f"{base_prompt}{continuation_instructions}" - - async def _processDirectWithContinuationLoop( - self, - enhanced_prompt: str, - options: AiCallOptions - ) -> str: - """ - Process direct AI call with continuation loop until complete. - """ - max_iterations = 10 # Prevent infinite loops - iteration = 0 - accumulated_content = [] - continuation_hint = None - - while iteration < max_iterations: - iteration += 1 - logger.info(f"Direct AI continuation iteration {iteration}/{max_iterations}") - - # Build prompt for this iteration - if continuation_hint: - iteration_prompt = self._buildDirectContinuationIterationPrompt( - enhanced_prompt, continuation_hint, accumulated_content - ) - else: - iteration_prompt = enhanced_prompt - - # Make AI call for this iteration - try: - request = AiCallRequest( - prompt=iteration_prompt, - context="", - options=options - ) - response = await self.aiObjects.call(request) - result = response.content - - # Emit stats for this iteration - self.services.workflow.storeWorkflowStat( - self.services.currentWorkflow, - response, - f"ai.call.{options.operationType}.iteration_{iteration}" - ) - - if not result or not result.strip(): - logger.warning(f"Iteration {iteration}: Empty response, stopping") - break - - # Check for continuation marker - if "[CONTINUE:" in result: - # Extract the continuation hint - import re - continue_match = re.search(r'\[CONTINUE:\s*([^\]]+)\]', result) - if continue_match: - continuation_hint = continue_match.group(1).strip() - # Remove the continuation marker from the result - result = re.sub(r'\s*\[CONTINUE:[^\]]+\]', '', result).strip() - else: - continuation_hint = "Continue from where you left off" - - # Add this partial result to accumulated content - if result.strip(): - accumulated_content.append(result.strip()) - - logger.info(f"Iteration {iteration}: Partial result added, continue hint: {continuation_hint}") - else: - # No continuation marker - this is the final result - if result.strip(): - accumulated_content.append(result.strip()) - - logger.info(f"Direct AI continuation complete after {iteration} iterations") - break - - except Exception as e: - logger.error(f"Direct AI iteration {iteration} failed: {str(e)}") - break - - if iteration >= max_iterations: - logger.warning(f"Direct AI continuation stopped after maximum iterations ({max_iterations})") - - # For JSON responses, we need to merge them properly instead of concatenating - if accumulated_content: - import json - # Parse each part as JSON and merge them - merged_documents = [] - merged_metadata = None - - for content in accumulated_content: - parsed = json.loads(content) - if isinstance(parsed, dict): - # Extract metadata from first valid JSON - if merged_metadata is None and "metadata" in parsed: - merged_metadata = parsed["metadata"] - - # Extract documents from this part - if "documents" in parsed and isinstance(parsed["documents"], list): - merged_documents.extend(parsed["documents"]) - - # Create final merged JSON - NO FALLBACK - final_result = json.dumps({ - "metadata": merged_metadata or { - "title": "Generated Document", - "splitStrategy": "single_document", - "source_documents": [], - "extraction_method": "ai_generation" - }, - "documents": merged_documents - }, indent=2) - else: - # Return empty JSON structure if no content - final_result = json.dumps({ - "metadata": { - "title": "Generated Document", - "splitStrategy": "single_document", - "source_documents": [], - "extraction_method": "ai_generation" - }, - "documents": [] - }, indent=2) - - logger.info(f"Final direct AI result: {len(accumulated_content)} parts from {iteration} iterations") - return final_result - - def _buildDirectContinuationIterationPrompt( - self, - base_prompt: str, - continuation_hint: str, - accumulated_content: List[str] - ) -> str: - """ - Build a prompt for continuation iteration with context. - """ - # Build context of what's already been generated - context_summary = "PREVIOUSLY GENERATED CONTENT:\n" - for i, content in enumerate(accumulated_content[-2:]): # Show last 2 parts for context - preview = content[:200] + "..." if len(content) > 200 else content - context_summary += f"Part {i+1}: {preview}\n" - - continuation_prompt = f""" -{base_prompt} - -{context_summary} - -CONTINUATION INSTRUCTIONS: -- Continue from where you left off -- Continuation hint: {continuation_hint} -- Generate the next part of the content -- Maintain consistency with previously generated content -- End with [CONTINUE: description] if more content is needed -- End without [CONTINUE] if the response is complete -""" - - return continuation_prompt - - async def _callAiUnifiedGeneration( - self, - prompt: str, - extracted_content: Optional[str] = None, - options: Optional[AiCallOptions] = None, - outputFormat: str = "json", - title: str = "Generated Document" - ) -> str: - """ - Unified generation method that handles both scenarios: - - With extracted content (from documents) - - Without extracted content (direct generation) - - Always uses continuation logic for long responses. - Always returns standardized JSON format using the multi-document schema. - """ - if options is None: - options = AiCallOptions() - - logger.info("Starting unified AI generation with continuation logic") - - # Use the existing buildGenerationPrompt to get the proper canonical format instructions - from modules.services.serviceGeneration.subPromptBuilder import buildGenerationPrompt - - # Build the generation prompt using the existing system - generation_prompt = await buildGenerationPrompt( - outputFormat=outputFormat, - userPrompt=prompt, - title=title, - aiService=self, - services=self.services - ) - - # If we have extracted content, prepend it to the prompt - if extracted_content: - generation_prompt = f"""EXTRACTED CONTENT FROM DOCUMENTS: -{extracted_content} - -{generation_prompt}""" - - # Use continuation logic for long responses - return await self._processDirectWithContinuationLoop(generation_prompt, options) - - async def _callAiDirect( - self, - prompt: str, - documents: Optional[List[ChatDocument]], - options: AiCallOptions, - documentProcessor=None - ) -> Dict[str, Any]: - """ - Call AI directly with prompt and documents for JSON output. - Used for multi-file generation - uses the existing generation pipeline. - """ - # Use the existing generation pipeline that already works - # This ensures proper document processing and content extraction - logger.info(f"Using existing generation pipeline for {len(documents) if documents else 0} documents") - - if documentProcessor: - # Process documents with JSON merging using the existing pipeline - result = await documentProcessor.processDocumentsPerChunkJson(documents, prompt, options) - else: - # Fallback to simple AI call - request = AiCallRequest( - prompt=prompt, - context="", - options=options - ) - response = await self.aiObjects.call(request) - result = {"metadata": {"title": "AI Response"}, "sections": [{"id": "section_1", "content_type": "paragraph", "elements": [{"text": response.content}]}]} - - # Convert single-file result to multi-file format if needed - if "sections" in result and "documents" not in result: - logger.info("Converting single-file result to multi-file format") - # This is a single-file result, convert it to multi-file format - return { - "metadata": result.get("metadata", {"title": "Converted Document"}), - "documents": [{ - "id": "doc_1", - "title": result.get("metadata", {}).get("title", "Document"), - "filename": "document.txt", - "sections": result.get("sections", []) - }] - } - - return result def _getModelCapabilitiesForContent(self, prompt: str, documents: Optional[List[ChatDocument]], options: AiCallOptions) -> Dict[str, int]: """ diff --git a/modules/services/serviceAi/subDocumentGeneration.py b/modules/services/serviceAi/subDocumentGeneration.py index a4eded1d..d11a1122 100644 --- a/modules/services/serviceAi/subDocumentGeneration.py +++ b/modules/services/serviceAi/subDocumentGeneration.py @@ -100,6 +100,10 @@ class SubDocumentGeneration: # Update progress - generating extraction prompt progressLogger.updateProgress(operationId, 0.1, "Generating prompt") + + # Write prompt to debug file + from modules.shared.debugLogger import writeDebugFile + writeDebugFile(extractionPrompt, "extraction_prompt", documents) # Process with unified JSON pipeline using continuation logic aiResponse = await self.documentProcessor.processDocumentsWithContinuation( @@ -109,11 +113,13 @@ class SubDocumentGeneration: # Update progress - AI processing completed progressLogger.updateProgress(operationId, 0.6, "Processing done") - # Log the AI response for debugging - logger.info(f"AI response received for validation:") - logger.info(f" - Type: {type(aiResponse)}") - logger.info(f" - Keys: {list(aiResponse.keys()) if isinstance(aiResponse, dict) else 'Not a dict'}") - logger.info(f" - Content: {aiResponse}") + + + # Write AI response to debug file + from modules.shared.debugLogger import writeDebugFile + import json + response_json = json.dumps(aiResponse, indent=2, ensure_ascii=False) if isinstance(aiResponse, dict) else str(aiResponse) + writeDebugFile(response_json, "ai_response", documents) # Validate response structure if not self._validateUnifiedResponseStructure(aiResponse): diff --git a/modules/services/serviceAi/subDocumentProcessing.py b/modules/services/serviceAi/subDocumentProcessing.py index 9757350a..8fe3714a 100644 --- a/modules/services/serviceAi/subDocumentProcessing.py +++ b/modules/services/serviceAi/subDocumentProcessing.py @@ -605,7 +605,13 @@ CONTINUATION INSTRUCTIONS: import os from datetime import datetime, UTC ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") - debug_root = "./test-chat/ai" + # Use configured log directory instead of hardcoded test-chat + from modules.shared.configuration import APP_CONFIG + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + debug_root = os.path.join(logDir, 'debug') os.makedirs(debug_root, exist_ok=True) with open(os.path.join(debug_root, f"{ts}_extraction_image_chunk_{chunk_index}.txt"), "w", encoding="utf-8") as f: f.write(f"EXTRACTION IMAGE RESPONSE:\n{ai_result if ai_result else 'No response'}\n") diff --git a/modules/services/serviceAi/subUtilities.py b/modules/services/serviceAi/subUtilities.py index 0f5bcc4d..64508d71 100644 --- a/modules/services/serviceAi/subUtilities.py +++ b/modules/services/serviceAi/subUtilities.py @@ -61,7 +61,7 @@ class SubUtilities: pass def _writeAiResponseDebug(self, label: str, content: str, partIndex: int = 1, modelName: str = None, continuation: bool = None) -> None: - """Persist raw AI response parts for debugging under test-chat/ai - only if debug enabled.""" + """Persist raw AI response parts for debugging under configured log directory - only if debug enabled.""" try: # Check if debug logging is enabled debug_enabled = self.services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False) @@ -70,10 +70,13 @@ class SubUtilities: import os from datetime import datetime, UTC - # Base dir: gateway/test-chat/ai (go up 4 levels from this file) - # .../gateway/modules/services/serviceAi/subUtilities.py -> up to gateway root - gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - outDir = os.path.join(gatewayDir, 'test-chat', 'ai') + # Use configured log directory instead of hardcoded test-chat + from modules.shared.configuration import APP_CONFIG + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + logDir = os.path.join(gatewayDir, logDir) + outDir = os.path.join(logDir, 'debug') os.makedirs(outDir, exist_ok=True) ts = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] suffix = [] diff --git a/modules/services/serviceGeneration/subPromptBuilder.py b/modules/services/serviceGeneration/subPromptBuilder.py index bfb4052f..c2ba3c3e 100644 --- a/modules/services/serviceGeneration/subPromptBuilder.py +++ b/modules/services/serviceGeneration/subPromptBuilder.py @@ -403,7 +403,13 @@ DO NOT return a schema description - return actual extracted content in the JSON import os from datetime import datetime, UTC ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") - debug_root = "./test-chat/ai" + # Use configured log directory instead of hardcoded test-chat + from modules.shared.configuration import APP_CONFIG + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + debug_root = os.path.join(logDir, 'debug') os.makedirs(debug_root, exist_ok=True) with open(os.path.join(debug_root, f"{ts}_extraction_prompt.txt"), "w", encoding="utf-8") as f: f.write(finalPrompt) @@ -435,118 +441,70 @@ async def buildGenerationPrompt( # Debug output services.utils.debugLogToFile(f"GENERATION PROMPT REQUEST: buildGenerationPrompt called with outputFormat='{outputFormat}', title='{title}'", "PROMPT_BUILDER") - # AI call to generate the appropriate generation prompt - generationPromptRequest = f""" -You are creating instructions for an AI to generate JSON content in the CANONICAL FORMAT that will be converted to a {outputFormat} document. + # Return static generation prompt template instead of calling AI + services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Using static template instead of AI call", "PROMPT_BUILDER") + + # Return static generation prompt template + result = f"""You are an AI assistant that generates structured JSON content for document creation. -User request: "{safeUserPrompt}" -Document title: "{title}" -Target format: {outputFormat} +USER REQUEST: "{safeUserPrompt}" +DOCUMENT TITLE: "{title}" +TARGET FORMAT: {outputFormat} -Write clear, detailed instructions that tell the AI how to generate JSON content using the CANONICAL JSON FORMAT. Focus on: +TASK: Generate JSON content that fulfills the user's request. -1. What content is most important for the user -2. How to structure and organize the content using the canonical JSON format with 'sections' -3. Specific formatting requirements for the target format -4. Language requirements to preserve -5. How to ensure the JSON content meets the user's needs - -CRITICAL: The AI MUST generate content using the CANONICAL JSON FORMAT with this exact structure: +CRITICAL: You MUST return ONLY valid JSON in this exact structure: {{ "metadata": {{ - "title": "Document Title" + "title": "{title}", + "splitStrategy": "single_document", + "source_documents": [], + "extraction_method": "ai_generation" }}, - "sections": [ + "documents": [ {{ - "id": "section_1", - "content_type": "heading", - "elements": [ + "id": "doc_1", + "title": "{title}", + "filename": "document.{outputFormat}", + "sections": [ {{ - "level": 1, - "text": "1. SECTION TITLE" - }} - ], - "order": 1 - }}, - {{ - "id": "section_2", - "content_type": "paragraph", - "elements": [ + "id": "section_1", + "content_type": "heading", + "elements": [ + {{ + "level": 1, + "text": "1. SECTION TITLE" + }} + ], + "order": 1 + }}, {{ - "text": "This is the actual content that should be extracted from the document." + "id": "section_2", + "content_type": "paragraph", + "elements": [ + {{ + "text": "This is the actual content that should be generated." + }} + ], + "order": 2 }} - ], - "order": 2 - }}, - {{ - "id": "section_3", - "content_type": "table", - "elements": [ - {{ - "headers": ["Column 1", "Column 2", "Column 3"], - "rows": [ - ["Value 1", "Value 2", "Value 3"], - ["Value 4", "Value 5", "Value 6"] - ] - }} - ], - "order": 3 + ] }} - ], - "continue": false + ] }} -IMPORTANT CHUNKING LOGIC: -- If the document is too large to generate completely in one response, set "continue": true -- When "continue": true, include a "continuation_context" field with: - - "last_section_id": "id of the last completed section" - - "last_element_index": "index of the last completed element in that section" - - "remaining_requirements": "brief description of what still needs to be generated" -- The AI will be called again with this context to continue generation -- Only set "continue": false when the document is completely generated - -The AI should NOT create format-specific structures like "sheets" or "columns" - only use the canonical format with "sections" and "elements". - -Write the instructions as plain text, not JSON. Start with "Generate JSON content that..." and provide clear, actionable instructions for creating structured JSON data in the canonical format. +IMPORTANT: +- Return ONLY the JSON structure above +- Do NOT include any text before or after the JSON +- Fill in the actual content based on the user request: {safeUserPrompt} +- If the content is too large, you can split it into multiple sections +- Each section should have a unique id and appropriate content_type """ - # Call AI service to generate the prompt - services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Calling AI for generation prompt...", "PROMPT_BUILDER") - - # Import and set proper options for AI call - from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType - request_options = AiCallOptions() - request_options.operationType = OperationType.GENERAL - - request = AiCallRequest(prompt=generationPromptRequest, context="", options=request_options) - response = await aiService.aiObjects.call(request) - result = response.content if response else "" - - # Replace the placeholder that the AI created with actual format rules - if result: - formatRules = _getFormatRules(outputFormat) - result = result.replace("PLACEHOLDER_FOR_FORMAT_RULES", formatRules) - # Debug output services.utils.debugLogToFile(f"GENERATION PROMPT: Generated successfully", "PROMPT_BUILDER") - # Save full generation prompt and AI response to debug file - only if debug enabled - try: - debug_enabled = services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False) - if debug_enabled: - import os - from datetime import datetime, UTC - ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") - debug_root = "./test-chat/ai" - os.makedirs(debug_root, exist_ok=True) - with open(os.path.join(debug_root, f"{ts}_generation_prompt.txt"), "w", encoding="utf-8") as f: - f.write(f"GENERATION PROMPT REQUEST:\n{generationPromptRequest}\n\n") - f.write(f"GENERATION PROMPT AI RESPONSE:\n{response.content if response else 'No response'}\n\n") - f.write(f"GENERATION PROMPT FINAL:\n{result if result else 'None'}\n") - except Exception: - pass - - return result if result else f"Generate a comprehensive {outputFormat} document titled '{title}' based on the extracted content." + return result.strip() except Exception as e: # Fallback on any error - preserve user prompt for language instructions diff --git a/modules/services/serviceNormalization/mainServiceNormalization.py b/modules/services/serviceNormalization/mainServiceNormalization.py index 34805ef2..4dfbf9cb 100644 --- a/modules/services/serviceNormalization/mainServiceNormalization.py +++ b/modules/services/serviceNormalization/mainServiceNormalization.py @@ -90,7 +90,7 @@ class NormalizationService: " \"Date\": {\"formats\": [\"DD.MM.YYYY\",\"YYYY-MM-DD\"]}\n }\n}\n" ) - response = await self.services.ai.callAi(prompt=prompt) + response = await self.services.ai.coreAi.callAiPlanning(prompt=prompt, placeholders=None, options=None) if not response: return {"mapping": {}, "normalizationPolicy": {}} @@ -244,7 +244,13 @@ class NormalizationService: debugEnabled = self.services.utils.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False) if not debugEnabled: return - root = "./test-chat/ai" + # Use configured log directory instead of hardcoded test-chat + from modules.shared.configuration import APP_CONFIG + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + root = os.path.join(logDir, 'debug') os.makedirs(root, exist_ok=True) # Prefix timestamp for files that are frequently overwritten ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 8379382a..caa07528 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -157,7 +157,12 @@ class UtilsService: return # Get debug directory - debug_dir = self.configGet("APP_DEBUG_CHAT_WORKFLOW_DIR", "./test-chat") + # Use configured log directory instead of hardcoded test-chat + logDir = self.configGet("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + debug_dir = os.path.join(logDir, 'debug') if not os.path.isabs(debug_dir): # If relative path, make it relative to the gateway directory gateway_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/modules/services/serviceWorkflow/mainServiceWorkflow.py b/modules/services/serviceWorkflow/mainServiceWorkflow.py index ff814ac2..1edafafa 100644 --- a/modules/services/serviceWorkflow/mainServiceWorkflow.py +++ b/modules/services/serviceWorkflow/mainServiceWorkflow.py @@ -59,7 +59,7 @@ class WorkflowService: # Get summary using AI service directly (avoiding circular dependency) ai_service = AiService(self) - return await ai_service.callAi( + return await ai_service.coreAi.callAiDocuments( prompt=prompt, documents=None, options={ @@ -69,7 +69,9 @@ class WorkflowService: "compress_prompt": True, "compress_documents": False, "max_cost": 0.01 - } + }, + documentProcessor=ai_service.documentProcessor, + documentGenerator=ai_service.documentGenerator ) except Exception as e: diff --git a/modules/shared/debugLogger.py b/modules/shared/debugLogger.py index 82dcb1c9..947bf816 100644 --- a/modules/shared/debugLogger.py +++ b/modules/shared/debugLogger.py @@ -1,16 +1,25 @@ """ Simple debug logger for AI prompts and responses. -Writes files chronologically to gateway/test-chat/ai/ with sequential numbering. +Writes files chronologically to the configured log directory with sequential numbering. """ import os from datetime import datetime, UTC from typing import List, Optional +from modules.shared.configuration import APP_CONFIG def _getDebugDir() -> str: - """Get the debug directory path.""" - gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - return os.path.join(gatewayDir, 'test-chat', 'ai') + """Get the debug directory path from configuration.""" + # Get log directory from config (same as used by main logging system) + logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./") + if not os.path.isabs(logDir): + # If relative path, make it relative to the gateway directory + gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + logDir = os.path.join(gatewayDir, logDir) + + # Create debug subdirectory within the log directory + debugDir = os.path.join(logDir, 'debug') + return debugDir def _getNextSequenceNumber() -> int: diff --git a/modules/workflows/methods/methodAi.py b/modules/workflows/methods/methodAi.py index c820114e..e10f7fe9 100644 --- a/modules/workflows/methods/methodAi.py +++ b/modules/workflows/methods/methodAi.py @@ -106,23 +106,8 @@ class MethodAi(MethodBase): if chatDocuments: logger.info(f"Prepared {len(chatDocuments)} documents for AI processing") - # Update progress - building prompt - progressLogger.updateProgress(operationId, 0.4, "Building prompt") - - # Build enhanced prompt - enhanced_prompt = aiPrompt - - # Add processing mode instructions if specified (generic, not analysis-specific) - if processingMode == "detailed": - enhanced_prompt += "\n\nPlease provide a detailed response with comprehensive information." - elif processingMode == "advanced": - enhanced_prompt += "\n\nPlease provide an advanced response with deep insights." - - # Note: customInstructions parameter was removed as it's not defined in the method signature - - # Add format guidance to prompt - if normalized_result_type != "txt": - enhanced_prompt += f"\n\nPlease deliver the result in {normalized_result_type.upper()} format. Ensure the output follows the proper {normalized_result_type.upper()} syntax and structure." + # Update progress - preparing AI call + progressLogger.updateProgress(operationId, 0.4, "Preparing AI call") # Build options and delegate document handling to AI/Extraction/Generation services output_format = output_extension.replace('.', '') or 'txt' @@ -139,17 +124,16 @@ class MethodAi(MethodBase): requiredTags=requiredTags ) - supported_generation_formats = {"html", "pdf", "docx", "txt", "md", "json", "csv", "xlsx"} - output_format_arg = output_format if output_format in supported_generation_formats else None - # Update progress - calling AI progressLogger.updateProgress(operationId, 0.6, "Calling AI") - result = await self.services.ai.callAi( - prompt=enhanced_prompt, + result = await self.services.ai.coreAi.callAiDocuments( + prompt=aiPrompt, # Use original prompt, let unified generation handle prompt building documents=chatDocuments if chatDocuments else None, options=options, - outputFormat=output_format_arg + outputFormat=output_format, + documentProcessor=self.services.ai.documentProcessor, + documentGenerator=self.services.ai.documentGenerator ) # Update progress - processing result diff --git a/modules/workflows/methods/methodOutlook.py b/modules/workflows/methods/methodOutlook.py index 9909bd9f..a4949a0d 100644 --- a/modules/workflows/methods/methodOutlook.py +++ b/modules/workflows/methods/methodOutlook.py @@ -1186,7 +1186,7 @@ Return JSON: # Call AI service to generate email content try: - ai_response = await self.services.ai.callAi( + ai_response = await self.services.ai.coreAi.callAiDocuments( prompt=ai_prompt, documents=chatDocuments, options=AiCallOptions( @@ -1199,7 +1199,9 @@ Return JSON: resultFormat="json", maxCost=0.50, maxProcessingTime=30 - ) + ), + documentProcessor=self.services.ai.documentProcessor, + documentGenerator=self.services.ai.documentGenerator ) # Parse AI response diff --git a/modules/workflows/processing/adaptive/contentValidator.py b/modules/workflows/processing/adaptive/contentValidator.py index 91896373..156dd2c9 100644 --- a/modules/workflows/processing/adaptive/contentValidator.py +++ b/modules/workflows/processing/adaptive/contentValidator.py @@ -120,9 +120,9 @@ DELIVERED CONTENT TO CHECK: request_options = AiCallOptions() request_options.operationType = OperationType.GENERAL - response = await self.services.ai.callAi( + response = await self.services.ai.coreAi.callAiPlanning( prompt=validationPrompt, - documents=None, + placeholders=None, options=request_options ) diff --git a/modules/workflows/processing/adaptive/intentAnalyzer.py b/modules/workflows/processing/adaptive/intentAnalyzer.py index 7d21b8d1..74283629 100644 --- a/modules/workflows/processing/adaptive/intentAnalyzer.py +++ b/modules/workflows/processing/adaptive/intentAnalyzer.py @@ -63,9 +63,9 @@ CRITICAL: Respond with ONLY the JSON object below. Do not include any explanator request_options = AiCallOptions() request_options.operationType = OperationType.GENERAL - response = await self.services.ai.callAi( + response = await self.services.ai.coreAi.callAiPlanning( prompt=analysisPrompt, - documents=None, + placeholders=None, options=request_options ) diff --git a/modules/workflows/processing/core/taskPlanner.py b/modules/workflows/processing/core/taskPlanner.py index 3bea9c76..361c86b2 100644 --- a/modules/workflows/processing/core/taskPlanner.py +++ b/modules/workflows/processing/core/taskPlanner.py @@ -105,7 +105,7 @@ class TaskPlanner: maxProcessingTime=30 ) - prompt = await self.services.ai.callAi( + prompt = await self.services.ai.coreAi.callAiPlanning( prompt=taskPlanningPromptTemplate, placeholders=placeholders, options=options diff --git a/modules/workflows/processing/modes/modeActionplan.py b/modules/workflows/processing/modes/modeActionplan.py index c1521a71..0e11ac88 100644 --- a/modules/workflows/processing/modes/modeActionplan.py +++ b/modules/workflows/processing/modes/modeActionplan.py @@ -137,7 +137,7 @@ class ActionplanMode(BaseMode): maxProcessingTime=30 ) - prompt = await self.services.ai.callAi(prompt=actionPromptTemplate, placeholders=placeholders, options=options) + prompt = await self.services.ai.coreAi.callAiPlanning(prompt=actionPromptTemplate, placeholders=placeholders, options=options) # Check if AI response is valid if not prompt: @@ -476,7 +476,7 @@ class ActionplanMode(BaseMode): maxProcessingTime=30 ) - response = await self.services.ai.callAi(prompt=promptTemplate, placeholders=placeholders, options=options) + response = await self.services.ai.coreAi.callAiPlanning(prompt=promptTemplate, placeholders=placeholders, options=options) # Log result review response received logger.info("=== RESULT REVIEW AI RESPONSE RECEIVED ===") diff --git a/modules/workflows/processing/modes/modeReact.py b/modules/workflows/processing/modes/modeReact.py index cbe29eee..405f530e 100644 --- a/modules/workflows/processing/modes/modeReact.py +++ b/modules/workflows/processing/modes/modeReact.py @@ -201,7 +201,7 @@ class ReactMode(BaseMode): maxProcessingTime=30 ) - response = await self.services.ai.callAi( + response = await self.services.ai.coreAi.callAiPlanning( prompt=promptTemplate, placeholders=placeholders, options=options @@ -313,7 +313,7 @@ class ReactMode(BaseMode): resultFormat="json" # Explicitly request JSON format ) - paramsResp = await self.services.ai.callAi( + paramsResp = await self.services.ai.coreAi.callAiPlanning( prompt=promptTemplate, placeholders=placeholders, options=options @@ -625,7 +625,7 @@ class ReactMode(BaseMode): maxProcessingTime=30 ) - resp = await self.services.ai.callAi( + resp = await self.services.ai.coreAi.callAiPlanning( prompt=promptTemplate, placeholders=placeholders, options=options @@ -719,8 +719,9 @@ User language: {userLanguage} Return only the user-friendly message, no technical details.""" # Call AI to generate user-friendly message - response = await self.services.ai.callAi( + response = await self.services.ai.coreAi.callAiPlanning( prompt=prompt, + placeholders=None, options=AiCallOptions( operationType=OperationType.GENERATE_CONTENT, priority=Priority.SPEED, @@ -759,8 +760,9 @@ Result context: {resultContext} Return only the user-friendly message, no technical details.""" # Call AI to generate user-friendly result message - response = await self.services.ai.callAi( + response = await self.services.ai.coreAi.callAiPlanning( prompt=prompt, + placeholders=None, options=AiCallOptions( operationType=OperationType.GENERATE_CONTENT, priority=Priority.SPEED, diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index cb3a09b5..1dfebd77 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -220,7 +220,7 @@ class WorkflowManager: ) # Call AI analyzer - aiResponse = await self.services.ai.callAi(prompt=analyzerPrompt) + aiResponse = await self.services.ai.coreAi.callAiPlanning(prompt=analyzerPrompt, placeholders=None, options=None) detectedLanguage = None normalizedRequest = None diff --git a/test_unified_architecture.py b/test_unified_architecture.py deleted file mode 100644 index bf0e0750..00000000 --- a/test_unified_architecture.py +++ /dev/null @@ -1,258 +0,0 @@ -import asyncio -import sys -import os -from unittest.mock import AsyncMock, MagicMock - -# Add the project root to the sys.path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) - -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType -from modules.datamodels.datamodelChat import ChatDocument -from modules.services.serviceAi.subCoreAi import SubCoreAi - -class MockAiObjects: - def __init__(self, responses): - self.responses = responses - self.call_count = 0 - - async def call(self, request: AiCallRequest): - if self.call_count < len(self.responses): - response_content = self.responses[self.call_count] - self.call_count += 1 - mock_response = MagicMock() - mock_response.content = response_content - mock_response.modelName = "mock-model" - mock_response.priceUsd = 0.001 - mock_response.processingTime = 0.1 - print(f" Mock AI Call {self.call_count}: Responding with partial result (length: {len(response_content)})") - return mock_response - else: - print(" Mock AI Call: No more mock responses, returning empty.") - mock_response = MagicMock() - mock_response.content = "" - return mock_response - -class MockServices: - def __init__(self): - self.currentWorkflow = MagicMock() - self.currentWorkflow.id = "test_workflow_123" - self.workflow = MagicMock() - self.workflow.createProgressLogger.return_value = MagicMock() - self.workflow.storeWorkflowStat = AsyncMock() - self.ai = MagicMock() - self.ai.sanitizePromptContent.side_effect = lambda content, type: content - self.utils = MagicMock() - self.utils.debugLogToFile.side_effect = lambda msg, tag: print(f" DEBUG ({tag}): {msg}") - self.utils.configGet.return_value = False # Disable debug files for tests - -class MockDocumentProcessor: - async def callAiText(self, prompt, documents, options): - return "Extracted content from documents: Sample text content" - -async def test_unified_architecture(): - print("\n=== Testing Unified Architecture ===") - - # Mock responses: 1 for generation prompt building + 2 for actual generation - mock_responses = [ - # Response 1: Generation prompt building - "Generate JSON content that creates a structured document with prime numbers in a table format. Use the canonical JSON format with sections and elements.", - - # Response 2: First part of generation - """{ - "metadata": { - "title": "Prime Numbers List", - "splitStrategy": "single_document", - "source_documents": [], - "extraction_method": "ai_generation" - }, - "documents": [ - { - "id": "doc_primes_1_500", - "title": "Prime Numbers 1-500", - "filename": "primes_1_500.docx", - "sections": [ - { - "id": "section_1", - "content_type": "table", - "elements": [ - { - "headers": ["Number", "Prime"], - "rows": [ - ["1", "2"], ["2", "3"], ["3", "5"], ["4", "7"], ["5", "11"] - ] - } - ], - "order": 1 - } - ] - } - ] -} [CONTINUE: Generate remaining prime numbers from 501 to 1000]""", - - # Response 3: Second part of generation - """{ - "metadata": { - "title": "Prime Numbers List", - "splitStrategy": "single_document", - "source_documents": [], - "extraction_method": "ai_generation" - }, - "documents": [ - { - "id": "doc_primes_501_1000", - "title": "Prime Numbers 501-1000", - "filename": "primes_501_1000.docx", - "sections": [ - { - "id": "section_2", - "content_type": "table", - "elements": [ - { - "headers": ["Number", "Prime"], - "rows": [ - ["501", "3571"], ["502", "3572"], ["503", "3581"] - ] - } - ], - "order": 2 - } - ] - } - ] -}""" - ] - - mock_ai_objects = MockAiObjects(mock_responses) - mock_services = MockServices() - mock_document_processor = MockDocumentProcessor() - - core_ai_service = SubCoreAi(mock_services, mock_ai_objects) - - prompt = "Generate the first 1000 prime numbers and arrange them in a structured table format." - options = AiCallOptions(operationType=OperationType.GENERATE_CONTENT) - output_format = "docx" - title = "Prime Numbers List" - - print(f"User Prompt: '{prompt}'") - print("Testing unified architecture with direct generation (no documents)...") - - # Test the unified generation method directly - result = await core_ai_service._callAiUnifiedGeneration(prompt, None, options, output_format, title) - - print("\n--- Generated JSON Result ---") - print(f"Result length: {len(result)} characters") - print(f"Result preview: {result[:300]}...") - - # Verify it's valid JSON - import json - try: - parsed_result = json.loads(result) - print(f"✅ Valid JSON with {len(parsed_result.get('documents', []))} documents") - - # Verify it's using the multi-document format - if "documents" in parsed_result and "metadata" in parsed_result: - print("✅ Using unified multi-document format") - print("✅ Architecture is properly unified!") - return True - else: - print("❌ Not using multi-document format") - return False - except json.JSONDecodeError as e: - print(f"❌ Invalid JSON: {str(e)}") - return False - -async def test_with_documents(): - print("\n=== Testing Unified Architecture WITH Documents ===") - - # Mock responses: 1 for generation prompt building + 1 for actual generation - mock_responses = [ - # Response 1: Generation prompt building - "Generate JSON content that creates a comprehensive fruit analysis report based on the extracted content. Use the canonical JSON format with sections and elements.", - - # Response 2: Generation with extracted content - """{ - "metadata": { - "title": "Fruit Analysis Report", - "splitStrategy": "single_document", - "source_documents": ["doc1"], - "extraction_method": "ai_generation" - }, - "documents": [ - { - "id": "doc_fruit_analysis", - "title": "Fruit Analysis Report", - "filename": "fruit_analysis.docx", - "sections": [ - { - "id": "section_1", - "content_type": "paragraph", - "elements": [ - { - "text": "Based on the extracted content, here is a comprehensive fruit analysis..." - } - ], - "order": 1 - } - ] - } - ] -}""" - ] - - mock_ai_objects = MockAiObjects(mock_responses) - mock_services = MockServices() - mock_document_processor = MockDocumentProcessor() - - core_ai_service = SubCoreAi(mock_services, mock_ai_objects) - - prompt = "Extract all fruit information and create a comprehensive analysis report." - options = AiCallOptions(operationType=OperationType.GENERATE_CONTENT) - output_format = "docx" - title = "Fruit Analysis Report" - - print(f"User Prompt: '{prompt}'") - print("Testing unified architecture with document extraction...") - - # Test the unified generation method with extracted content - result = await core_ai_service._callAiUnifiedGeneration(prompt, "Sample fruit data: apples, oranges, bananas", options, output_format, title) - - print("\n--- Generated JSON Result ---") - print(f"Result length: {len(result)} characters") - print(f"Result preview: {result[:300]}...") - - # Verify it's valid JSON - import json - try: - parsed_result = json.loads(result) - print(f"✅ Valid JSON with {len(parsed_result.get('documents', []))} documents") - - # Verify it's using the multi-document format - if "documents" in parsed_result and "metadata" in parsed_result: - print("✅ Using unified multi-document format") - print("✅ Architecture is properly unified!") - return True - else: - print("❌ Not using multi-document format") - return False - except json.JSONDecodeError as e: - print(f"❌ Invalid JSON: {str(e)}") - return False - -async def main(): - print("🚀 Testing Unified Architecture Implementation") - print("=" * 60) - - success1 = await test_unified_architecture() - success2 = await test_with_documents() - - if success1 and success2: - print("\n🎉 ALL TESTS PASSED! Unified architecture is properly implemented.") - print("✅ Single document = multi-document with n=1") - print("✅ Always uses multi-document JSON format") - print("✅ Continuation logic works for long responses") - print("✅ Both scenarios (with/without documents) work") - else: - print("\n❌ Some tests failed. Please check the implementation.") - -if __name__ == "__main__": - asyncio.run(main())