From c135321aee00ff9d93b2b661c0320624be119373 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 28 Nov 2025 16:57:53 +0100 Subject: [PATCH] fixed documents handling --- AZURE_AD_CONSENT_LINKS.md | 121 -- modules/aicore/aicorePluginAnthropic.py | 2 + modules/datamodels/datamodelChat.py | 1 + modules/services/serviceAi/mainServiceAi.py | 271 +-- .../services/serviceChat/mainServiceChat.py | 45 +- .../renderers/rendererXlsx.py | 24 +- .../subPromptBuilderGeneration.py | 134 +- modules/shared/jsonUtils.py | 1448 ++++++++++------- modules/workflows/methods/methodAi.py | 385 ++++- .../workflows/processing/adaptive/__init__.py | 3 +- .../processing/adaptive/contentValidator.py | 206 ++- .../processing/adaptive/intentAnalyzer.py | 179 -- .../processing/adaptive/learningEngine.py | 57 +- .../processing/adaptive/progressTracker.py | 23 +- .../workflows/processing/core/taskPlanner.py | 17 +- .../workflows/processing/modes/modeDynamic.py | 267 ++- .../shared/promptGenerationActionsDynamic.py | 156 +- .../shared/promptGenerationTaskplan.py | 33 +- .../workflows/processing/workflowProcessor.py | 32 +- modules/workflows/workflowManager.py | 54 +- tests/functional/OPENAI_TIMEOUT_ANALYSIS.md | 219 --- .../test06_workflow_prompt_variations.py | 466 ++++++ tests/functional/test07_json_extraction.py | 517 ++++++ 23 files changed, 2913 insertions(+), 1747 deletions(-) delete mode 100644 AZURE_AD_CONSENT_LINKS.md delete mode 100644 modules/workflows/processing/adaptive/intentAnalyzer.py delete mode 100644 tests/functional/OPENAI_TIMEOUT_ANALYSIS.md create mode 100644 tests/functional/test06_workflow_prompt_variations.py create mode 100644 tests/functional/test07_json_extraction.py diff --git a/AZURE_AD_CONSENT_LINKS.md b/AZURE_AD_CONSENT_LINKS.md deleted file mode 100644 index 45cf6511..00000000 --- a/AZURE_AD_CONSENT_LINKS.md +++ /dev/null @@ -1,121 +0,0 @@ -# Azure AD Consent Links - -## Konfiguration -- **Client ID**: `c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c` -- **Tenant ID**: `common` (Multi-Tenant) -- **Redirect URI (Prod)**: `https://gateway-prod.poweron-center.net/api/msft/auth/callback` -- **Redirect URI (Int)**: `https://gateway-int.poweron-center.net/api/msft/auth/callback` - -## Berechtigungen (Scopes) -- `Mail.ReadWrite` - E-Mails lesen und schreiben -- `Mail.Send` - E-Mails senden -- `Mail.ReadWrite.Shared` - Zugriff auf geteilte Postfächer -- `User.Read` - Benutzerprofil lesen -- `Sites.ReadWrite.All` - Alle SharePoint-Standorte lesen und schreiben -- `Files.ReadWrite.All` - Alle Dateien lesen und schreiben - -## Admin Consent Link (für Tenant-Administrator) - -**WICHTIG:** Der Admin Consent Endpoint gibt `admin_consent` und `tenant` zurück, nicht `code` und `state`. -Der bestehende `/auth/callback` Handler erwartet `code` und `state` für den normalen OAuth-Flow. - -**Option 1: Admin Consent über Azure Portal (für eigenen Tenant)** -1. Gehe zu Azure Portal → Azure Active Directory → App registrations -2. Wähle die App `c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c` -3. Gehe zu "API permissions" -4. Klicke auf "Grant admin consent for [Tenant Name]" - -**Option 1b: App für andere Tenants verfügbar machen** - -Um die App für andere Tenants sichtbar zu machen, müssen folgende Schritte durchgeführt werden: - -1. **Multi-Tenant Konfiguration prüfen:** - - Azure Portal → Azure Active Directory → App registrations - - Wähle die App `c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c` - - Gehe zu "Authentication" - - Stelle sicher, dass "Supported account types" auf **"Accounts in any organizational directory and personal Microsoft accounts"** oder **"Accounts in any organizational directory"** gesetzt ist - -2. **App für andere Tenants verfügbar machen:** - - **Methode A: Direkter Admin Consent Link (empfohlen)** - - Andere Tenant-Administratoren können den Admin Consent Link verwenden: - ``` - https://login.microsoftonline.com/{TENANT_ID}/adminconsent?client_id=c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c&redirect_uri=https://gateway-prod.poweron-center.net/api/msft/adminconsent/callback - ``` - - Ersetze `{TENANT_ID}` durch die Tenant-ID des Ziel-Tenants (oder verwende `common` für Multi-Tenant) - - **Methode B: Manuell über Azure Portal (für andere Tenants)** - - Tenant-Administrator des anderen Tenants: - 1. Gehe zu Azure Portal → Azure Active Directory → Enterprise applications - 2. Klicke auf "+ New application" - 3. Wähle "Browse Azure AD Gallery" (optional) oder "Create your own application" - 4. Wenn nicht in Gallery: Wähle "Non-gallery application" - 5. Gib die Client ID ein: `c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c` - 6. Oder verwende direkt diesen Link: - ``` - https://portal.azure.com/#blade/Microsoft_AAD_IAM/ManagedAppMenuBlade/Overview/objectId/{CLIENT_ID} - ``` - (Ersetze `{CLIENT_ID}` mit `c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c`) - 7. Gehe zu "Permissions" → "Grant admin consent" - - **Methode C: App in Azure AD Gallery veröffentlichen (optional)** - - Für größere Sichtbarkeit kann die App in der Azure AD App Gallery veröffentlicht werden - - Azure Portal → App registrations → App → "Branding & properties" - - Kontaktiere Microsoft für Gallery-Veröffentlichung - -3. **Wichtig für Multi-Tenant Apps:** - - Die Redirect URIs müssen öffentlich erreichbar sein - - Die App muss die richtigen Berechtigungen deklarieren - - Tenant-Administratoren müssen explizit zustimmen (Admin Consent) - -**Option 2: Admin Consent Link (mit Callback-Handler)** -### Production -``` -https://login.microsoftonline.com/common/adminconsent?client_id=c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c&redirect_uri=https://gateway-prod.poweron-center.net/api/msft/adminconsent/callback -``` - -### Integration -``` -https://login.microsoftonline.com/common/adminconsent?client_id=c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c&redirect_uri=https://gateway-int.poweron-center.net/api/msft/adminconsent/callback -``` - -**Hinweis:** Der `/adminconsent/callback` Endpoint ist implementiert und verarbeitet die `admin_consent` und `tenant` Parameter. Nach erfolgreichem Admin Consent wird eine Bestätigungsseite angezeigt. - -## User Consent Link (für einzelne Benutzer) - -### Production -``` -https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c&response_type=code&redirect_uri=https://gateway-prod.poweron-center.net/api/msft/auth/callback&response_mode=query&scope=Mail.ReadWrite Mail.Send Mail.ReadWrite.Shared User.Read Sites.ReadWrite.All Files.ReadWrite.All offline_access openid profile&state=login -``` - -### Integration -``` -https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c&response_type=code&redirect_uri=https://gateway-int.poweron-center.net/api/msft/auth/callback&response_mode=query&scope=Mail.ReadWrite Mail.Send Mail.ReadWrite.Shared User.Read Sites.ReadWrite.All Files.ReadWrite.All offline_access openid profile&state=login -``` - -## Hinweise - -1. **Admin Consent**: Muss von einem Tenant-Administrator durchgeführt werden, um die App für alle Benutzer im Tenant zu genehmigen -2. **User Consent**: Jeder Benutzer kann individuell zustimmen (wenn Admin Consent nicht durchgeführt wurde) -3. **Multi-Tenant**: Da `common` als Tenant verwendet wird, funktioniert die App für alle Azure AD Tenants -4. **Redirect URI**: Muss exakt in der Azure AD App-Registrierung konfiguriert sein - -## Azure Portal Konfiguration - -Stelle sicher, dass in der Azure AD App-Registrierung (`c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c`) folgendes konfiguriert ist: - -1. **Redirect URIs**: - - `https://gateway-prod.poweron-center.net/api/msft/auth/callback` - - `https://gateway-int.poweron-center.net/api/msft/auth/callback` - -2. **API Permissions** (Delegated): - - ✅ Mail.ReadWrite - - ✅ Mail.Send - - ✅ Mail.ReadWrite.Shared - - ✅ User.Read - - ✅ Sites.ReadWrite.All - - ✅ Files.ReadWrite.All - -3. **Supported account types**: - - "Accounts in any organizational directory and personal Microsoft accounts" (Multi-tenant) - diff --git a/modules/aicore/aicorePluginAnthropic.py b/modules/aicore/aicorePluginAnthropic.py index 422056d0..50bcf3ca 100644 --- a/modules/aicore/aicorePluginAnthropic.py +++ b/modules/aicore/aicorePluginAnthropic.py @@ -44,6 +44,8 @@ class AiAnthropic(BaseConnectorAi): return "anthropic" def getModels(self) -> List[AiModel]: + return [] # TODO: DEBUG TO TURN ON AFTER TESTING + """Get all available Anthropic models.""" return [ AiModel( diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 9d75fcd5..01f8c433 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -827,6 +827,7 @@ class TaskContext(BaseModel): parametersContext: Optional[str] = Field(None, description="Context for parameter generation") learnings: Optional[list[str]] = Field(default_factory=list, description="Learnings from previous actions") stage1Selection: Optional[dict] = Field(None, description="Stage 1 selection data") + nextActionGuidance: Optional[Dict[str, Any]] = Field(None, description="Guidance for the next action from previous refinement") def updateFromSelection(self, selection: Any): """Update context from Stage 1 selection diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 2e1f3b3e..9303713f 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -167,8 +167,7 @@ Respond with ONLY a JSON object in this exact format: promptBuilder: Optional[callable] = None, promptArgs: Optional[Dict[str, Any]] = None, operationId: Optional[str] = None, - userPrompt: Optional[str] = None, - workflowIntent: Optional[Dict[str, Any]] = None + userPrompt: Optional[str] = None ) -> str: """ Shared core function for AI calls with repair-based looping system. @@ -212,17 +211,14 @@ Respond with ONLY a JSON object in this exact format: ) # Build iteration prompt - if len(allSections) > 0 and promptBuilder and promptArgs: + # CRITICAL: Build continuation prompt if we have sections OR if we have a previous response (even if broken) + # This ensures continuation prompts are built even when JSON is so broken that no sections can be extracted + if (len(allSections) > 0 or lastRawResponse) and promptBuilder and promptArgs: # This is a continuation - build continuation context with raw JSON and rebuild prompt continuationContext = buildContinuationContext(allSections, lastRawResponse) if not lastRawResponse: logger.warning(f"Iteration {iteration}: No previous response available for continuation!") - # CRITICAL: Add workflowIntent (actionIntent) to continuationContext for DoD-based progress filtering - # This allows buildGenerationPrompt to filter progress stats based on Definition of Done KPIs - if workflowIntent: - continuationContext['taskIntent'] = workflowIntent # Keep key name 'taskIntent' for compatibility - # Filter promptArgs to only include parameters that buildGenerationPrompt accepts # buildGenerationPrompt accepts: outputFormat, userPrompt, title, extracted_content, continuationContext filteredPromptArgs = { @@ -277,14 +273,37 @@ Respond with ONLY a JSON object in this exact format: # Don't break the main loop if stat storage fails logger.warning(f"Failed to store workflow stat: {str(statError)}") + # Check for error response using generic error detection (errorCount > 0 or modelName == "error") + if hasattr(response, 'errorCount') and response.errorCount > 0: + errorMsg = f"Iteration {iteration}: Error response detected (errorCount={response.errorCount}), stopping loop: {result[:200] if result else 'empty'}" + logger.error(errorMsg) + break + + if hasattr(response, 'modelName') and response.modelName == "error": + errorMsg = f"Iteration {iteration}: Error response detected (modelName=error), stopping loop: {result[:200] if result else 'empty'}" + logger.error(errorMsg) + break + if not result or not result.strip(): logger.warning(f"Iteration {iteration}: Empty response, stopping") break + # Check if this is a text response (not document generation) + # Text responses don't need JSON parsing - return immediately after first successful response + isTextResponse = (promptBuilder is None and promptArgs is None) or debugPrefix == "text" + + if isTextResponse: + # For text responses, return the text immediately - no JSON parsing needed + logger.info(f"Iteration {iteration}: Text response received, returning immediately") + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, True) + return result + # Store raw response for continuation (even if broken) lastRawResponse = result # Extract sections from response (handles both valid and broken JSON) + # Only for document generation (JSON responses) extractedSections, wasJsonComplete, parsedResult = self._extractSectionsFromResponse(result, iteration, debugPrefix) # Extract document metadata from first iteration if available @@ -312,25 +331,12 @@ Respond with ONLY a JSON object in this exact format: allSections = self._mergeSectionsIntelligently(allSections, extractedSections, iteration) # Check if we should continue (completion detection) - # Extract user prompt from promptArgs if available - extractedUserPrompt = userPrompt - if not extractedUserPrompt and promptArgs: - extractedUserPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt") - if not extractedUserPrompt: - # Try to extract from original prompt - if "User request:" in prompt: - try: - extractedUserPrompt = prompt.split("User request:")[1].split("\n")[0].strip('"') - except: - pass - + # Simple logic: JSON completeness determines continuation shouldContinue = self._shouldContinueGeneration( allSections, iteration, wasJsonComplete, - result, - userPrompt=extractedUserPrompt, - workflowIntent=workflowIntent + result ) if shouldContinue: @@ -842,39 +848,22 @@ Respond with ONLY a JSON object in this exact format: Determines completion based on JSON structure (complete JSON = complete, broken/incomplete = incomplete). Returns (sections, wasJsonComplete, parsedResult) """ + # First, try to parse as valid JSON + # CRITICAL: JSON completeness is determined by parsing, NOT by last character check! + # Last character could be } or ] by chance, JSON still incomplete try: extracted = extractJsonString(result) - # CRITICAL: Check if raw response suggests incomplete JSON BEFORE parsing - # extractFirstBalancedJson can return partial but valid JSON if raw is incomplete - from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText - raw_normalized = normalizeJsonText(stripCodeFences(result.strip())).strip() - extracted_stripped = extracted.strip() - - # If extracted is shorter than raw, or raw doesn't end properly, it's incomplete - is_raw_incomplete = False - if len(extracted_stripped) < len(raw_normalized): - is_raw_incomplete = True - logger.info(f"Iteration {iteration}: Extracted JSON ({len(extracted_stripped)} chars) shorter than raw ({len(raw_normalized)} chars) - raw is incomplete") - elif raw_normalized and not raw_normalized.endswith(('}', ']')): - is_raw_incomplete = True - logger.info(f"Iteration {iteration}: Raw response doesn't end with }} or ] - raw is incomplete") - + # Try to parse the extracted JSON + # If parsing succeeds, JSON is complete parsed_result = json.loads(extracted) # Extract sections from parsed JSON sections = extractSectionsFromDocument(parsed_result) - # CRITICAL: If raw response is incomplete, mark as incomplete - # JSON structure determines completion, not any flag - if is_raw_incomplete: - logger.info(f"Iteration {iteration}: JSON parseable but raw response incomplete - marking as incomplete") - return sections, False, parsed_result - - # JSON was parseable and has sections or complete structure - # Raw response ends properly = complete - logger.info(f"Iteration {iteration}: JSON parseable and raw response complete - marking as complete") + # JSON parsed successfully = complete + logger.info(f"Iteration {iteration}: JSON parsed successfully - marking as complete") return sections, True, parsed_result except json.JSONDecodeError as e: @@ -906,9 +895,7 @@ Respond with ONLY a JSON object in this exact format: allSections: List[Dict[str, Any]], iteration: int, wasJsonComplete: bool, - rawResponse: str = None, - userPrompt: Optional[str] = None, - workflowIntent: Optional[Dict[str, Any]] = None + rawResponse: str = None ) -> bool: """ Determine if AI generation loop should continue. @@ -917,23 +904,22 @@ Respond with ONLY a JSON object in this exact format: Action DoD is checked AFTER the AI Loop completes in _refineDecide. Simple logic: - - If JSON is incomplete/broken → continue (needs more content) - - If JSON is complete → stop (all content delivered) + - If JSON parsing failed or incomplete → continue (needs more content) + - If JSON parses successfully and is complete → stop (all content delivered) - Loop detection prevents infinite loops + CRITICAL: JSON completeness is determined by parsing, NOT by last character check! Returns True if we should continue, False if AI Loop is done. """ if len(allSections) == 0: return True # No sections yet, continue - # CRITERION 1: If JSON was incomplete/broken - continue to repair/complete + # CRITERION 1: If JSON was incomplete/broken (parsing failed or incomplete) - continue to repair/complete if not wasJsonComplete: logger.info(f"Iteration {iteration}: JSON incomplete/broken - continuing to complete") return True - # CRITERION 2: JSON is complete - check for loop detection - # If JSON is complete, we're done (all content delivered) - # But check for infinite loops first + # CRITERION 2: JSON is complete (parsed successfully) - check for loop detection if self._isStuckInLoop(allSections, iteration): logger.warning(f"Iteration {iteration}: Detected potential infinite loop - stopping AI loop") return False @@ -942,153 +928,6 @@ Respond with ONLY a JSON object in this exact format: logger.info(f"Iteration {iteration}: JSON complete - AI loop done") return False - def _analyzeTaskCompletion( - self, - allSections: List[Dict[str, Any]], - userPrompt: Optional[str], - iteration: int, - workflowIntent: Optional[Dict[str, Any]] = None - ) -> bool: - """ - GENERIC task completion analysis using KPIs from Intent Analyzer. - - Uses definitionOfDone KPIs from workflowIntent to check completion. - Falls back to simple heuristics if workflowIntent not available. - - Returns True if task appears complete, False otherwise. - """ - if not allSections: - return False - - # Calculate current metrics from JSON structure - totalSections = len(allSections) - totalContentSize = 0 - totalRows = 0 - totalItems = 0 - totalParagraphs = 0 - totalHeadings = 0 - totalCodeLines = 0 - contentTypes = set() - lastSectionComplete = True - - for section in allSections: - contentType = section.get("content_type", "") - contentTypes.add(contentType) - elements = section.get("elements", []) - - if isinstance(elements, list) and elements: - lastElem = elements[-1] if elements else {} - else: - lastElem = elements if isinstance(elements, dict) else {} - - if isinstance(lastElem, dict): - if contentType == "code_block": - code = lastElem.get("code", "") - if code: - lines = [l for l in code.split('\n') if l.strip()] - totalCodeLines += len(lines) - totalContentSize += len(code) - if code and not code.rstrip().endswith('\n'): - lastSectionComplete = False - - elif contentType == "table": - rows = lastElem.get("rows", []) - if isinstance(rows, list): - totalRows += len(rows) - totalContentSize += len(str(rows)) - if not lastElem.get("headers"): - lastSectionComplete = False - - elif contentType in ["bullet_list", "numbered_list"]: - items = lastElem.get("items", []) - if isinstance(items, list): - totalItems += len(items) - totalContentSize += len(str(items)) - - elif contentType == "heading": - totalHeadings += 1 - text = lastElem.get("text", "") - if text: - totalContentSize += len(text) - - elif contentType == "paragraph": - totalParagraphs += 1 - text = lastElem.get("text", "") - if text: - totalContentSize += len(text) - if text and not text.rstrip()[-1] in '.!?': - lastSectionComplete = False - - # STRATEGY 1: Use KPIs from Intent Analyzer (preferred method) - if workflowIntent and isinstance(workflowIntent, dict): - definitionOfDone = workflowIntent.get("definitionOfDone", {}) - if definitionOfDone: - # Check all KPI thresholds - allKPIsMet = True - kpiChecks = [] - - minSections = definitionOfDone.get("minSections", 0) - if minSections > 0: - met = totalSections >= minSections - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"sections: {totalSections}/{minSections}") - - minParagraphs = definitionOfDone.get("minParagraphs", 0) - if minParagraphs > 0: - met = totalParagraphs >= minParagraphs - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"paragraphs: {totalParagraphs}/{minParagraphs}") - - minHeadings = definitionOfDone.get("minHeadings", 0) - if minHeadings > 0: - met = totalHeadings >= minHeadings - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"headings: {totalHeadings}/{minHeadings}") - - minTableRows = definitionOfDone.get("minTableRows", 0) - if minTableRows > 0: - met = totalRows >= minTableRows - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"tableRows: {totalRows}/{minTableRows}") - - minListItems = definitionOfDone.get("minListItems", 0) - if minListItems > 0: - met = totalItems >= minListItems - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"listItems: {totalItems}/{minListItems}") - - minCodeLines = definitionOfDone.get("minCodeLines", 0) - if minCodeLines > 0: - met = totalCodeLines >= minCodeLines - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"codeLines: {totalCodeLines}/{minCodeLines}") - - minContentSize = definitionOfDone.get("minContentSize", 0) - if minContentSize > 0: - met = totalContentSize >= minContentSize - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"contentSize: {totalContentSize}/{minContentSize}") - - # Check required content types - requiredContentTypes = definitionOfDone.get("requiredContentTypes", []) - if requiredContentTypes: - met = all(ct in contentTypes for ct in requiredContentTypes) - allKPIsMet = allKPIsMet and met - kpiChecks.append(f"contentTypes: {list(contentTypes)}/{requiredContentTypes}") - - # If all KPIs met and last section is complete, task is done - if allKPIsMet and lastSectionComplete: - logger.info(f"Task completion (KPI-based): All KPIs met - {', '.join(kpiChecks)}") - return True - - # STRATEGY 2: Fallback to simple heuristics if no workflowIntent - # Only use if substantial content and last section complete - if totalContentSize > 20000 and lastSectionComplete and iteration > 2: - logger.info(f"Task completion (fallback heuristic): Large content ({totalContentSize} chars) over {iteration} iterations, last section complete") - return True - - return False - def _isStuckInLoop( self, allSections: List[Dict[str, Any]], @@ -1436,36 +1275,14 @@ Respond with ONLY a JSON object in this exact format: if promptArgs: userPrompt = promptArgs.get("userPrompt") or promptArgs.get("user_prompt") - # CRITICAL: Get actionIntent (not taskIntent or workflowIntent) for Definition of Done - # Action Intent contains Definition of Done for THIS specific action - # Each action needs its own DoD because actions have different completion criteria - # Example: Action 1 "Generate 2000 primes" → DoD: 200 rows, Action 2 "Convert to CSV" → DoD: 1 document - actionIntent = None - if hasattr(self.services, 'workflow') and self.services.workflow: - # Priority 1: Use actionIntent (most specific - for THIS action) - actionIntent = getattr(self.services.workflow, '_actionIntent', None) - if not actionIntent: - # Priority 2: Fallback to taskIntent (for THIS task) - actionIntent = getattr(self.services.workflow, '_taskIntent', None) - if actionIntent: - logger.info("Action intent not found, using task intent as fallback") - if not actionIntent: - # Priority 3: Fallback to workflowIntent (for entire workflow) - actionIntent = getattr(self.services.workflow, '_workflowIntent', None) - logger.warning("Action and task intent not found, using workflow intent as fallback") - - # Store actionIntent separately (not in promptArgs - buildGenerationPrompt doesn't accept it) - # actionIntent is passed to _callAiWithLooping for completion detection, not for prompt building - generated_json = await self._callAiWithLooping( generation_prompt, options, "document_generation", buildGenerationPrompt, - promptArgs, # Does NOT contain taskIntent - buildGenerationPrompt doesn't accept it + promptArgs, aiOperationId, - userPrompt=userPrompt, - workflowIntent=actionIntent # Use actionIntent (contains Definition of Done for THIS action) + userPrompt=userPrompt ) self.services.chat.progressLogUpdate(aiOperationId, 0.7, "Parsing generated JSON") diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 8836712c..b1c4d879 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -90,11 +90,15 @@ class ChatService: allDocuments = [] for docRef in stringRefs: if docRef.startswith("docItem:"): - # docItem:: - extract ID and find document + # docItem:: or docItem: (filename is optional) + # ALWAYS try to match by documentId first (parts[1] is always the documentId when format is correct) parts = docRef.split(':') if len(parts) >= 2: - docId = parts[1] - # Find the document by ID + docId = parts[1] # This should be the documentId (UUID) + docFound = False + + # ALWAYS try to match by documentId first (regardless of number of parts) + # This handles: docItem:documentId and docItem:documentId:filename for message in workflow.messages: # Validate message belongs to this workflow msgWorkflowId = getattr(message, 'workflowId', None) @@ -104,9 +108,42 @@ class ChatService: if message.documents: for doc in message.documents: if doc.id == docId: - docName = getattr(doc, 'fileName', 'unknown') allDocuments.append(doc) + docFound = True + logger.debug(f"Matched document reference '{docRef}' to document {doc.id} (fileName: {getattr(doc, 'fileName', 'unknown')}) by documentId") break + if docFound: + break + + # Fallback: If not found by documentId and it looks like a filename (has file extension), try filename matching + # This handles cases where AI incorrectly generates docItem:filename.docx + if not docFound and '.' in docId and len(parts) == 2: + # Format: docItem:filename (AI generated wrong format) - try to match by filename + filename = parts[1] + logger.warning(f"Document reference '{docRef}' not found by documentId, attempting to match by filename: {filename}") + + for message in workflow.messages: + # Validate message belongs to this workflow + msgWorkflowId = getattr(message, 'workflowId', None) + if not msgWorkflowId or msgWorkflowId != workflowId: + continue + + if message.documents: + for doc in message.documents: + docFileName = getattr(doc, 'fileName', '') + # Match filename exactly or by base name (without path) + if docFileName == filename or docFileName.endswith(filename): + allDocuments.append(doc) + docFound = True + logger.info(f"Matched document reference '{docRef}' to document {doc.id} by filename {docFileName}") + break + if docFound: + break + + if not docFound: + logger.error(f"Could not resolve document reference '{docRef}' - no document found with filename '{filename}'") + elif not docFound: + logger.error(f"Could not resolve document reference '{docRef}' - no document found with documentId '{docId}'") elif docRef.startswith("docList:"): # docList::