# contentValidator.py # Content validation for adaptive React mode import re import logging import json from typing import List, Dict, Any logger = logging.getLogger(__name__) class ContentValidator: """Validates delivered content against user intent""" def __init__(self): pass def validateContent(self, documents: List[Any], intent: Dict[str, Any]) -> Dict[str, Any]: """Validates delivered content against user intent using AI""" try: # First, try AI-based validation for intelligent gap analysis aiValidation = self._validateWithAI(documents, intent) if aiValidation: return aiValidation # Fallback to rule-based validation if AI validation fails validationDetails = [] for doc in documents: content = self._extractContent(doc) detail = self._validateSingleDocument(content, doc, intent) validationDetails.append(detail) # Calculate overall success overallSuccess = all(detail.get("successCriteriaMet", [False]) for detail in validationDetails) # Calculate quality score qualityScore = self._calculateQualityScore(validationDetails) # Generate improvement suggestions improvementSuggestions = self._generateImprovementSuggestions(validationDetails, intent) return { "overallSuccess": overallSuccess, "qualityScore": qualityScore, "validationDetails": validationDetails, "improvementSuggestions": improvementSuggestions } except Exception as e: logger.error(f"Error validating content: {str(e)}") return self._createFailedValidationResult(str(e)) def _extractContent(self, doc: Any) -> str: """Extracts content from a document""" try: if hasattr(doc, 'documentData'): data = doc.documentData if isinstance(data, dict) and 'content' in data: return str(data['content']) else: return str(data) return "" except Exception: return "" def _validateSingleDocument(self, content: str, doc: Any, intent: Dict[str, Any]) -> Dict[str, Any]: """Validates a single document against intent""" # Check data type match dataTypeMatch = self._checkDataTypeMatch(content, intent.get("dataType", "unknown")) # Check format match formatMatch = self._checkFormatMatch(content, intent.get("expectedFormat", "unknown")) # Calculate quality score qualityScore = self._calculateDocumentQualityScore(content, intent) # Check success criteria successCriteriaMet = self._checkSuccessCriteria(content, intent) # Identify specific issues specificIssues = self._identifySpecificIssues(content, intent) # Generate improvement suggestions improvementSuggestions = self._generateDocumentImprovementSuggestions(content, intent) return { "documentName": getattr(doc, 'documentName', 'Unknown'), "dataTypeMatch": dataTypeMatch, "formatMatch": formatMatch, "qualityScore": qualityScore, "successCriteriaMet": successCriteriaMet, "specificIssues": specificIssues, "improvementSuggestions": improvementSuggestions } def _checkDataTypeMatch(self, content: str, dataType: str) -> bool: """Checks if content matches the expected data type""" if dataType == "numbers": return self._containsNumbers(content) elif dataType == "text": return self._containsText(content) elif dataType == "documents": return self._containsDocumentContent(content) elif dataType == "analysis": return self._containsAnalysis(content) elif dataType == "code": return self._containsCode(content) else: return True # Unknown type, assume match def _containsNumbers(self, content: str) -> bool: """Checks if content contains actual numbers (not code)""" # Look for actual numbers in the content numbers = re.findall(r'\b\d+\b', content) # Check if it's code (contains function definitions, etc.) isCode = any(keyword in content.lower() for keyword in [ 'def ', 'function', 'import ', 'class ', 'for ', 'while ', 'if ', 'return', 'print(', 'console.log', 'public ', 'private ' ]) # If it's code, it doesn't contain actual numbers if isCode: return False # If it has numbers and it's not code, it contains actual numbers return len(numbers) > 0 def _containsText(self, content: str) -> bool: """Checks if content contains readable text""" # Remove numbers and special characters textContent = re.sub(r'[^\w\s]', '', content) words = textContent.split() # Check if there are enough words to be considered text return len(words) > 5 def _containsDocumentContent(self, content: str) -> bool: """Checks if content is suitable for document creation""" # Check for structured content hasStructure = any(indicator in content for indicator in [ '\n', '\t', '|', '-', '*', '1.', '2.', '•', '◦' ]) # Check for meaningful content hasMeaningfulContent = len(content.strip()) > 50 return hasStructure and hasMeaningfulContent def _containsAnalysis(self, content: str) -> bool: """Checks if content contains analysis""" analysisIndicators = [ 'analysis', 'findings', 'conclusion', 'summary', 'insights', 'trends', 'patterns', 'comparison', 'evaluation', 'assessment' ] contentLower = content.lower() return any(indicator in contentLower for indicator in analysisIndicators) def _containsCode(self, content: str) -> bool: """Checks if content contains code""" codeIndicators = [ 'def ', 'function', 'import ', 'class ', 'for ', 'while ', 'if ', 'return', 'print(', 'console.log', 'public ', 'private ', 'void ', 'int ', 'string ', 'var ', 'let ', 'const ' ] contentLower = content.lower() return any(indicator in contentLower for indicator in codeIndicators) def _checkFormatMatch(self, content: str, expectedFormat: str) -> bool: """Checks if content matches expected format""" if expectedFormat == "raw_data": # Raw data should be simple, not heavily formatted return not any(indicator in content for indicator in [ '', '
', '', '## ', '### ', '**', '__' ]) elif expectedFormat == "formatted": # Formatted content should have structure return any(indicator in content for indicator in [ '\n', '\t', '|', '-', '*', '1.', '2.', '•' ]) elif expectedFormat == "structured": # Structured content should have clear organization return any(indicator in content for indicator in [ '{', '}', '[', ']', '|', '\t', ' ' ]) else: return True # Unknown format, assume match def _checkSuccessCriteria(self, content: str, intent: Dict[str, Any]) -> List[bool]: """Checks if content meets success criteria""" criteriaMet = [] successCriteria = intent.get("successCriteria", []) for criterion in successCriteria: if 'prime numbers' in criterion.lower(): # Check if content contains actual prime numbers, not code hasNumbers = bool(re.search(r'\b\d+\b', content)) isNotCode = not any(keyword in content.lower() for keyword in [ 'def ', 'function', 'import ', 'class ' ]) criteriaMet.append(hasNumbers and isNotCode) elif 'document' in criterion.lower(): # Check if content is suitable for document creation hasStructure = any(indicator in content for indicator in [ '\n', '\t', '|', '-', '*', '1.', '2.' ]) criteriaMet.append(hasStructure) elif 'format' in criterion.lower(): # Check if content is properly formatted hasFormatting = any(indicator in content for indicator in [ '\n', '\t', '|', '-', '*', '1.', '2.', '•' ]) criteriaMet.append(hasFormatting) else: # Generic check - content should not be empty criteriaMet.append(len(content.strip()) > 0) return criteriaMet def _calculateDocumentQualityScore(self, content: str, intent: Dict[str, Any]) -> float: """Calculates quality score for a single document""" score = 0.0 # Base score for having content if len(content.strip()) > 0: score += 0.2 # Score for data type match if self._checkDataTypeMatch(content, intent.get("dataType", "unknown")): score += 0.3 # Score for format match if self._checkFormatMatch(content, intent.get("expectedFormat", "unknown")): score += 0.2 # Score for success criteria successCriteriaMet = self._checkSuccessCriteria(content, intent) if successCriteriaMet: successRate = sum(successCriteriaMet) / len(successCriteriaMet) score += 0.3 * successRate return min(score, 1.0) def _calculateQualityScore(self, validationDetails: List[Dict[str, Any]]) -> float: """Calculates overall quality score from validation details""" if not validationDetails: return 0.0 totalScore = sum(detail.get("qualityScore", 0) for detail in validationDetails) return totalScore / len(validationDetails) def _identifySpecificIssues(self, content: str, intent: Dict[str, Any]) -> List[str]: """Identifies specific issues with the content""" issues = [] # Check for common issues if intent.get("dataType") == "numbers" and self._containsCode(content): issues.append("Content contains code instead of actual numbers") if intent.get("expectedFormat") == "raw_data" and any(indicator in content for indicator in ['', '## ', '**']): issues.append("Content is formatted when raw data was requested") if len(content.strip()) == 0: issues.append("Content is empty") return issues def _generateDocumentImprovementSuggestions(self, content: str, intent: Dict[str, Any]) -> List[str]: """Generates improvement suggestions for a single document""" suggestions = [] dataType = intent.get("dataType", "unknown") expectedFormat = intent.get("expectedFormat", "unknown") if dataType == "numbers" and self._containsCode(content): suggestions.append("Deliver actual numbers, not code to generate them") if expectedFormat == "raw_data" and any(indicator in content for indicator in ['', '## ']): suggestions.append("Provide raw data without formatting") if len(content.strip()) == 0: suggestions.append("Provide actual content") return suggestions def _generateImprovementSuggestions(self, validationDetails: List[Dict[str, Any]], intent: Dict[str, Any]) -> List[str]: """Generates improvement suggestions based on validation results""" suggestions = [] # Check for common issues if not any(detail.get("dataTypeMatch", False) for detail in validationDetails): dataType = intent.get("dataType", "unknown") suggestions.append(f"Content should contain {dataType} data, not code or other formats") if not any(detail.get("formatMatch", False) for detail in validationDetails): expectedFormat = intent.get("expectedFormat", "unknown") suggestions.append(f"Content should be in {expectedFormat} format") # Add specific suggestions from validation details for detail in validationDetails: suggestions.extend(detail.get("improvementSuggestions", [])) return list(set(suggestions)) # Remove duplicates def _createFailedValidationResult(self, error: str) -> Dict[str, Any]: """Creates a failed validation result""" return { "overallSuccess": False, "qualityScore": 0.0, "validationDetails": [], "improvementSuggestions": [f"Validation failed: {error}"] } def _validateWithAI(self, documents: List[Any], intent: Dict[str, Any]) -> Dict[str, Any]: """AI-based validation to intelligently assess task completion""" try: # Extract content from all documents documentContents = [] for doc in documents: content = self._extractContent(doc) documentContents.append({ "name": getattr(doc, 'documentName', 'Unknown'), "content": content[:2000] # Limit content for AI processing }) # Create AI validation prompt validationPrompt = f""" You are a task completion validator. Analyze if the delivered content actually fulfills the user's request. USER REQUEST: {intent.get('primaryGoal', 'Unknown')} DELIVERED CONTENT: {json.dumps(documentContents, indent=2)} TASK: Determine if the user's request has been fully completed. Analyze the gap between what was requested and what was delivered. Consider any missing elements, incorrect formats, incomplete work, or other discrepancies. Respond with JSON only: {{ "overallSuccess": true/false, "qualityScore": 0.0-1.0, "gapAnalysis": "Detailed analysis of what's missing or incorrect", "improvementSuggestions": ["specific action 1", "specific action 2"] }} """ # Call AI service for validation from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType request_options = AiCallOptions() request_options.operationType = OperationType.GENERAL request = AiCallRequest(prompt=validationPrompt, context="", options=request_options) # Get AI service from the workflow context if hasattr(self, 'services') and hasattr(self.services, 'ai'): response = self.services.ai.aiObjects.call(request) if response and response.content: import re result = response.content.strip() json_match = re.search(r'\{.*\}', result, re.DOTALL) if json_match: result = json_match.group(0) aiResult = json.loads(result) return { "overallSuccess": aiResult.get("overallSuccess", False), "qualityScore": aiResult.get("qualityScore", 0.0), "validationDetails": [{ "documentName": "AI Validation", "gapAnalysis": aiResult.get("gapAnalysis", ""), "successCriteriaMet": [aiResult.get("overallSuccess", False)] }], "improvementSuggestions": aiResult.get("improvementSuggestions", []) } return None # Fallback to rule-based validation except Exception as e: logger.error(f"AI validation failed: {str(e)}") return None # Fallback to rule-based validation