From 9521823958a42b1dff38c84b0db08144398e051e Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 17 Oct 2025 01:08:53 +0200 Subject: [PATCH] Workflow r4 ready for end 2 end testing --- modules/services/serviceAi/subCoreAi.py | 6 +- .../serviceAi/subDocumentGeneration.py | 77 +++++++++++++++---- .../mainServiceExtraction.py | 2 +- .../mainServiceGeneration.py | 4 +- .../workflows/processing/modes/modeReact.py | 30 ++++++-- 5 files changed, 93 insertions(+), 26 deletions(-) diff --git a/modules/services/serviceAi/subCoreAi.py b/modules/services/serviceAi/subCoreAi.py index 302d9713..7d1b849b 100644 --- a/modules/services/serviceAi/subCoreAi.py +++ b/modules/services/serviceAi/subCoreAi.py @@ -136,7 +136,7 @@ class SubCoreAi: # Emit stats for direct AI call self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, response, f"ai.call.{options.operationType}" ) @@ -187,7 +187,7 @@ class SubCoreAi: # Emit stats for image analysis self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, response, f"ai.image.{options.operationType}" ) @@ -228,7 +228,7 @@ class SubCoreAi: # Emit stats for image generation self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, response, f"ai.generate.image" ) diff --git a/modules/services/serviceAi/subDocumentGeneration.py b/modules/services/serviceAi/subDocumentGeneration.py index 21dc9cd6..eafed975 100644 --- a/modules/services/serviceAi/subDocumentGeneration.py +++ b/modules/services/serviceAi/subDocumentGeneration.py @@ -177,20 +177,44 @@ class SubDocumentGeneration: import re result = response.content.strip() - # Extract JSON from markdown if present - json_match = re.search(r'```json\s*\n(.*?)\n```', result, re.DOTALL) - if json_match: - result = json_match.group(1).strip() - elif result.startswith('```json'): - result = re.sub(r'^```json\s*', '', result) - result = re.sub(r'\s*```$', '', result) - elif result.startswith('```'): - result = re.sub(r'^```\s*', '', result) - result = re.sub(r'\s*```$', '', result) - - # Try to parse JSON - enhancedContent = json.loads(result) - logger.info(f"AI enhanced JSON content successfully") + # Check if result is empty after stripping + if not result: + logger.warning("AI generation returned empty content after stripping, using original content") + enhancedContent = aiResponseJson + else: + # Extract JSON from markdown if present + json_match = re.search(r'```json\s*\n(.*?)\n```', result, re.DOTALL) + if json_match: + result = json_match.group(1).strip() + elif result.startswith('```json'): + result = re.sub(r'^```json\s*', '', result) + result = re.sub(r'\s*```$', '', result) + elif result.startswith('```'): + result = re.sub(r'^```\s*', '', result) + result = re.sub(r'\s*```$', '', result) + + # Check if result is still empty after markdown extraction + if not result: + logger.warning("AI generation returned empty content after markdown extraction, using original content") + enhancedContent = aiResponseJson + else: + # Try to parse JSON with better error handling + try: + enhancedContent = json.loads(result) + logger.info(f"AI enhanced JSON content successfully") + except json.JSONDecodeError as jsonError: + # Try to fix common JSON issues + fixed_result = self._attemptJsonFix(result) + if fixed_result != result: + try: + enhancedContent = json.loads(fixed_result) + logger.info(f"AI enhanced JSON content successfully after fixing") + except json.JSONDecodeError: + logger.warning(f"AI generation returned invalid JSON even after fixing: {str(jsonError)}, using original content") + enhancedContent = aiResponseJson + else: + logger.warning(f"AI generation returned invalid JSON: {str(jsonError)}, using original content") + enhancedContent = aiResponseJson except json.JSONDecodeError as e: logger.warning(f"AI generation returned invalid JSON: {str(e)}, using original content") @@ -818,3 +842,28 @@ Return only the valid JSON: except Exception as e: logger.warning(f"AI JSON repair failed: {str(e)}") return malformed_json + + def _attemptJsonFix(self, json_string: str) -> str: + """Attempt to fix common JSON issues""" + try: + # Remove any trailing commas before closing braces/brackets + import re + fixed = re.sub(r',(\s*[}\]])', r'\1', json_string) + + # Try to fix unterminated strings by adding quotes at the end + if '"' in fixed and not fixed.strip().endswith('"'): + # Count quotes to see if we have an odd number (unterminated string) + quote_count = fixed.count('"') + if quote_count % 2 == 1: + # Find the last quote and add a closing quote + last_quote_pos = fixed.rfind('"') + if last_quote_pos != -1: + # Check if there's content after the last quote that needs to be quoted + after_quote = fixed[last_quote_pos + 1:].strip() + if after_quote and not after_quote.startswith(','): + # Add closing quote before any trailing content + fixed = fixed[:last_quote_pos + 1] + '"' + after_quote + + return fixed + except Exception: + return json_string diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index 1f910e73..2fb1b44d 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -117,7 +117,7 @@ class ExtractionService: ) self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, aiResponse, f"extraction.process.{doc.mimeType}" ) diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index eb4287af..dcc30a7f 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -478,7 +478,7 @@ class GenerationService: ) self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, aiResponse, f"generation.render.{outputFormat}" ) @@ -505,7 +505,7 @@ class GenerationService: ) self.services.workflow.storeWorkflowStat( - self.services.workflow, + self.services.currentWorkflow, aiResponse, f"generation.render.{outputFormat}" ) diff --git a/modules/workflows/processing/modes/modeReact.py b/modules/workflows/processing/modes/modeReact.py index 36ca90d9..1fd8e8bb 100644 --- a/modules/workflows/processing/modes/modeReact.py +++ b/modules/workflows/processing/modes/modeReact.py @@ -142,12 +142,12 @@ class ReactMode(BaseMode): context.previous_review_result.append(decision) # Update context with learnings from this step - if decision and decision.get('reason'): + if decision and isinstance(decision, dict) and decision.get('reason'): if not hasattr(context, 'improvements'): context.improvements = [] context.improvements.append(f"Step {step}: {decision.get('reason')}") - lastReviewDict = decision + lastReviewDict = decision if isinstance(decision, dict) else {} # Create user-friendly message AFTER action execution # Action completion message is now handled by the standard message creator in _actExecute @@ -646,11 +646,29 @@ class ReactMode(BaseMode): ) # Write refinement/validation response to debug writeDebugFile(resp or '', "validation_refinement_response") - js = resp[resp.find('{'):resp.rfind('}')+1] if resp else '{}' - try: - decision = json.loads(js) - except Exception: + + # More robust JSON extraction + if not resp: decision = {"decision": "continue", "reason": "default"} + else: + # Find JSON boundaries more safely + start_idx = resp.find('{') + end_idx = resp.rfind('}') + + if start_idx != -1 and end_idx != -1 and end_idx > start_idx: + js = resp[start_idx:end_idx+1] + else: + js = '{}' + + try: + decision = json.loads(js) + # Ensure decision is a dictionary + if not isinstance(decision, dict): + decision = {"decision": "continue", "reason": "default"} + except Exception as e: + logger.warning(f"Failed to parse refinement decision JSON: {e}") + decision = {"decision": "continue", "reason": "default"} + return decision async def _createReactActionMessage(self, workflow: ChatWorkflow, selection: Dict[str, Any],