Workflow r4 ready for end 2 end testing

modules/services/serviceAi/subCoreAi.py

@@ -136,7 +136,7 @@ class SubCoreAi:
                 
                 # Emit stats for direct AI call
                 self.services.workflow.storeWorkflowStat(
-                    self.services.workflow, 
+                    self.services.currentWorkflow, 
                     response, 
                     f"ai.call.{options.operationType}"
                 )
@@ -187,7 +187,7 @@ class SubCoreAi:
             
             # Emit stats for image analysis
             self.services.workflow.storeWorkflowStat(
-                self.services.workflow, 
+                self.services.currentWorkflow, 
                 response, 
                 f"ai.image.{options.operationType}"
             )
@@ -228,7 +228,7 @@ class SubCoreAi:
             
             # Emit stats for image generation
             self.services.workflow.storeWorkflowStat(
-                self.services.workflow, 
+                self.services.currentWorkflow, 
                 response, 
                 f"ai.generate.image"
             )

modules/services/serviceAi/subDocumentGeneration.py

@@ -177,20 +177,44 @@ class SubDocumentGeneration:
                             import re
                             result = response.content.strip()
                             
-                            # Extract JSON from markdown if present
+                            # Check if result is empty after stripping
-                            json_match = re.search(r'```json\s*\n(.*?)\n```', result, re.DOTALL)
+                            if not result:
-                            if json_match:
+                                logger.warning("AI generation returned empty content after stripping, using original content")
-                                result = json_match.group(1).strip()
+                                enhancedContent = aiResponseJson
-                            elif result.startswith('```json'):
+                            else:
-                                result = re.sub(r'^```json\s*', '', result)
+                                # Extract JSON from markdown if present
-                                result = re.sub(r'\s*```$', '', result)
+                                json_match = re.search(r'```json\s*\n(.*?)\n```', result, re.DOTALL)
-                            elif result.startswith('```'):
+                                if json_match:
-                                result = re.sub(r'^```\s*', '', result)
+                                    result = json_match.group(1).strip()
-                                result = re.sub(r'\s*```$', '', result)
+                                elif result.startswith('```json'):
-                            
+                                    result = re.sub(r'^```json\s*', '', result)
-                            # Try to parse JSON
+                                    result = re.sub(r'\s*```$', '', result)
-                            enhancedContent = json.loads(result)
+                                elif result.startswith('```'):
-                            logger.info(f"AI enhanced JSON content successfully")
+                                    result = re.sub(r'^```\s*', '', result)
+                                    result = re.sub(r'\s*```$', '', result)
+                                
+                                # Check if result is still empty after markdown extraction
+                                if not result:
+                                    logger.warning("AI generation returned empty content after markdown extraction, using original content")
+                                    enhancedContent = aiResponseJson
+                                else:
+                                    # Try to parse JSON with better error handling
+                                    try:
+                                        enhancedContent = json.loads(result)
+                                        logger.info(f"AI enhanced JSON content successfully")
+                                    except json.JSONDecodeError as jsonError:
+                                        # Try to fix common JSON issues
+                                        fixed_result = self._attemptJsonFix(result)
+                                        if fixed_result != result:
+                                            try:
+                                                enhancedContent = json.loads(fixed_result)
+                                                logger.info(f"AI enhanced JSON content successfully after fixing")
+                                            except json.JSONDecodeError:
+                                                logger.warning(f"AI generation returned invalid JSON even after fixing: {str(jsonError)}, using original content")
+                                                enhancedContent = aiResponseJson
+                                        else:
+                                            logger.warning(f"AI generation returned invalid JSON: {str(jsonError)}, using original content")
+                                            enhancedContent = aiResponseJson
                             
                         except json.JSONDecodeError as e:
                             logger.warning(f"AI generation returned invalid JSON: {str(e)}, using original content")
@@ -818,3 +842,28 @@ Return only the valid JSON:
         except Exception as e:
             logger.warning(f"AI JSON repair failed: {str(e)}")
             return malformed_json
+    
+    def _attemptJsonFix(self, json_string: str) -> str:
+        """Attempt to fix common JSON issues"""
+        try:
+            # Remove any trailing commas before closing braces/brackets
+            import re
+            fixed = re.sub(r',(\s*[}\]])', r'\1', json_string)
+            
+            # Try to fix unterminated strings by adding quotes at the end
+            if '"' in fixed and not fixed.strip().endswith('"'):
+                # Count quotes to see if we have an odd number (unterminated string)
+                quote_count = fixed.count('"')
+                if quote_count % 2 == 1:
+                    # Find the last quote and add a closing quote
+                    last_quote_pos = fixed.rfind('"')
+                    if last_quote_pos != -1:
+                        # Check if there's content after the last quote that needs to be quoted
+                        after_quote = fixed[last_quote_pos + 1:].strip()
+                        if after_quote and not after_quote.startswith(','):
+                            # Add closing quote before any trailing content
+                            fixed = fixed[:last_quote_pos + 1] + '"' + after_quote
+            
+            return fixed
+        except Exception:
+            return json_string

modules/services/serviceExtraction/mainServiceExtraction.py

@@ -117,7 +117,7 @@ class ExtractionService:
             )
             
             self.services.workflow.storeWorkflowStat(
-                self.services.workflow, 
+                self.services.currentWorkflow, 
                 aiResponse, 
                 f"extraction.process.{doc.mimeType}"
             )

modules/services/serviceGeneration/mainServiceGeneration.py

@@ -478,7 +478,7 @@ class GenerationService:
             )
             
             self.services.workflow.storeWorkflowStat(
-                self.services.workflow, 
+                self.services.currentWorkflow, 
                 aiResponse, 
                 f"generation.render.{outputFormat}"
             )
@@ -505,7 +505,7 @@ class GenerationService:
             )
             
             self.services.workflow.storeWorkflowStat(
-                self.services.workflow, 
+                self.services.currentWorkflow, 
                 aiResponse, 
                 f"generation.render.{outputFormat}"
             )

modules/workflows/processing/modes/modeReact.py

@@ -142,12 +142,12 @@ class ReactMode(BaseMode):
                     context.previous_review_result.append(decision)
                 
                 # Update context with learnings from this step
-                if decision and decision.get('reason'):
+                if decision and isinstance(decision, dict) and decision.get('reason'):
                     if not hasattr(context, 'improvements'):
                         context.improvements = []
                     context.improvements.append(f"Step {step}: {decision.get('reason')}")
                 
-                lastReviewDict = decision
+                lastReviewDict = decision if isinstance(decision, dict) else {}
                 
                 # Create user-friendly message AFTER action execution
                 # Action completion message is now handled by the standard message creator in _actExecute
@@ -646,11 +646,29 @@ class ReactMode(BaseMode):
         )
         # Write refinement/validation response to debug
         writeDebugFile(resp or '', "validation_refinement_response")
-        js = resp[resp.find('{'):resp.rfind('}')+1] if resp else '{}'
+        
-        try:
+        # More robust JSON extraction
-            decision = json.loads(js)
+        if not resp:
-        except Exception:
             decision = {"decision": "continue", "reason": "default"}
+        else:
+            # Find JSON boundaries more safely
+            start_idx = resp.find('{')
+            end_idx = resp.rfind('}')
+            
+            if start_idx != -1 and end_idx != -1 and end_idx > start_idx:
+                js = resp[start_idx:end_idx+1]
+            else:
+                js = '{}'
+            
+            try:
+                decision = json.loads(js)
+                # Ensure decision is a dictionary
+                if not isinstance(decision, dict):
+                    decision = {"decision": "continue", "reason": "default"}
+            except Exception as e:
+                logger.warning(f"Failed to parse refinement decision JSON: {e}")
+                decision = {"decision": "continue", "reason": "default"}
+        
         return decision
     
     async def _createReactActionMessage(self, workflow: ChatWorkflow, selection: Dict[str, Any],