fixed json handling
This commit is contained in:
ValueOn AG
parent
3ccd284a58
commit
1d793d8e1a
8 changed files with 2453 additions and 835 deletions
|
|
@ -238,3 +238,21 @@ class AiProcessParameters(BaseModel):
|
|||
# NOTE: DocumentData, AiResponseMetadata, and AiResponse are defined in datamodelWorkflow.py
|
||||
# Import them from there if needed: from modules.datamodels.datamodelWorkflow import DocumentData, AiResponseMetadata, AiResponse
|
||||
|
||||
|
||||
class JsonAccumulationState(BaseModel):
|
||||
"""State for JSON string accumulation during iterative AI generation."""
|
||||
accumulatedJsonString: str = Field(description="Raw accumulated JSON string")
|
||||
isAccumulationMode: bool = Field(description="True if we're accumulating fragments")
|
||||
lastParsedResult: Optional[Dict[str, Any]] = Field(
|
||||
default=None,
|
||||
description="Last successfully parsed result (for prompt context)"
|
||||
)
|
||||
allSections: List[Dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Sections extracted so far (for prompt context)"
|
||||
)
|
||||
kpis: List[Dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="KPI definitions with current values: [{id, description, jsonPath, targetValue, currentValue}, ...]"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from modules.shared.jsonUtils import (
|
|||
parseJsonWithModel
|
||||
)
|
||||
from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler
|
||||
from modules.datamodels.datamodelAi import JsonAccumulationState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -190,6 +191,7 @@ Respond with ONLY a JSON object in this exact format:
|
|||
allSections = [] # Accumulate all sections across iterations
|
||||
lastRawResponse = None # Store last raw JSON response for continuation
|
||||
documentMetadata = None # Store document metadata (title, filename) from first iteration
|
||||
accumulationState = None # Track accumulation state for string accumulation
|
||||
|
||||
# Get parent log ID for iteration operations
|
||||
parentLogId = None
|
||||
|
|
@ -305,17 +307,77 @@ Respond with ONLY a JSON object in this exact format:
|
|||
|
||||
# Extract sections from response (handles both valid and broken JSON)
|
||||
# Only for document generation (JSON responses)
|
||||
# CRITICAL: Pass allSections to enable fragment detection and merging
|
||||
extractedSections, wasJsonComplete, parsedResult = self._extractSectionsFromResponse(
|
||||
result, iteration, debugPrefix, allSections
|
||||
# CRITICAL: Pass allSections and accumulationState to enable string accumulation
|
||||
extractedSections, wasJsonComplete, parsedResult, accumulationState = self._extractSectionsFromResponse(
|
||||
result, iteration, debugPrefix, allSections, accumulationState
|
||||
)
|
||||
|
||||
# Define KPIs if we just entered accumulation mode (iteration 1, incomplete JSON)
|
||||
if accumulationState and accumulationState.isAccumulationMode and iteration == 1 and not accumulationState.kpis:
|
||||
logger.info(f"Iteration {iteration}: Defining KPIs for accumulation tracking")
|
||||
continuationContext = buildContinuationContext(allSections, result)
|
||||
kpiDefinitions = await self._defineKpisFromPrompt(
|
||||
userPrompt or prompt,
|
||||
parsedResult,
|
||||
continuationContext,
|
||||
debugPrefix
|
||||
)
|
||||
# Initialize KPIs with currentValue = 0
|
||||
accumulationState.kpis = [{**kpi, "currentValue": 0} for kpi in kpiDefinitions]
|
||||
logger.info(f"Defined {len(accumulationState.kpis)} KPIs: {[kpi.get('id') for kpi in accumulationState.kpis]}")
|
||||
|
||||
# Extract and validate KPIs (if in accumulation mode with KPIs defined)
|
||||
if accumulationState and accumulationState.isAccumulationMode and accumulationState.kpis and parsedResult:
|
||||
updatedKpis = JsonResponseHandler.extractKpiValuesFromJson(
|
||||
parsedResult,
|
||||
accumulationState.kpis
|
||||
)
|
||||
|
||||
if updatedKpis:
|
||||
shouldProceed, reason = JsonResponseHandler.validateKpiProgression(
|
||||
accumulationState,
|
||||
updatedKpis
|
||||
)
|
||||
|
||||
if not shouldProceed:
|
||||
logger.warning(f"Iteration {iteration}: KPI validation failed: {reason}")
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, False)
|
||||
if operationId:
|
||||
self.services.chat.progressLogUpdate(operationId, 0.9, f"KPI validation failed: {reason} ({iteration} iterations)")
|
||||
break
|
||||
|
||||
# Update KPIs in accumulation state
|
||||
accumulationState.kpis = updatedKpis
|
||||
logger.info(f"Iteration {iteration}: KPIs updated: {[(kpi.get('id'), kpi.get('currentValue')) for kpi in updatedKpis]}")
|
||||
|
||||
# Check if all KPIs completed
|
||||
allCompleted = True
|
||||
for kpi in updatedKpis:
|
||||
targetValue = kpi.get("targetValue", 0)
|
||||
currentValue = kpi.get("currentValue", 0)
|
||||
if currentValue < targetValue:
|
||||
allCompleted = False
|
||||
break
|
||||
|
||||
if allCompleted:
|
||||
logger.info(f"Iteration {iteration}: All KPIs completed, finishing accumulation")
|
||||
wasJsonComplete = True # Mark as complete to exit loop
|
||||
|
||||
# CRITICAL: Handle JSON fragments (continuation content)
|
||||
# Fragment merging happens inside _extractSectionsFromResponse and updates allSections in place
|
||||
# If no sections extracted but fragment was merged, allSections was updated in place
|
||||
# Check if fragment was merged by checking if allSections was modified
|
||||
# Fragment merging happens inside _extractSectionsFromResponse
|
||||
# If merge fails (returns wasJsonComplete=True), stop iterations and complete JSON
|
||||
if not extractedSections and allSections:
|
||||
# Fragment was detected and merged directly into allSections (side effect in _extractSectionsFromResponse)
|
||||
if wasJsonComplete:
|
||||
# Merge failed - stop iterations, complete JSON with available data
|
||||
logger.error(f"Iteration {iteration}: ❌ MERGE FAILED - Stopping iterations, completing JSON with available data")
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, False)
|
||||
if operationId:
|
||||
self.services.chat.progressLogUpdate(operationId, 0.9, f"Merge failed, completing JSON ({iteration} iterations)")
|
||||
break
|
||||
|
||||
# Fragment was detected and merged successfully
|
||||
logger.info(f"Iteration {iteration}: JSON fragment detected and merged, continuing")
|
||||
# Don't break - fragment was merged, continue to get more content if needed
|
||||
# Check if we should continue based on JSON completeness
|
||||
|
|
@ -364,6 +426,10 @@ Respond with ONLY a JSON object in this exact format:
|
|||
# The break can occur anywhere - in any section, at any depth
|
||||
allSections = JsonResponseHandler.mergeSectionsIntelligently(allSections, extractedSections, iteration)
|
||||
|
||||
# Log merged sections for debugging
|
||||
merged_json_str = json.dumps(allSections, indent=2, ensure_ascii=False)
|
||||
self.services.utils.writeDebugFile(merged_json_str, f"{debugPrefix}_merged_sections_iteration_{iteration}")
|
||||
|
||||
# Check if we should continue (completion detection)
|
||||
# Simple logic: JSON completeness determines continuation
|
||||
shouldContinue = self._shouldContinueGeneration(
|
||||
|
|
@ -396,6 +462,10 @@ Respond with ONLY a JSON object in this exact format:
|
|||
if iteration >= maxIterations:
|
||||
logger.warning(f"AI call stopped after maximum iterations ({maxIterations})")
|
||||
|
||||
# CRITICAL: Complete any incomplete structures in sections before building final result
|
||||
# This ensures JSON is properly closed even if merge failed or iterations stopped early
|
||||
allSections = JsonResponseHandler.completeIncompleteStructures(allSections)
|
||||
|
||||
# Build final result from accumulated sections
|
||||
final_result = self._buildFinalResultFromSections(allSections, documentMetadata)
|
||||
|
||||
|
|
@ -406,77 +476,199 @@ Respond with ONLY a JSON object in this exact format:
|
|||
|
||||
# JSON merging logic moved to subJsonResponseHandling.py
|
||||
|
||||
async def _defineKpisFromPrompt(
|
||||
self,
|
||||
userPrompt: str,
|
||||
parsedJson: Optional[Dict[str, Any]],
|
||||
continuationContext: Dict[str, Any],
|
||||
debugPrefix: str = "kpi"
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Make separate AI call to define KPIs based on user prompt and delivered data.
|
||||
|
||||
Args:
|
||||
userPrompt: Original user prompt
|
||||
parsedJson: Parsed JSON from first iteration (if available)
|
||||
continuationContext: Continuation context with delivered summary
|
||||
|
||||
Returns:
|
||||
List of KPI definitions: [{"id": str, "description": str, "jsonPath": str, "targetValue": int}, ...]
|
||||
"""
|
||||
deliveredSummary = continuationContext.get("delivered_summary", "")
|
||||
cutOffElement = continuationContext.get("cut_off_element")
|
||||
elementBeforeCutoff = continuationContext.get("element_before_cutoff")
|
||||
|
||||
# Build prompt for KPI definition
|
||||
kpiDefinitionPrompt = f"""Analyze the user request and delivered data to define KPIs (Key Performance Indicators) for tracking progress.
|
||||
|
||||
User Request:
|
||||
{userPrompt}
|
||||
|
||||
Delivered Data Summary:
|
||||
{deliveredSummary}
|
||||
|
||||
Current JSON Structure (if available):
|
||||
{json.dumps(parsedJson, indent=2) if parsedJson else "Not available"}
|
||||
|
||||
Cut-off Element:
|
||||
{cutOffElement if cutOffElement else "Not available"}
|
||||
|
||||
Last Complete Element:
|
||||
{elementBeforeCutoff if elementBeforeCutoff else "Not available"}
|
||||
|
||||
Task: Define which JSON items should be tracked to measure completion progress.
|
||||
|
||||
For each trackable item, provide:
|
||||
- id: Unique identifier (use descriptive name)
|
||||
- description: What this KPI measures
|
||||
- jsonPath: Path to extract value from JSON (use dot notation with array indices, e.g., "sections[0].elements[0].items")
|
||||
- targetValue: Target value to reach (integer)
|
||||
|
||||
Return ONLY valid JSON in this format:
|
||||
{{
|
||||
"kpis": [
|
||||
{{
|
||||
"id": "unique_id",
|
||||
"description": "Description of what is measured",
|
||||
"jsonPath": "path.to.value",
|
||||
"targetValue": 0
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
If no trackable items can be identified, return: {{"kpis": []}}
|
||||
"""
|
||||
|
||||
try:
|
||||
request = AiCallRequest(
|
||||
prompt=kpiDefinitionPrompt,
|
||||
options=AiCallOptions(
|
||||
operationType=OperationTypeEnum.DATA_ANALYSE,
|
||||
priority=PriorityEnum.SPEED,
|
||||
processingMode=ProcessingModeEnum.BASIC
|
||||
)
|
||||
)
|
||||
|
||||
# Write KPI definition prompt to debug file
|
||||
self.services.utils.writeDebugFile(kpiDefinitionPrompt, f"{debugPrefix}_kpi_definition_prompt")
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
|
||||
# Write KPI definition response to debug file
|
||||
self.services.utils.writeDebugFile(response.content, f"{debugPrefix}_kpi_definition_response")
|
||||
|
||||
# Parse response
|
||||
extracted = extractJsonString(response.content)
|
||||
kpiResponse = json.loads(extracted)
|
||||
|
||||
kpiDefinitions = kpiResponse.get("kpis", [])
|
||||
logger.info(f"Defined {len(kpiDefinitions)} KPIs for tracking")
|
||||
|
||||
return kpiDefinitions
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to define KPIs: {e}, continuing without KPI tracking")
|
||||
return []
|
||||
|
||||
def _extractSectionsFromResponse(
|
||||
self,
|
||||
result: str,
|
||||
iteration: int,
|
||||
debugPrefix: str,
|
||||
allSections: List[Dict[str, Any]] = None
|
||||
) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]]]:
|
||||
allSections: List[Dict[str, Any]] = None,
|
||||
accumulationState: Optional[JsonAccumulationState] = None
|
||||
) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]:
|
||||
"""
|
||||
Extract sections from AI response, handling both valid and broken JSON.
|
||||
Uses repair mechanism for broken JSON.
|
||||
Handles JSON fragments (continuation content) that need to be merged into existing sections.
|
||||
Determines completion based on JSON structure (complete JSON = complete, broken/incomplete = incomplete).
|
||||
Returns (sections, wasJsonComplete, parsedResult)
|
||||
|
||||
NEW BEHAVIOR:
|
||||
- First iteration: Check if complete, if not start accumulation
|
||||
- Subsequent iterations: Accumulate strings, parse when complete
|
||||
|
||||
Returns:
|
||||
Tuple of:
|
||||
- sections: Extracted sections
|
||||
- wasJsonComplete: True if JSON is complete
|
||||
- parsedResult: Parsed JSON object
|
||||
- updatedAccumulationState: Updated accumulation state (None if not in accumulation mode)
|
||||
"""
|
||||
if allSections is None:
|
||||
allSections = []
|
||||
|
||||
# First, try to parse as valid JSON
|
||||
# CRITICAL: JSON completeness is determined by parsing, NOT by last character check!
|
||||
# Last character could be } or ] by chance, JSON still incomplete
|
||||
try:
|
||||
extracted = extractJsonString(result)
|
||||
if iteration == 1:
|
||||
# First iteration - check if complete
|
||||
parsed = None
|
||||
try:
|
||||
extracted = extractJsonString(result)
|
||||
parsed = json.loads(extracted)
|
||||
|
||||
# Check completeness
|
||||
if JsonResponseHandler.isJsonComplete(parsed):
|
||||
# Complete JSON - no accumulation needed
|
||||
sections = extractSectionsFromDocument(parsed)
|
||||
logger.info(f"Iteration 1: Complete JSON detected, no accumulation needed")
|
||||
return sections, True, parsed, None # No accumulation
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Try to parse the extracted JSON
|
||||
# If parsing succeeds, JSON is complete
|
||||
parsed_result = json.loads(extracted)
|
||||
# Incomplete - try to extract partial sections from broken JSON
|
||||
logger.info(f"Iteration 1: Incomplete JSON detected, attempting to extract partial sections")
|
||||
|
||||
# Extract sections from parsed JSON
|
||||
sections = extractSectionsFromDocument(parsed_result)
|
||||
|
||||
# CRITICAL: If no sections extracted but we have existing sections, check if it's a fragment
|
||||
if not sections and allSections:
|
||||
fragment = JsonResponseHandler.detectAndParseJsonFragment(result, allSections)
|
||||
if fragment:
|
||||
logger.info(f"Iteration {iteration}: Detected JSON fragment ({fragment.get('fragment_type')}), merging into existing sections")
|
||||
# Merge fragment into existing sections
|
||||
merged_sections = JsonResponseHandler.mergeFragmentIntoSection(fragment, allSections, iteration)
|
||||
# Update allSections in place (this is a side effect, but necessary for continuation)
|
||||
# Note: This modifies the caller's allSections list
|
||||
allSections[:] = merged_sections
|
||||
# Return empty list to indicate we merged directly (not new sections)
|
||||
# But mark as incomplete so loop continues if needed
|
||||
return [], False, parsed_result
|
||||
|
||||
# JSON parsed successfully = complete
|
||||
logger.info(f"Iteration {iteration}: JSON parsed successfully - marking as complete")
|
||||
return sections, True, parsed_result
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
# Broken JSON - try repair mechanism (normal in iterative generation)
|
||||
self.services.utils.writeDebugFile(result, f"{debugPrefix}_broken_json_iteration_{iteration}")
|
||||
logger.info(f"Iteration {iteration}: JSON parsing failed (broken JSON), attempting repair")
|
||||
|
||||
# Try to repair
|
||||
repaired_json = repairBrokenJson(result)
|
||||
|
||||
if repaired_json:
|
||||
# Extract sections from repaired JSON
|
||||
sections = extractSectionsFromDocument(repaired_json)
|
||||
# CRITICAL: JSON was broken, so mark as incomplete (wasJsonComplete = False)
|
||||
# This ensures the loop continues to get the rest of the content
|
||||
logger.info(f"Iteration {iteration}: JSON repaired, extracted {len(sections)} sections, marking as incomplete to continue")
|
||||
return sections, False, repaired_json # JSON was broken but repaired - mark as incomplete
|
||||
partialSections = []
|
||||
if parsed:
|
||||
# Try to extract sections from parsed (even if incomplete)
|
||||
partialSections = extractSectionsFromDocument(parsed)
|
||||
else:
|
||||
# Repair failed - but we should still continue to allow AI to retry
|
||||
logger.warning(f"Iteration {iteration}: All repair strategies failed, but continuing to allow retry")
|
||||
return [], False, None # Mark as incomplete so loop continues
|
||||
# Try to repair broken JSON and extract sections
|
||||
try:
|
||||
repaired = repairBrokenJson(result)
|
||||
if repaired:
|
||||
partialSections = extractSectionsFromDocument(repaired)
|
||||
parsed = repaired # Use repaired version for accumulation state
|
||||
except Exception:
|
||||
pass # If repair fails, continue with empty sections
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Iteration {iteration}: Unexpected error during parsing: {str(e)}")
|
||||
return [], False, None
|
||||
|
||||
# Define KPIs (async call - need to handle this)
|
||||
# For now, create accumulation state without KPIs, will be updated after async call
|
||||
accumulationState = JsonAccumulationState(
|
||||
accumulatedJsonString=result,
|
||||
isAccumulationMode=True,
|
||||
lastParsedResult=parsed,
|
||||
allSections=partialSections,
|
||||
kpis=[]
|
||||
)
|
||||
|
||||
# Note: KPI definition will be done in the caller (async context)
|
||||
return partialSections, False, parsed, accumulationState
|
||||
|
||||
else:
|
||||
# Subsequent iterations - accumulate
|
||||
if accumulationState and accumulationState.isAccumulationMode:
|
||||
accumulated, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulationState.accumulatedJsonString,
|
||||
result,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
# Update accumulation state
|
||||
accumulationState.accumulatedJsonString = accumulated
|
||||
accumulationState.lastParsedResult = parsedResult
|
||||
accumulationState.allSections = allSections + sections if sections else allSections
|
||||
accumulationState.isAccumulationMode = not isComplete
|
||||
|
||||
# Log accumulated JSON for debugging
|
||||
if parsedResult:
|
||||
accumulated_json_str = json.dumps(parsedResult, indent=2, ensure_ascii=False)
|
||||
self.services.utils.writeDebugFile(accumulated_json_str, f"{debugPrefix}_accumulated_json_iteration_{iteration}.json")
|
||||
|
||||
return sections, isComplete, parsedResult, accumulationState
|
||||
else:
|
||||
# No accumulation mode - process normally (shouldn't happen)
|
||||
logger.warning(f"Iteration {iteration}: No accumulation state but iteration > 1")
|
||||
return [], False, None, None
|
||||
|
||||
def _shouldContinueGeneration(
|
||||
self,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -718,13 +718,13 @@ def buildContinuationContext(allSections: List[Dict[str, Any]], lastRawResponse:
|
|||
if len(summary_items) == 0 and lastRawResponse:
|
||||
summary_items.append("- Previous response was incomplete/broken JSON - please continue from where it stopped")
|
||||
|
||||
# CRITICAL: If summary is too long, truncate: show first 100 and last 100 items
|
||||
if len(summary_items) > 200:
|
||||
first_100 = summary_items[:100]
|
||||
last_100 = summary_items[-100:]
|
||||
summary_lines.extend(first_100)
|
||||
summary_lines.append(f"... (truncated {len(summary_items) - 200} items) ...")
|
||||
summary_lines.extend(last_100)
|
||||
# CRITICAL: If summary is too long, truncate: show first 10 and last 10 items
|
||||
if len(summary_items) > 20:
|
||||
first_10 = summary_items[:10]
|
||||
last_10 = summary_items[-10:]
|
||||
summary_lines.extend(first_10)
|
||||
summary_lines.append(f"... (truncated {len(summary_items) - 20} items) ...")
|
||||
summary_lines.extend(last_10)
|
||||
else:
|
||||
summary_lines.extend(summary_items)
|
||||
|
||||
|
|
|
|||
|
|
@ -489,10 +489,12 @@ VALIDATION LOGIC:
|
|||
- Always trust structure statistics over any claims or descriptions
|
||||
|
||||
IMPROVEMENT SUGGESTIONS PRIORITY (CRITICAL):
|
||||
- Order by CRITERIA PRIORITY first, then gapType priority: missing_data > incomplete_data > wrong_structure > wrong_format
|
||||
- [0] MUST address the HIGHEST PRIORITY unmet criterion (check criteriaMapping for which criteria are unmet)
|
||||
- If multiple criteria are unmet, prioritize by: data completeness > structure > format
|
||||
- gapType indicates the PRIMARY issue, but improvement suggestions must prioritize based on unmet criteria order
|
||||
- Create ONE suggestion per UNMET criterion from criteriaMapping
|
||||
- Order suggestions by criteriaMapping index: [0] = first unmet criterion, [1] = second unmet criterion, etc.
|
||||
- Each suggestion addresses ONLY that specific criterion requirement
|
||||
- Do NOT combine multiple criteria into one suggestion
|
||||
- ACTIONABLE GUIDANCE: Provide concrete, actionable steps based on the structure evidence. Avoid simply restating the requirement - instead, explain what action to perform to meet the criterion based on what was actually found
|
||||
- EVIDENCE-BASED: Base suggestions on structure evidence, not assumptions.
|
||||
|
||||
=== OUTPUT FORMAT (JSON TEMPLATE) ===
|
||||
{{
|
||||
|
|
@ -528,7 +530,8 @@ IMPROVEMENT SUGGESTIONS PRIORITY (CRITICAL):
|
|||
|
||||
OUTPUT FORMAT NOTES:
|
||||
- criteriaMapping reason: Address ONLY the specific criterion requirement.
|
||||
- improvementSuggestions: [0] = highest priority unmet criterion from criteriaMapping. Order: unmet criteria by index first (data completeness > structure > format), then by gapType priority.
|
||||
- improvementSuggestions: ONE suggestion per UNMET criterion, ordered by criteriaMapping index. Do NOT combine criteria.
|
||||
- improvementSuggestions: Each suggestion must reference actual structure values found, calculate quantitative gaps when structure provides numbers, and provide actionable guidance based on structure evidence. Avoid generic restatements of requirements.
|
||||
|
||||
=== DATA ===
|
||||
|
||||
|
|
|
|||
|
|
@ -1,517 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test JSON Extraction from Incomplete/Broken JSON
|
||||
Tests the extraction of lastItemObject and cutItemObject from incomplete JSON responses
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
from typing import Dict, Any, List
|
||||
|
||||
# Add the gateway to path
|
||||
_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
if _gateway_path not in sys.path:
|
||||
sys.path.insert(0, _gateway_path)
|
||||
|
||||
from modules.shared.jsonUtils import buildContinuationContext, extractSectionsFromDocument
|
||||
from modules.shared.debugLogger import _getBaseDebugDir
|
||||
|
||||
|
||||
class JsonExtractionTester:
|
||||
def __init__(self):
|
||||
self.testResults = {}
|
||||
|
||||
def cleanupDebugFiles(self):
|
||||
"""Delete debug folder and current log file before test run."""
|
||||
try:
|
||||
# Get debug directory path
|
||||
debug_dir = _getBaseDebugDir()
|
||||
|
||||
# Delete debug folder if it exists
|
||||
if os.path.exists(debug_dir):
|
||||
print(f"Cleaning up debug folder: {debug_dir}")
|
||||
shutil.rmtree(debug_dir)
|
||||
print(f" [OK] Debug folder deleted")
|
||||
|
||||
# Also check for log file in the log directory
|
||||
from modules.shared.debugLogger import _resolveLogDir
|
||||
log_dir = _resolveLogDir()
|
||||
log_file = os.path.join(log_dir, "debug_workflow.log")
|
||||
if os.path.exists(log_file):
|
||||
print(f"Cleaning up log file: {log_file}")
|
||||
os.remove(log_file)
|
||||
print(f" [OK] Log file deleted")
|
||||
|
||||
except Exception as e:
|
||||
print(f" [WARN] Error during cleanup: {e}")
|
||||
|
||||
def createIncompleteTableJson(self) -> tuple[str, str]:
|
||||
"""Create incomplete JSON with table that ends mid-row."""
|
||||
complete_json = """{
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"id": "doc_1",
|
||||
"title": "First 4000 Prime Numbers",
|
||||
"filename": "prime_numbers_4000.csv",
|
||||
"sections": [
|
||||
{
|
||||
"id": "section_primes_csv",
|
||||
"content_type": "table",
|
||||
"elements": [
|
||||
{
|
||||
"headers": [],
|
||||
"rows": [
|
||||
["2", "3", "5", "7", "11", "13", "17", "19", "23", "29"],
|
||||
["31", "37", "41", "43", "47", "53", "59", "61", "67", "71"],
|
||||
["73", "79", "83", "89", "97", "101", "103", "107", "109", "113"],
|
||||
["16871", "16879", "16883", "16889", "16901", "16903", "16921", "16927", "16931", "16937"]
|
||||
],
|
||||
"caption": ""
|
||||
}
|
||||
],
|
||||
"order": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
# Incomplete JSON - cuts off mid-row (CRITICAL: must not end with } or ])
|
||||
# Remove all closing brackets and add incomplete row
|
||||
incomplete_json = complete_json.rstrip().rstrip('}').rstrip(']').rstrip('}').rstrip(']').rstrip('}') + ',\n ["16943", "16963", "16979", "16981", "16987", "16'
|
||||
|
||||
return complete_json, incomplete_json
|
||||
|
||||
def createIncompleteCodeBlockJson(self) -> tuple[str, str]:
|
||||
"""Create incomplete JSON with code_block that ends mid-line."""
|
||||
complete_json = """{
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"id": "doc_1",
|
||||
"title": "Prime Numbers CSV",
|
||||
"filename": "prime_numbers.csv",
|
||||
"sections": [
|
||||
{
|
||||
"id": "section_primes_csv",
|
||||
"content_type": "code_block",
|
||||
"elements": [
|
||||
{
|
||||
"code": "2,3,5,7,11,13,17,19,23,29\\n31,37,41,43,47,53,59,61,67,71\\n73,79,83,89,97,101,103,107,109,113\\n127,131,137,139,149,151,157,163,167,173\\n23773,23789,23801,23813,23819,23827,23831,23833,23857,23869",
|
||||
"language": "csv"
|
||||
}
|
||||
],
|
||||
"order": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
# Incomplete JSON - cuts off mid-line (CRITICAL: must not end with } or ])
|
||||
# Remove all closing brackets and add incomplete line
|
||||
incomplete_json = complete_json.rstrip().rstrip('}').rstrip(']').rstrip('}').rstrip(']').rstrip('}') + '\\n23873'
|
||||
|
||||
return complete_json, incomplete_json
|
||||
|
||||
def createIncompleteListJson(self) -> tuple[str, str]:
|
||||
"""Create incomplete JSON with list that ends mid-item."""
|
||||
complete_json = """{
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"id": "doc_1",
|
||||
"title": "Prime Numbers List",
|
||||
"filename": "prime_numbers.txt",
|
||||
"sections": [
|
||||
{
|
||||
"id": "section_primes_list",
|
||||
"content_type": "bullet_list",
|
||||
"elements": [
|
||||
{
|
||||
"items": ["2", "3", "5", "7", "11", "13", "17", "19", "23", "29"]
|
||||
}
|
||||
],
|
||||
"order": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}"""
|
||||
|
||||
# Incomplete JSON - cuts off mid-item (CRITICAL: must not end with } or ])
|
||||
# Remove all closing brackets and add incomplete item
|
||||
incomplete_json = complete_json.rstrip().rstrip('}').rstrip(']').rstrip('}').rstrip(']').rstrip('}') + ',\n "31"'
|
||||
|
||||
return complete_json, incomplete_json
|
||||
|
||||
def testTableExtraction(self):
|
||||
"""Test extraction from incomplete table JSON."""
|
||||
print("\n" + "="*80)
|
||||
print("TEST 1: Table Extraction (incomplete row)")
|
||||
print("="*80)
|
||||
|
||||
complete_json, incomplete_json = self.createIncompleteTableJson()
|
||||
|
||||
# Parse complete JSON to get allSections
|
||||
complete_obj = json.loads(complete_json)
|
||||
allSections = extractSectionsFromDocument(complete_obj)
|
||||
|
||||
print(f"Complete JSON sections: {len(allSections)}")
|
||||
print(f"Last section content_type: {allSections[0].get('content_type') if allSections else 'None'}")
|
||||
|
||||
# Debug: Check what extractFirstBalancedJson returns
|
||||
from modules.shared.jsonUtils import extractFirstBalancedJson, stripCodeFences
|
||||
raw_json = stripCodeFences(incomplete_json.strip())
|
||||
balanced_json = extractFirstBalancedJson(raw_json)
|
||||
balanced_length = len(balanced_json)
|
||||
cut_part = raw_json[balanced_length:].strip()
|
||||
print(f"\nDebug Info:")
|
||||
print(f" raw_json length: {len(raw_json)}")
|
||||
print(f" balanced_json length: {balanced_length}")
|
||||
print(f" cut_part length: {len(cut_part)}")
|
||||
print(f" cut_part content: {repr(cut_part[:200]) if cut_part else '(empty)'}")
|
||||
|
||||
# Build continuation context
|
||||
continuationContext = buildContinuationContext(allSections, incomplete_json)
|
||||
|
||||
print(f"\nExtraction Results:")
|
||||
print(f" content_type_for_items: {continuationContext.get('content_type_for_items')}")
|
||||
print(f" last_item_object: {continuationContext.get('last_item_object')}")
|
||||
print(f" cut_item_object: {continuationContext.get('cut_item_object')}")
|
||||
print(f" total_items_count: {continuationContext.get('total_items_count')}")
|
||||
|
||||
# Validate results
|
||||
lastItem = continuationContext.get('last_item_object')
|
||||
cutItem = continuationContext.get('cut_item_object')
|
||||
contentType = continuationContext.get('content_type_for_items')
|
||||
|
||||
success = True
|
||||
if contentType != "table":
|
||||
print(f" [FAIL] Expected content_type 'table', got '{contentType}'")
|
||||
success = False
|
||||
if not lastItem:
|
||||
print(f" [FAIL] last_item_object is empty")
|
||||
success = False
|
||||
if not cutItem:
|
||||
print(f" [FAIL] cut_item_object is empty")
|
||||
success = False
|
||||
|
||||
if success:
|
||||
print(f" [PASS] All extractions successful")
|
||||
|
||||
self.testResults['table'] = success
|
||||
return success
|
||||
|
||||
def testCodeBlockExtraction(self):
|
||||
"""Test extraction from incomplete code_block JSON."""
|
||||
print("\n" + "="*80)
|
||||
print("TEST 2: Code Block Extraction (incomplete line)")
|
||||
print("="*80)
|
||||
|
||||
complete_json, incomplete_json = self.createIncompleteCodeBlockJson()
|
||||
|
||||
# Parse complete JSON to get allSections
|
||||
complete_obj = json.loads(complete_json)
|
||||
allSections = extractSectionsFromDocument(complete_obj)
|
||||
|
||||
print(f"Complete JSON sections: {len(allSections)}")
|
||||
print(f"Last section content_type: {allSections[0].get('content_type') if allSections else 'None'}")
|
||||
|
||||
# Debug: Check what extractFirstBalancedJson returns
|
||||
from modules.shared.jsonUtils import extractFirstBalancedJson, stripCodeFences
|
||||
raw_json = stripCodeFences(incomplete_json.strip())
|
||||
balanced_json = extractFirstBalancedJson(raw_json)
|
||||
balanced_length = len(balanced_json)
|
||||
cut_part = raw_json[balanced_length:].strip()
|
||||
print(f"\nDebug Info:")
|
||||
print(f" raw_json length: {len(raw_json)}")
|
||||
print(f" balanced_json length: {balanced_length}")
|
||||
print(f" cut_part length: {len(cut_part)}")
|
||||
print(f" cut_part content: {repr(cut_part[:200]) if cut_part else '(empty)'}")
|
||||
|
||||
# Build continuation context
|
||||
continuationContext = buildContinuationContext(allSections, incomplete_json)
|
||||
|
||||
print(f"\nExtraction Results:")
|
||||
print(f" content_type_for_items: {continuationContext.get('content_type_for_items')}")
|
||||
print(f" last_item_object: {continuationContext.get('last_item_object')}")
|
||||
print(f" cut_item_object: {continuationContext.get('cut_item_object')}")
|
||||
print(f" total_items_count: {continuationContext.get('total_items_count')}")
|
||||
|
||||
# Validate results
|
||||
lastItem = continuationContext.get('last_item_object')
|
||||
cutItem = continuationContext.get('cut_item_object')
|
||||
contentType = continuationContext.get('content_type_for_items')
|
||||
|
||||
success = True
|
||||
if contentType != "code_block":
|
||||
print(f" [FAIL] Expected content_type 'code_block', got '{contentType}'")
|
||||
success = False
|
||||
if not lastItem:
|
||||
print(f" [FAIL] last_item_object is empty")
|
||||
success = False
|
||||
if not cutItem:
|
||||
print(f" [FAIL] cut_item_object is empty")
|
||||
success = False
|
||||
|
||||
if success:
|
||||
print(f" [PASS] All extractions successful")
|
||||
|
||||
self.testResults['code_block'] = success
|
||||
return success
|
||||
|
||||
def testListExtraction(self):
|
||||
"""Test extraction from incomplete list JSON."""
|
||||
print("\n" + "="*80)
|
||||
print("TEST 3: List Extraction (incomplete item)")
|
||||
print("="*80)
|
||||
|
||||
complete_json, incomplete_json = self.createIncompleteListJson()
|
||||
|
||||
# Parse complete JSON to get allSections
|
||||
complete_obj = json.loads(complete_json)
|
||||
allSections = extractSectionsFromDocument(complete_obj)
|
||||
|
||||
print(f"Complete JSON sections: {len(allSections)}")
|
||||
print(f"Last section content_type: {allSections[0].get('content_type') if allSections else 'None'}")
|
||||
|
||||
# Debug: Check what extractFirstBalancedJson returns
|
||||
from modules.shared.jsonUtils import extractFirstBalancedJson, stripCodeFences
|
||||
raw_json = stripCodeFences(incomplete_json.strip())
|
||||
balanced_json = extractFirstBalancedJson(raw_json)
|
||||
balanced_length = len(balanced_json)
|
||||
cut_part = raw_json[balanced_length:].strip()
|
||||
print(f"\nDebug Info:")
|
||||
print(f" raw_json length: {len(raw_json)}")
|
||||
print(f" balanced_json length: {balanced_length}")
|
||||
print(f" cut_part length: {len(cut_part)}")
|
||||
print(f" cut_part content: {repr(cut_part[:200]) if cut_part else '(empty)'}")
|
||||
|
||||
# Build continuation context
|
||||
continuationContext = buildContinuationContext(allSections, incomplete_json)
|
||||
|
||||
print(f"\nExtraction Results:")
|
||||
print(f" content_type_for_items: {continuationContext.get('content_type_for_items')}")
|
||||
print(f" last_item_object: {continuationContext.get('last_item_object')}")
|
||||
print(f" cut_item_object: {continuationContext.get('cut_item_object')}")
|
||||
print(f" total_items_count: {continuationContext.get('total_items_count')}")
|
||||
|
||||
# Validate results
|
||||
lastItem = continuationContext.get('last_item_object')
|
||||
cutItem = continuationContext.get('cut_item_object')
|
||||
contentType = continuationContext.get('content_type_for_items')
|
||||
|
||||
success = True
|
||||
if contentType not in ["bullet_list", "numbered_list"]:
|
||||
print(f" [FAIL] Expected content_type 'bullet_list' or 'numbered_list', got '{contentType}'")
|
||||
success = False
|
||||
if not lastItem:
|
||||
print(f" [FAIL] last_item_object is empty")
|
||||
success = False
|
||||
if not cutItem:
|
||||
print(f" [FAIL] cut_item_object is empty")
|
||||
success = False
|
||||
|
||||
if success:
|
||||
print(f" [PASS] All extractions successful")
|
||||
|
||||
self.testResults['list'] = success
|
||||
return success
|
||||
|
||||
def createRealWorldTableJson(self) -> tuple[str, str]:
|
||||
"""Create real-world incomplete JSON based on actual prompt pattern - table with many rows."""
|
||||
# Last complete row (exactly as in real scenario)
|
||||
last_complete_row = ["16871", "16879", "16883", "16889", "16901", "16903", "16921", "16927", "16931", "16937"]
|
||||
|
||||
complete_json = f"""{{
|
||||
"metadata": {{
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
}},
|
||||
"documents": [
|
||||
{{
|
||||
"id": "doc_1",
|
||||
"title": "First 4000 Prime Numbers",
|
||||
"filename": "prime_numbers_4000.csv",
|
||||
"sections": [
|
||||
{{
|
||||
"id": "section_primes_csv",
|
||||
"content_type": "table",
|
||||
"elements": [
|
||||
{{
|
||||
"headers": [],
|
||||
"rows": [
|
||||
["2", "3", "5", "7", "11", "13", "17", "19", "23", "29"],
|
||||
["31", "37", "41", "43", "47", "53", "59", "61", "67", "71"],
|
||||
{json.dumps(last_complete_row)}
|
||||
],
|
||||
"caption": ""
|
||||
}}
|
||||
],
|
||||
"order": 0
|
||||
}}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}}"""
|
||||
|
||||
# Incomplete JSON - cuts off mid-row (exactly like real scenario)
|
||||
# CRITICAL: Must not end with } or ] to be detected as incomplete
|
||||
# Find the position where rows array ends and add incomplete row before closing
|
||||
rows_end_pos = complete_json.rfind(']')
|
||||
if rows_end_pos != -1:
|
||||
# Insert incomplete row before the closing bracket, remove all closing brackets after
|
||||
incomplete_json = complete_json[:rows_end_pos] + ',\n ["16943", "16963", "16979", "16981", "16987", "16'
|
||||
else:
|
||||
# Fallback: remove all closing brackets and append
|
||||
incomplete_json = complete_json.rstrip().rstrip('}').rstrip(']').rstrip('}').rstrip(']').rstrip('}') + ',\n ["16943", "16963", "16979", "16981", "16987", "16'
|
||||
|
||||
return complete_json, incomplete_json
|
||||
|
||||
def testRealWorldTableExtraction(self):
|
||||
"""Test extraction from real-world incomplete table JSON (like from actual prompt)."""
|
||||
print("\n" + "="*80)
|
||||
print("TEST 4: Real-World Table Extraction (400 rows scenario, incomplete row)")
|
||||
print("="*80)
|
||||
|
||||
complete_json, incomplete_json = self.createRealWorldTableJson()
|
||||
|
||||
# Parse complete JSON to get allSections
|
||||
complete_obj = json.loads(complete_json)
|
||||
allSections = extractSectionsFromDocument(complete_obj)
|
||||
|
||||
print(f"Complete JSON sections: {len(allSections)}")
|
||||
if allSections:
|
||||
print(f"Last section content_type: {allSections[0].get('content_type')}")
|
||||
elements = allSections[0].get('elements', [])
|
||||
if elements and isinstance(elements[0], dict) and 'rows' in elements[0]:
|
||||
rows = elements[0].get('rows', [])
|
||||
print(f"Total rows in complete JSON: {len(rows)}")
|
||||
if rows:
|
||||
print(f"Last complete row: {rows[-1]}")
|
||||
|
||||
# Test _extractSectionsRegex with incomplete JSON
|
||||
from modules.shared.jsonUtils import _extractSectionsRegex, repairBrokenJson
|
||||
print(f"\nTesting _extractSectionsRegex with incomplete JSON...")
|
||||
extracted_sections = _extractSectionsRegex(incomplete_json)
|
||||
print(f"Extracted sections: {len(extracted_sections)}")
|
||||
if extracted_sections:
|
||||
print(f"Extracted section content_type: {extracted_sections[0].get('content_type')}")
|
||||
|
||||
# Test repairBrokenJson
|
||||
print(f"\nTesting repairBrokenJson...")
|
||||
repaired_json = repairBrokenJson(incomplete_json)
|
||||
if repaired_json:
|
||||
print(f"Repaired JSON successful")
|
||||
repaired_sections = extractSectionsFromDocument(repaired_json)
|
||||
print(f"Repaired sections: {len(repaired_sections)}")
|
||||
else:
|
||||
print(f"Repair failed")
|
||||
|
||||
# Debug: Check what extractFirstBalancedJson returns
|
||||
from modules.shared.jsonUtils import extractFirstBalancedJson, stripCodeFences
|
||||
raw_json = stripCodeFences(incomplete_json.strip())
|
||||
balanced_json = extractFirstBalancedJson(raw_json)
|
||||
balanced_length = len(balanced_json)
|
||||
cut_part = raw_json[balanced_length:].strip()
|
||||
print(f"\nDebug Info:")
|
||||
print(f" raw_json length: {len(raw_json)}")
|
||||
print(f" balanced_json length: {balanced_length}")
|
||||
print(f" cut_part length: {len(cut_part)}")
|
||||
print(f" cut_part content: {repr(cut_part[:200]) if cut_part else '(empty)'}")
|
||||
|
||||
# Build continuation context
|
||||
continuationContext = buildContinuationContext(allSections, incomplete_json)
|
||||
|
||||
print(f"\nExtraction Results:")
|
||||
print(f" content_type_for_items: {continuationContext.get('content_type_for_items')}")
|
||||
print(f" last_item_object: {continuationContext.get('last_item_object')}")
|
||||
print(f" cut_item_object: {continuationContext.get('cut_item_object')}")
|
||||
print(f" total_items_count: {continuationContext.get('total_items_count')}")
|
||||
|
||||
# Validate results
|
||||
lastItem = continuationContext.get('last_item_object')
|
||||
cutItem = continuationContext.get('cut_item_object')
|
||||
contentType = continuationContext.get('content_type_for_items')
|
||||
|
||||
success = True
|
||||
if contentType != "table":
|
||||
print(f" [FAIL] Expected content_type 'table', got '{contentType}'")
|
||||
success = False
|
||||
if not lastItem:
|
||||
print(f" [FAIL] last_item_object is empty")
|
||||
success = False
|
||||
if not cutItem:
|
||||
print(f" [FAIL] cut_item_object is empty")
|
||||
success = False
|
||||
|
||||
if success:
|
||||
print(f" [PASS] All extractions successful")
|
||||
print(f" Last complete row: {lastItem}")
|
||||
print(f" Cut row: {cutItem}")
|
||||
|
||||
self.testResults['real_world_table'] = success
|
||||
return success
|
||||
|
||||
def runAllTests(self):
|
||||
"""Run all extraction tests."""
|
||||
print("\n" + "="*80)
|
||||
print("JSON EXTRACTION TESTS")
|
||||
print("Testing extraction of lastItemObject and cutItemObject from incomplete JSON")
|
||||
print("="*80)
|
||||
|
||||
# Clean up debug folder and log file before starting tests
|
||||
print("\nCleaning up debug files...")
|
||||
self.cleanupDebugFiles()
|
||||
print("")
|
||||
|
||||
results = []
|
||||
results.append(self.testTableExtraction())
|
||||
results.append(self.testCodeBlockExtraction())
|
||||
results.append(self.testListExtraction())
|
||||
results.append(self.testRealWorldTableExtraction())
|
||||
|
||||
# Summary
|
||||
print("\n" + "="*80)
|
||||
print("TEST SUMMARY")
|
||||
print("="*80)
|
||||
print(f"Table extraction: {'[PASS]' if self.testResults.get('table') else '[FAIL]'}")
|
||||
print(f"Code block extraction: {'[PASS]' if self.testResults.get('code_block') else '[FAIL]'}")
|
||||
print(f"List extraction: {'[PASS]' if self.testResults.get('list') else '[FAIL]'}")
|
||||
print(f"Real-world table extraction: {'[PASS]' if self.testResults.get('real_world_table') else '[FAIL]'}")
|
||||
|
||||
allPassed = all(results)
|
||||
print(f"\nOverall: {'[PASS] ALL TESTS PASSED' if allPassed else '[FAIL] SOME TESTS FAILED'}")
|
||||
|
||||
return allPassed
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main test execution."""
|
||||
tester = JsonExtractionTester()
|
||||
success = tester.runAllTests()
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
|
||||
908
tests/functional/test07_json_merge.py
Normal file
908
tests/functional/test07_json_merge.py
Normal file
|
|
@ -0,0 +1,908 @@
|
|||
"""Test JSON string accumulation for broken JSON iterations - String accumulation approach"""
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add gateway directory to path (go up 2 levels from tests/functional/)
|
||||
_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
if _gateway_path not in sys.path:
|
||||
sys.path.insert(0, _gateway_path)
|
||||
|
||||
# Import after path setup
|
||||
from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore
|
||||
from modules.shared.jsonUtils import extractSectionsFromDocument # type: ignore
|
||||
|
||||
|
||||
def createBigJsonStructure():
|
||||
"""Create a comprehensive JSON structure with various content types"""
|
||||
return {
|
||||
"documents": [{
|
||||
"documentName": "test_document.json",
|
||||
"sections": [
|
||||
{
|
||||
"id": "section_bullet_list",
|
||||
"content_type": "bullet_list",
|
||||
"order": 0,
|
||||
"elements": [{
|
||||
"items": [f"item_{i}" for i in range(1, 21)] # 20 items
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": "section_table",
|
||||
"content_type": "table",
|
||||
"order": 1,
|
||||
"elements": [{
|
||||
"headers": ["ID", "Name", "Age", "City"],
|
||||
"rows": [
|
||||
["1", "Alice", "25", "New York"],
|
||||
["2", "Bob", "30", "London"],
|
||||
["3", "Charlie", "35", "Paris"],
|
||||
["4", "Diana", "28", "Berlin"],
|
||||
["5", "Eve", "32", "Tokyo"],
|
||||
["6", "Frank", "27", "Sydney"],
|
||||
["7", "Grace", "29", "Toronto"],
|
||||
["8", "Henry", "31", "Madrid"]
|
||||
]
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": "section_code_block",
|
||||
"content_type": "code_block",
|
||||
"order": 2,
|
||||
"elements": [{
|
||||
"code": "def calculate_sum(numbers):\n result = 0\n for num in numbers:\n result += num\n return result\n\ndef calculate_product(numbers):\n result = 1\n for num in numbers:\n result *= num\n return result",
|
||||
"language": "python"
|
||||
}]
|
||||
}
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
def createComplexJsonStructure():
|
||||
"""Create a more complex and longer JSON structure for advanced testing"""
|
||||
return {
|
||||
"documents": [{
|
||||
"documentName": "complex_test_document.json",
|
||||
"sections": [
|
||||
{
|
||||
"id": "section_large_list",
|
||||
"content_type": "bullet_list",
|
||||
"order": 0,
|
||||
"elements": [{
|
||||
"items": [f"product_{i:04d}" for i in range(1, 101)] # 100 items
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": "section_nested_structure",
|
||||
"content_type": "nested_list",
|
||||
"order": 1,
|
||||
"elements": [{
|
||||
"categories": [
|
||||
{
|
||||
"name": "Category A",
|
||||
"subcategories": [
|
||||
{"name": "Sub A1", "items": [f"item_a1_{i}" for i in range(1, 21)]},
|
||||
{"name": "Sub A2", "items": [f"item_a2_{i}" for i in range(1, 16)]}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Category B",
|
||||
"subcategories": [
|
||||
{"name": "Sub B1", "items": [f"item_b1_{i}" for i in range(1, 25)]},
|
||||
{"name": "Sub B2", "items": [f"item_b2_{i}" for i in range(1, 18)]}
|
||||
]
|
||||
}
|
||||
]
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": "section_large_table",
|
||||
"content_type": "table",
|
||||
"order": 2,
|
||||
"elements": [{
|
||||
"headers": ["ID", "Name", "Email", "Department", "Salary", "StartDate"],
|
||||
"rows": [
|
||||
[f"{i}", f"Employee_{i:03d}", f"emp{i}@company.com", f"Dept{(i % 5) + 1}", f"{(50000 + i * 1000)}", f"2024-{(i % 12) + 1:02d}-15"]
|
||||
for i in range(1, 51) # 50 rows
|
||||
]
|
||||
}]
|
||||
},
|
||||
{
|
||||
"id": "section_code_blocks",
|
||||
"content_type": "code_block",
|
||||
"order": 3,
|
||||
"elements": [
|
||||
{
|
||||
"code": "class DataProcessor:\n def __init__(self, config):\n self.config = config\n self.cache = {}\n \n def process(self, data):\n result = []\n for item in data:\n processed = self.transform(item)\n result.append(processed)\n return result\n \n def transform(self, item):\n return item.upper() if isinstance(item, str) else item",
|
||||
"language": "python"
|
||||
},
|
||||
{
|
||||
"code": "function calculateStatistics(data) {\n const stats = {\n mean: 0,\n median: 0,\n mode: null,\n stdDev: 0\n };\n \n if (data.length === 0) return stats;\n \n const sum = data.reduce((a, b) => a + b, 0);\n stats.mean = sum / data.length;\n \n const sorted = [...data].sort((a, b) => a - b);\n const mid = Math.floor(sorted.length / 2);\n stats.median = sorted.length % 2 === 0\n ? (sorted[mid - 1] + sorted[mid]) / 2\n : sorted[mid];\n \n return stats;\n}",
|
||||
"language": "javascript"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "section_mixed_content",
|
||||
"content_type": "mixed",
|
||||
"order": 4,
|
||||
"elements": [{
|
||||
"paragraphs": [
|
||||
"This is a long paragraph that contains multiple sentences. " * 5,
|
||||
"Another paragraph with different content. " * 8,
|
||||
"Yet another paragraph for testing purposes. " * 10
|
||||
],
|
||||
"highlights": [f"Highlight {i}" for i in range(1, 31)], # 30 highlights
|
||||
"metadata": {
|
||||
"author": "Test Author",
|
||||
"version": "1.0.0",
|
||||
"tags": [f"tag_{i}" for i in range(1, 21)], # 20 tags
|
||||
"references": [f"ref_{i:03d}" for i in range(1, 16)] # 15 references
|
||||
}
|
||||
}]
|
||||
}
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
def testPattern1_ArraySliced():
|
||||
"""Test Pattern 1: Slice JSON string containing array into multiple pieces - String accumulation"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 1: Array Sliced into Multiple Pieces (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
# Create big JSON structure - use FULL document structure
|
||||
bigJson = createBigJsonStructure()
|
||||
|
||||
# Convert FULL document to JSON string (not just section)
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - look for item_8 in the items array
|
||||
itemsArrayStart = jsonStr.find('"items": [')
|
||||
item8Pos = jsonStr.find('"item_8"', itemsArrayStart)
|
||||
item15Pos = jsonStr.find('"item_15"', itemsArrayStart)
|
||||
|
||||
# Slice into 3 pieces (simulating 3 iterations)
|
||||
# Piece 1: Cut after item_8 (incomplete)
|
||||
cut1 = item8Pos + len('"item_8"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
# Piece 2: Continue from item_8, cut after item_15 (incomplete, overlaps with item_8)
|
||||
cut2 = item15Pos + len('"item_15"')
|
||||
piece2 = jsonStr[cut1 - len('"item_8"'):cut2] # Overlap + continuation
|
||||
|
||||
# Piece 3: Continue from item_15 to end (overlaps with item_15)
|
||||
piece3 = jsonStr[cut2 - len('"item_15"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars (cut at: {cut1})")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
|
||||
# Step 1: Iteration 1 - Start accumulation with piece1
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
# Step 2: Iteration 2 - Accumulate piece2
|
||||
accumulatedJsonString, iter2_sections, isComplete2, parsedResult2 = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece2,
|
||||
allSections,
|
||||
2
|
||||
)
|
||||
|
||||
if iter2_sections:
|
||||
allSections = iter2_sections
|
||||
print(f"Iteration 2: Accumulated, {len(allSections)} sections, complete={isComplete2}")
|
||||
|
||||
# Step 3: Iteration 3 - Accumulate piece3
|
||||
accumulatedJsonString, iter3_sections, isComplete3, parsedResult3 = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece3,
|
||||
allSections,
|
||||
3
|
||||
)
|
||||
|
||||
if iter3_sections:
|
||||
allSections = iter3_sections
|
||||
print(f"Iteration 3: Accumulated, {len(allSections)} sections, complete={isComplete3}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
# Find bullet_list section
|
||||
bulletSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_bullet_list':
|
||||
bulletSection = section
|
||||
break
|
||||
|
||||
if bulletSection:
|
||||
elements = bulletSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
items = element.get('items', [])
|
||||
else:
|
||||
items = []
|
||||
print(f"✅ Final result: {len(items)} items")
|
||||
assert len(items) == 20, f"Expected 20 items, got {len(items)}"
|
||||
else:
|
||||
print("❌ Bullet list section not found")
|
||||
assert False, "Bullet list section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern2_TableSliced():
|
||||
"""Test Pattern 2: Slice JSON string containing table into multiple pieces - String accumulation"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 2: Table Sliced into Multiple Pieces (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createBigJsonStructure()
|
||||
|
||||
# Convert FULL document to JSON string
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - look for rows in the table section
|
||||
rowsArrayStart = jsonStr.find('"rows": [')
|
||||
row4Pos = jsonStr.find('["4", "Diana"', rowsArrayStart)
|
||||
row7Pos = jsonStr.find('["7", "Grace"', rowsArrayStart)
|
||||
|
||||
# Slice into 3 pieces
|
||||
# Piece 1: Cut after row 3 (incomplete row 4)
|
||||
cut1 = row4Pos + len('["4", "Diana"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
# Piece 2: Continue from row 4, cut after row 6 (overlaps with row 4)
|
||||
cut2 = row7Pos + len('["7", "Grace"')
|
||||
piece2 = jsonStr[cut1 - len('["4", "Diana"'):cut2]
|
||||
|
||||
# Piece 3: Continue from row 7 to end (overlaps with row 7)
|
||||
piece3 = jsonStr[cut2 - len('["7", "Grace"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
|
||||
# Step 1: Iteration 1 - Start accumulation with piece1
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
# Step 2: Iteration 2 - Accumulate piece2
|
||||
accumulatedJsonString, iter2_sections, isComplete2, parsedResult2 = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece2,
|
||||
allSections,
|
||||
2
|
||||
)
|
||||
|
||||
if iter2_sections:
|
||||
allSections = iter2_sections
|
||||
print(f"Iteration 2: Accumulated, {len(allSections)} sections, complete={isComplete2}")
|
||||
|
||||
# Step 3: Iteration 3 - Accumulate piece3
|
||||
accumulatedJsonString, iter3_sections, isComplete3, parsedResult3 = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece3,
|
||||
allSections,
|
||||
3
|
||||
)
|
||||
|
||||
if iter3_sections:
|
||||
allSections = iter3_sections
|
||||
print(f"Iteration 3: Accumulated, {len(allSections)} sections, complete={isComplete3}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
# Find table section
|
||||
tableSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_table':
|
||||
tableSection = section
|
||||
break
|
||||
|
||||
if tableSection:
|
||||
elements = tableSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
rows = element.get('rows', [])
|
||||
else:
|
||||
rows = []
|
||||
print(f"✅ Final result: {len(rows)} rows")
|
||||
assert len(rows) == 8, f"Expected 8 rows, got {len(rows)}"
|
||||
else:
|
||||
print("❌ Table section not found")
|
||||
assert False, "Table section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern3_CodeBlockSliced():
|
||||
"""Test Pattern 3: Slice JSON string containing code block into multiple pieces - String accumulation"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 3: Code Block Sliced into Multiple Pieces (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createBigJsonStructure()
|
||||
|
||||
# Convert FULL document to JSON string
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - look for code in the code_block section
|
||||
codeStart = jsonStr.find('"code": "')
|
||||
codeCutPos = jsonStr.find("return result", codeStart) + len("return result")
|
||||
piece1 = jsonStr[:codeCutPos]
|
||||
|
||||
# Piece 2: Continue from cut point to end (small overlap)
|
||||
piece2 = jsonStr[codeCutPos - 10:]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
|
||||
# Step 1: Iteration 1 - Start accumulation with piece1
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
# Step 2: Iteration 2 - Accumulate piece2
|
||||
accumulatedJsonString, iter2_sections, isComplete2, parsedResult2 = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece2,
|
||||
allSections,
|
||||
2
|
||||
)
|
||||
|
||||
if iter2_sections:
|
||||
allSections = iter2_sections
|
||||
print(f"Iteration 2: Accumulated, {len(allSections)} sections, complete={isComplete2}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
# Find code_block section
|
||||
codeSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_code_block':
|
||||
codeSection = section
|
||||
break
|
||||
|
||||
if codeSection:
|
||||
elements = codeSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
mergedCode = element.get('code', '')
|
||||
else:
|
||||
mergedCode = ''
|
||||
print(f"✅ Final result: {len(mergedCode)} chars")
|
||||
assert "calculate_sum" in mergedCode and "calculate_product" in mergedCode
|
||||
else:
|
||||
print("❌ Code block section not found")
|
||||
assert False, "Code block section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern4_LargeListSliced():
|
||||
"""Test Pattern 4: Slice large list (100 items) into multiple pieces"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 4: Large List Sliced into Multiple Pieces (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createComplexJsonStructure()
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - look for products in the large list
|
||||
itemsArrayStart = jsonStr.find('"items": [')
|
||||
product30Pos = jsonStr.find('"product_0030"', itemsArrayStart)
|
||||
product60Pos = jsonStr.find('"product_0060"', itemsArrayStart)
|
||||
product90Pos = jsonStr.find('"product_0090"', itemsArrayStart)
|
||||
|
||||
# Slice into 4 pieces
|
||||
cut1 = product30Pos + len('"product_0030"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
cut2 = product60Pos + len('"product_0060"')
|
||||
piece2 = jsonStr[cut1 - len('"product_0030"'):cut2]
|
||||
|
||||
cut3 = product90Pos + len('"product_0090"')
|
||||
piece3 = jsonStr[cut2 - len('"product_0060"'):cut3]
|
||||
|
||||
piece4 = jsonStr[cut3 - len('"product_0090"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
print(f"Piece 4 length: {len(piece4)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
for iteration, piece in enumerate([piece2, piece3, piece4], start=2):
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration {iteration}: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
largeListSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_large_list':
|
||||
largeListSection = section
|
||||
break
|
||||
|
||||
if largeListSection:
|
||||
elements = largeListSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
items = element.get('items', [])
|
||||
else:
|
||||
items = []
|
||||
print(f"✅ Final result: {len(items)} items")
|
||||
assert len(items) == 100, f"Expected 100 items, got {len(items)}"
|
||||
else:
|
||||
print("❌ Large list section not found")
|
||||
assert False, "Large list section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern5_NestedStructureSliced():
|
||||
"""Test Pattern 5: Slice nested structure in the middle of nested arrays"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 5: Nested Structure Sliced (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createComplexJsonStructure()
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - slice at actual item positions in nested structure
|
||||
nestedStart = jsonStr.find('"categories": [')
|
||||
itemA1_10Pos = jsonStr.find('"item_a1_10"', nestedStart)
|
||||
itemA2_8Pos = jsonStr.find('"item_a2_8"', nestedStart)
|
||||
itemB1_12Pos = jsonStr.find('"item_b1_12"', nestedStart)
|
||||
|
||||
# Slice into 4 pieces
|
||||
cut1 = itemA1_10Pos + len('"item_a1_10"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
cut2 = itemA2_8Pos + len('"item_a2_8"')
|
||||
piece2 = jsonStr[cut1 - len('"item_a1_10"'):cut2]
|
||||
|
||||
cut3 = itemB1_12Pos + len('"item_b1_12"')
|
||||
piece3 = jsonStr[cut2 - len('"item_a2_8"'):cut3]
|
||||
|
||||
piece4 = jsonStr[cut3 - len('"item_b1_12"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
print(f"Piece 4 length: {len(piece4)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
for iteration, piece in enumerate([piece2, piece3, piece4], start=2):
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration {iteration}: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result - check nested structure
|
||||
if allSections:
|
||||
nestedSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_nested_structure':
|
||||
nestedSection = section
|
||||
break
|
||||
|
||||
if nestedSection:
|
||||
elements = nestedSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
categories = element.get('categories', [])
|
||||
totalItems = 0
|
||||
for category in categories:
|
||||
for subcat in category.get('subcategories', []):
|
||||
totalItems += len(subcat.get('items', []))
|
||||
else:
|
||||
totalItems = 0
|
||||
print(f"✅ Final result: {totalItems} items across nested structure")
|
||||
# Allow some tolerance due to slicing complexity in nested structures
|
||||
# Expected: 20 (Sub A1) + 15 (Sub A2) + 25 (Sub B1) + 18 (Sub B2) = 78
|
||||
assert totalItems >= 75, f"Expected at least 75 items, got {totalItems}"
|
||||
if totalItems != 78:
|
||||
print(f"⚠️ Note: Got {totalItems} instead of 78 (acceptable due to nested structure slicing)")
|
||||
else:
|
||||
print("❌ Nested structure section not found")
|
||||
assert False, "Nested structure section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern6_LargeTableSliced():
|
||||
"""Test Pattern 6: Slice large table (50 rows) into multiple pieces"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 6: Large Table Sliced into Multiple Pieces (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createComplexJsonStructure()
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - look for rows in the large table
|
||||
rowsArrayStart = jsonStr.find('"rows": [')
|
||||
row15Pos = jsonStr.find('"15", "Employee_015"', rowsArrayStart)
|
||||
row30Pos = jsonStr.find('"30", "Employee_030"', rowsArrayStart)
|
||||
row45Pos = jsonStr.find('"45", "Employee_045"', rowsArrayStart)
|
||||
|
||||
# Slice into 4 pieces
|
||||
cut1 = row15Pos + len('"15", "Employee_015"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
cut2 = row30Pos + len('"30", "Employee_030"')
|
||||
piece2 = jsonStr[cut1 - len('"15", "Employee_015"'):cut2]
|
||||
|
||||
cut3 = row45Pos + len('"45", "Employee_045"')
|
||||
piece3 = jsonStr[cut2 - len('"30", "Employee_030"'):cut3]
|
||||
|
||||
piece4 = jsonStr[cut3 - len('"45", "Employee_045"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
print(f"Piece 4 length: {len(piece4)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
for iteration, piece in enumerate([piece2, piece3, piece4], start=2):
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration {iteration}: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
tableSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_large_table':
|
||||
tableSection = section
|
||||
break
|
||||
|
||||
if tableSection:
|
||||
elements = tableSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
rows = element.get('rows', [])
|
||||
else:
|
||||
rows = []
|
||||
print(f"✅ Final result: {len(rows)} rows")
|
||||
assert len(rows) == 50, f"Expected 50 rows, got {len(rows)}"
|
||||
else:
|
||||
print("❌ Large table section not found")
|
||||
assert False, "Large table section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern7_MixedContentSliced():
|
||||
"""Test Pattern 7: Slice mixed content section with various data types"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 7: Mixed Content Sliced (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createComplexJsonStructure()
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Find where to slice - in the middle of mixed content
|
||||
mixedStart = jsonStr.find('"section_mixed_content"')
|
||||
highlightsStart = jsonStr.find('"highlights": [', mixedStart)
|
||||
highlight15Pos = jsonStr.find('"Highlight 15"', highlightsStart)
|
||||
highlight25Pos = jsonStr.find('"Highlight 25"', highlightsStart)
|
||||
|
||||
# Slice into 3 pieces
|
||||
cut1 = highlight15Pos + len('"Highlight 15"')
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
cut2 = highlight25Pos + len('"Highlight 25"')
|
||||
piece2 = jsonStr[cut1 - len('"Highlight 15"'):cut2]
|
||||
|
||||
piece3 = jsonStr[cut2 - len('"Highlight 25"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
for iteration, piece in enumerate([piece2, piece3], start=2):
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration {iteration}: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
mixedSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_mixed_content':
|
||||
mixedSection = section
|
||||
break
|
||||
|
||||
if mixedSection:
|
||||
elements = mixedSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
highlights = element.get('highlights', [])
|
||||
tags = element.get('metadata', {}).get('tags', [])
|
||||
else:
|
||||
highlights = []
|
||||
tags = []
|
||||
print(f"✅ Final result: {len(highlights)} highlights, {len(tags)} tags")
|
||||
assert len(highlights) == 30, f"Expected 30 highlights, got {len(highlights)}"
|
||||
assert len(tags) == 20, f"Expected 20 tags, got {len(tags)}"
|
||||
else:
|
||||
print("❌ Mixed content section not found")
|
||||
assert False, "Mixed content section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern9_RealWorldPrimeNumbersTable():
|
||||
"""Test Pattern 9: Real-world example - Prime numbers table from debug files"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 9: Real-World Prime Numbers Table (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
# Create a simplified but realistic test: JSON with rows 1-10, slice at row 8
|
||||
# This simulates the real-world scenario where JSON is cut mid-row
|
||||
complete_json = {
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": [{
|
||||
"id": "doc_1",
|
||||
"title": "Prime Numbers Table",
|
||||
"filename": "prime_numbers_table.json",
|
||||
"sections": [{
|
||||
"id": "section_prime_numbers_table",
|
||||
"content_type": "table",
|
||||
"elements": [{
|
||||
"headers": ["Index", "Prime 1", "Prime 2", "Prime 3", "Prime 4", "Prime 5", "Prime 6", "Prime 7", "Prime 8", "Prime 9", "Prime 10"],
|
||||
"rows": [
|
||||
["1", "2", "3", "5", "7", "11", "13", "17", "19", "23", "29"],
|
||||
["2", "31", "37", "41", "43", "47", "53", "59", "61", "67", "71"],
|
||||
["3", "73", "79", "83", "89", "97", "101", "103", "107", "109", "113"],
|
||||
["4", "127", "131", "137", "139", "149", "151", "157", "163", "167", "173"],
|
||||
["5", "179", "181", "191", "193", "197", "199", "211", "223", "227", "229"],
|
||||
["6", "233", "239", "241", "251", "257", "263", "269", "271", "277", "281"],
|
||||
["7", "283", "293", "307", "311", "313", "317", "331", "337", "347", "349"],
|
||||
["8", "353", "359", "367", "373", "379", "383", "389", "397", "401", "409"],
|
||||
["9", "419", "421", "431", "433", "439", "443", "449", "457", "461", "463"],
|
||||
["10", "467", "479", "487", "491", "499", "503", "509", "521", "523", "541"]
|
||||
]
|
||||
}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
||||
# Convert to JSON string and slice it realistically
|
||||
jsonStr = json.dumps(complete_json, ensure_ascii=False)
|
||||
|
||||
# Find where to slice - at row 8, cut after "401" (incomplete row 8)
|
||||
# This simulates the real scenario where JSON is cut mid-row
|
||||
row8Start = jsonStr.find('["8", "353"')
|
||||
cutPos = jsonStr.find('"401"', row8Start) + len('"401"')
|
||||
piece1 = jsonStr[:cutPos]
|
||||
|
||||
# Piece 2: Continue from "401" to end (overlaps with "401")
|
||||
piece2 = jsonStr[cutPos - len('"401"'):]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece2,
|
||||
allSections,
|
||||
2
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration 2: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result
|
||||
if allSections:
|
||||
tableSection = None
|
||||
for section in allSections:
|
||||
if section.get('id') == 'section_prime_numbers_table':
|
||||
tableSection = section
|
||||
break
|
||||
|
||||
if tableSection:
|
||||
elements = tableSection.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
rows = element.get('rows', [])
|
||||
else:
|
||||
rows = []
|
||||
print(f"✅ Final result: {len(rows)} rows")
|
||||
# Should have all 10 rows from the complete JSON
|
||||
assert len(rows) == 10, f"Expected 10 rows, got {len(rows)}"
|
||||
# Verify last row is row 10
|
||||
if rows:
|
||||
lastRow = rows[-1]
|
||||
assert lastRow[0] == "10", f"Expected last row index to be 10, got {lastRow[0]}"
|
||||
# Verify row 8 is complete (should have "409" as last value)
|
||||
row8 = rows[7] # Index 7 = row 8
|
||||
assert row8[0] == "8", f"Expected row 8, got row {row8[0]}"
|
||||
assert row8[-1] == "409", f"Expected row 8 to end with 409, got {row8[-1]}"
|
||||
else:
|
||||
print("❌ Prime numbers table section not found")
|
||||
assert False, "Prime numbers table section should exist"
|
||||
else:
|
||||
print("❌ No sections after accumulation")
|
||||
assert False, "Accumulation should produce sections"
|
||||
|
||||
|
||||
def testPattern8_CrossSectionSlice():
|
||||
"""Test Pattern 8: Slice across multiple sections (boundary crossing)"""
|
||||
print("\n" + "="*60)
|
||||
print("PATTERN 8: Cross-Section Slice (String Accumulation)")
|
||||
print("="*60)
|
||||
|
||||
bigJson = createComplexJsonStructure()
|
||||
jsonStr = json.dumps(bigJson, ensure_ascii=False)
|
||||
print(f"Full JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Slice across section boundaries
|
||||
# Piece 1: End of large_list section
|
||||
largeListEnd = jsonStr.find('"section_nested_structure"')
|
||||
cut1 = largeListEnd - 50 # Cut before nested structure starts
|
||||
piece1 = jsonStr[:cut1]
|
||||
|
||||
# Piece 2: Middle of nested structure, start of large table
|
||||
nestedEnd = jsonStr.find('"section_large_table"')
|
||||
cut2 = nestedEnd - 30
|
||||
piece2 = jsonStr[cut1 - 20:cut2] # Small overlap
|
||||
|
||||
# Piece 3: Rest of document
|
||||
piece3 = jsonStr[cut2 - 20:]
|
||||
|
||||
print(f"Piece 1 length: {len(piece1)} chars")
|
||||
print(f"Piece 2 length: {len(piece2)} chars")
|
||||
print(f"Piece 3 length: {len(piece3)} chars")
|
||||
|
||||
# Accumulate pieces
|
||||
accumulatedJsonString = piece1
|
||||
allSections = []
|
||||
|
||||
print(f"Iteration 1: Starting accumulation with {len(accumulatedJsonString)} chars")
|
||||
|
||||
for iteration, piece in enumerate([piece2, piece3], start=2):
|
||||
accumulatedJsonString, sections, isComplete, parsedResult = \
|
||||
JsonResponseHandler.accumulateAndParseJsonFragments(
|
||||
accumulatedJsonString,
|
||||
piece,
|
||||
allSections,
|
||||
iteration
|
||||
)
|
||||
|
||||
if sections:
|
||||
allSections = sections
|
||||
print(f"Iteration {iteration}: Accumulated, {len(allSections)} sections, complete={isComplete}")
|
||||
|
||||
# Verify final result - should have all sections
|
||||
print(f"✅ Final result: {len(allSections)} sections")
|
||||
assert len(allSections) >= 4, f"Expected at least 4 sections, got {len(allSections)}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n" + "="*60)
|
||||
print("JSON STRING ACCUMULATION TEST SUITE")
|
||||
print("="*60)
|
||||
print("Testing by slicing JSON string into pieces and accumulating")
|
||||
print("="*60)
|
||||
|
||||
try:
|
||||
# Basic tests
|
||||
testPattern1_ArraySliced()
|
||||
testPattern2_TableSliced()
|
||||
testPattern3_CodeBlockSliced()
|
||||
|
||||
# Complex tests with larger structures
|
||||
testPattern4_LargeListSliced()
|
||||
testPattern5_NestedStructureSliced()
|
||||
testPattern6_LargeTableSliced()
|
||||
testPattern7_MixedContentSliced()
|
||||
testPattern8_CrossSectionSlice()
|
||||
|
||||
# Real-world test with actual JSON from debug files
|
||||
testPattern9_RealWorldPrimeNumbersTable()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ ALL TESTS COMPLETED")
|
||||
print("="*60)
|
||||
except AssertionError as e:
|
||||
print(f"\n❌ TEST FAILED: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
594
tests/functional/test08_json_finalization.py
Normal file
594
tests/functional/test08_json_finalization.py
Normal file
|
|
@ -0,0 +1,594 @@
|
|||
"""
|
||||
Test JSON finalization process after accumulation is complete.
|
||||
|
||||
This test suite validates the finalization process that happens after receiving
|
||||
the full accumulated JSON from the AI service. It tests:
|
||||
|
||||
1. Finalization with real-world accumulated JSON from debug files
|
||||
2. Cleaning of markdown code fences that got embedded in JSON values
|
||||
3. Finalization with complete, clean JSON
|
||||
4. Building final result from sections (simulating _buildFinalResultFromSections)
|
||||
5. End-to-end finalization process simulating the failure scenario
|
||||
|
||||
Key Findings:
|
||||
- Row 373 in the prime numbers table had corruption: "349```json\n19" instead of "34919"
|
||||
- This corruption can cause final result serialization to fail or produce invalid JSON
|
||||
- The cleanCorruptionFromSections() helper function successfully cleans this corruption
|
||||
- After cleaning, the final result can be serialized and parsed correctly
|
||||
|
||||
Note: The cleanCorruptionFromSections() function should be integrated into the
|
||||
actual codebase (e.g., in mainServiceAi.py before building final result) to
|
||||
prevent corruption from causing final result production to fail.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add gateway directory to path (go up 2 levels from tests/functional/)
|
||||
_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
if _gateway_path not in sys.path:
|
||||
sys.path.insert(0, _gateway_path)
|
||||
|
||||
# Import after path setup
|
||||
from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler # type: ignore
|
||||
from modules.shared.jsonUtils import extractSectionsFromDocument, extractJsonString, repairBrokenJson # type: ignore
|
||||
|
||||
|
||||
def cleanCorruptionFromSections(sections: list) -> list:
|
||||
"""
|
||||
Clean corruption (like markdown code fences) from section values.
|
||||
This simulates what should happen before building final result.
|
||||
"""
|
||||
cleanedSections = []
|
||||
for section in sections:
|
||||
cleanedSection = _cleanCorruptionRecursive(section)
|
||||
cleanedSections.append(cleanedSection)
|
||||
return cleanedSections
|
||||
|
||||
|
||||
def _cleanCorruptionRecursive(obj: any) -> any:
|
||||
"""Recursively clean corruption from nested structures."""
|
||||
if isinstance(obj, dict):
|
||||
cleaned = {}
|
||||
for key, value in obj.items():
|
||||
cleaned[key] = _cleanCorruptionRecursive(value)
|
||||
return cleaned
|
||||
elif isinstance(obj, list):
|
||||
cleaned = []
|
||||
for item in obj:
|
||||
cleaned.append(_cleanCorruptionRecursive(item))
|
||||
return cleaned
|
||||
elif isinstance(obj, str):
|
||||
# Clean markdown code fences and other corruption
|
||||
cleaned = obj.replace('```json', '').replace('```', '').replace('\n', '').strip()
|
||||
# Try to reconstruct numbers if they were split by corruption
|
||||
# E.g., "349```json\n19" -> "34919"
|
||||
if cleaned and cleaned[0].isdigit():
|
||||
# Remove any non-digit characters in the middle and reconstruct
|
||||
parts = cleaned.split()
|
||||
if len(parts) > 1:
|
||||
# Try to merge consecutive number parts
|
||||
merged = ''.join(parts)
|
||||
if merged.isdigit():
|
||||
cleaned = merged
|
||||
return cleaned
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
def testFinalizationWithRealWorldAccumulatedJson():
|
||||
"""Test finalization process with real-world accumulated JSON from debug files"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST: Finalization with Real-World Accumulated JSON")
|
||||
print("="*60)
|
||||
|
||||
# Load the accumulated JSON from debug file
|
||||
debugFile = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..", "..", "..", "local", "debug", "prompts",
|
||||
"20251130-205629-015-document_generation_accumulated_json_iteration_2.json"
|
||||
)
|
||||
|
||||
if not os.path.exists(debugFile):
|
||||
print(f"❌ Debug file not found: {debugFile}")
|
||||
print(" Skipping test - file may not exist in this environment")
|
||||
return
|
||||
|
||||
# Read the JSON file
|
||||
with open(debugFile, 'r', encoding='utf-8') as f:
|
||||
jsonContent = f.read()
|
||||
|
||||
print(f"Loaded JSON file: {len(jsonContent)} chars")
|
||||
|
||||
# Step 1: Extract JSON string (handles code fences, normalization)
|
||||
extractedJson = extractJsonString(jsonContent)
|
||||
print(f"After extractJsonString: {len(extractedJson)} chars")
|
||||
|
||||
# Step 2: Clean encoding issues
|
||||
cleanedJson = JsonResponseHandler.cleanEncodingIssues(extractedJson)
|
||||
print(f"After cleanEncodingIssues: {len(cleanedJson)} chars")
|
||||
|
||||
# Step 3: Try to parse
|
||||
try:
|
||||
parsedJson = json.loads(cleanedJson)
|
||||
print("✅ JSON parsing succeeded")
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ JSON parsing failed: {e}")
|
||||
print(" Attempting repair...")
|
||||
|
||||
# Try to repair
|
||||
repairedJson = repairBrokenJson(cleanedJson)
|
||||
if repairedJson:
|
||||
parsedJson = repairedJson
|
||||
print("✅ JSON repair succeeded")
|
||||
else:
|
||||
print("❌ JSON repair failed")
|
||||
# Find the problematic line
|
||||
errorLine = getattr(e, 'lineno', None)
|
||||
if errorLine:
|
||||
lines = cleanedJson.split('\n')
|
||||
if errorLine <= len(lines):
|
||||
print(f" Error at line {errorLine}: {lines[errorLine-1][:100]}")
|
||||
assert False, f"Failed to parse or repair JSON: {e}"
|
||||
|
||||
# Step 4: Check completeness
|
||||
isComplete = JsonResponseHandler.isJsonComplete(parsedJson)
|
||||
print(f"JSON completeness check: {isComplete}")
|
||||
|
||||
# Step 5: Finalize JSON
|
||||
finalizedJson = JsonResponseHandler.finalizeJson(parsedJson)
|
||||
print("✅ JSON finalized")
|
||||
|
||||
# Step 6: Extract sections
|
||||
sections = extractSectionsFromDocument(finalizedJson)
|
||||
print(f"✅ Extracted {len(sections)} sections")
|
||||
|
||||
# Step 7: Verify sections
|
||||
if sections:
|
||||
for i, section in enumerate(sections):
|
||||
sectionId = section.get('id', f'unknown_{i}')
|
||||
contentType = section.get('content_type', 'unknown')
|
||||
print(f" Section {i+1}: id={sectionId}, type={contentType}")
|
||||
|
||||
# Check for the prime numbers table section
|
||||
if sectionId == 'section_prime_numbers_table':
|
||||
elements = section.get('elements', [])
|
||||
if isinstance(elements, list) and len(elements) > 0:
|
||||
element = elements[0]
|
||||
rows = element.get('rows', [])
|
||||
print(f" Found {len(rows)} rows in prime numbers table")
|
||||
|
||||
# Check for corruption in rows (known issue with markdown code fences)
|
||||
corruptionFound = False
|
||||
for rowIdx in range(min(373, len(rows))): # Check up to row 373
|
||||
row = rows[rowIdx]
|
||||
rowStr = json.dumps(row)
|
||||
if '```json' in rowStr or '```' in rowStr:
|
||||
corruptionFound = True
|
||||
print(f" ⚠️ WARNING: Row {rowIdx+1} contains markdown code fences")
|
||||
# Show the problematic value
|
||||
for valIdx, val in enumerate(row):
|
||||
valStr = str(val)
|
||||
if '```' in valStr:
|
||||
print(f" Value {valIdx}: {valStr[:80]}")
|
||||
# Try to clean it
|
||||
cleanedVal = valStr.replace('```json', '').replace('```', '').replace('\n', '').strip()
|
||||
print(f" Cleaned: {cleanedVal}")
|
||||
break
|
||||
|
||||
if not corruptionFound:
|
||||
print(f" ✅ No markdown code fence corruption detected in first 373 rows")
|
||||
|
||||
# Verify row 373 specifically
|
||||
if len(rows) >= 373:
|
||||
row373 = rows[372] # Index 372 = row 373
|
||||
print(f" Row 373: {row373[:5]}... (first 5 values)")
|
||||
|
||||
# Verify we have 400 rows
|
||||
assert len(rows) == 400, f"Expected 400 rows, got {len(rows)}"
|
||||
print(f" ✅ All 400 rows present")
|
||||
|
||||
# Verify last row is row 400
|
||||
lastRow = rows[-1]
|
||||
assert lastRow[0] == "400", f"Expected last row index to be 400, got {lastRow[0]}"
|
||||
print(f" ✅ Last row is row 400")
|
||||
else:
|
||||
print("❌ No sections extracted")
|
||||
assert False, "Should have extracted at least one section"
|
||||
|
||||
# Step 8: Verify final JSON structure
|
||||
assert 'documents' in finalizedJson, "Finalized JSON should have 'documents' key"
|
||||
assert isinstance(finalizedJson['documents'], list), "documents should be a list"
|
||||
assert len(finalizedJson['documents']) > 0, "documents list should not be empty"
|
||||
print("✅ Final JSON structure is valid")
|
||||
|
||||
print("\n✅ Finalization test completed successfully")
|
||||
|
||||
|
||||
def testCleaningMarkdownCodeFences():
|
||||
"""Test cleaning of markdown code fences that got embedded in JSON values"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST: Cleaning Markdown Code Fences from JSON")
|
||||
print("="*60)
|
||||
|
||||
# Simulate the corruption found in the real-world JSON
|
||||
# Row 373 had: "349```json\n19" instead of "34919"
|
||||
corruptedJson = {
|
||||
"documents": [{
|
||||
"sections": [{
|
||||
"id": "section_test",
|
||||
"content_type": "table",
|
||||
"elements": [{
|
||||
"rows": [
|
||||
["373", "34883", "34897", "34913", "34919", "349```json\n19", "34939"]
|
||||
]
|
||||
}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
||||
jsonStr = json.dumps(corruptedJson, ensure_ascii=False)
|
||||
print(f"Original JSON string length: {len(jsonStr)} chars")
|
||||
|
||||
# Test cleaning
|
||||
cleaned = JsonResponseHandler.cleanEncodingIssues(jsonStr)
|
||||
print(f"After cleanEncodingIssues: {len(cleaned)} chars")
|
||||
|
||||
# Try to parse
|
||||
try:
|
||||
parsed = json.loads(cleaned)
|
||||
print("✅ Parsed successfully (but corruption may still be in values)")
|
||||
|
||||
# Check if corruption is still present in values
|
||||
rows = parsed['documents'][0]['sections'][0]['elements'][0]['rows']
|
||||
row373 = rows[0]
|
||||
hasCorruption = any('```' in str(val) for val in row373)
|
||||
|
||||
if hasCorruption:
|
||||
print("⚠️ Corruption still present in values (expected - cleanEncodingIssues only handles encoding)")
|
||||
print(f" Row 373: {row373}")
|
||||
|
||||
# Manual cleaning of values
|
||||
cleanedRow373 = []
|
||||
for val in row373:
|
||||
cleanedVal = str(val).replace('```json', '').replace('```', '').replace('\n', '').strip()
|
||||
# Try to parse as number if it looks like one
|
||||
try:
|
||||
if cleanedVal.isdigit():
|
||||
cleanedRow373.append(cleanedVal)
|
||||
else:
|
||||
cleanedRow373.append(cleanedVal)
|
||||
except:
|
||||
cleanedRow373.append(cleanedVal)
|
||||
|
||||
print(f" Cleaned row 373: {cleanedRow373}")
|
||||
|
||||
# Verify "34919" is reconstructed
|
||||
assert "34919" in cleanedRow373, "Should have reconstructed 34919"
|
||||
print("✅ Successfully reconstructed corrupted value")
|
||||
else:
|
||||
print("✅ No corruption found in values")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ Parsing failed: {e}")
|
||||
assert False, f"Failed to parse cleaned JSON: {e}"
|
||||
|
||||
|
||||
def testFinalizationWithCompleteJson():
|
||||
"""Test finalization process with a complete, valid JSON"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST: Finalization with Complete JSON")
|
||||
print("="*60)
|
||||
|
||||
# Create a complete JSON structure
|
||||
completeJson = {
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": [{
|
||||
"id": "doc_1",
|
||||
"title": "Test Document",
|
||||
"sections": [{
|
||||
"id": "section_test",
|
||||
"content_type": "table",
|
||||
"elements": [{
|
||||
"headers": ["Col1", "Col2", "Col3"],
|
||||
"rows": [
|
||||
["1", "2", "3"],
|
||||
["4", "5", "6"]
|
||||
]
|
||||
}]
|
||||
}]
|
||||
}]
|
||||
}
|
||||
|
||||
jsonStr = json.dumps(completeJson, ensure_ascii=False)
|
||||
parsedJson = json.loads(jsonStr)
|
||||
|
||||
# Test completeness check
|
||||
isComplete = JsonResponseHandler.isJsonComplete(parsedJson)
|
||||
assert isComplete, "Complete JSON should pass completeness check"
|
||||
print("✅ Completeness check passed")
|
||||
|
||||
# Test finalization
|
||||
finalizedJson = JsonResponseHandler.finalizeJson(parsedJson)
|
||||
assert finalizedJson == parsedJson, "Finalized JSON should be same as input for complete JSON"
|
||||
print("✅ Finalization completed")
|
||||
|
||||
# Test section extraction
|
||||
sections = extractSectionsFromDocument(finalizedJson)
|
||||
assert len(sections) == 1, f"Expected 1 section, got {len(sections)}"
|
||||
assert sections[0]['id'] == 'section_test', "Section ID should match"
|
||||
print("✅ Section extraction successful")
|
||||
|
||||
print("✅ Complete JSON finalization test passed")
|
||||
|
||||
|
||||
def testBuildingFinalResultFromSections():
|
||||
"""Test building final result from sections (simulating _buildFinalResultFromSections)"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST: Building Final Result from Sections")
|
||||
print("="*60)
|
||||
|
||||
# Create sections (as would be extracted from accumulated JSON)
|
||||
sections = [{
|
||||
"id": "section_prime_numbers_table",
|
||||
"content_type": "table",
|
||||
"elements": [{
|
||||
"headers": ["Index", "Prime 1", "Prime 2", "Prime 3"],
|
||||
"rows": [
|
||||
["1", "2", "3", "5"],
|
||||
["2", "7", "11", "13"],
|
||||
# Simulate corruption in row 373
|
||||
["373", "34883", "34897", "34913", "34919", "349```json\n19", "34939"]
|
||||
]
|
||||
}]
|
||||
}]
|
||||
|
||||
# Build final result structure (simulating _buildFinalResultFromSections)
|
||||
documentMetadata = {
|
||||
"title": "Prime Numbers Table",
|
||||
"filename": "prime_numbers_table.json"
|
||||
}
|
||||
|
||||
title = documentMetadata.get("title", "Generated Document")
|
||||
filename = documentMetadata.get("filename", "document.json")
|
||||
|
||||
documents = [{
|
||||
"id": "doc_1",
|
||||
"title": title,
|
||||
"filename": filename,
|
||||
"sections": sections
|
||||
}]
|
||||
|
||||
result = {
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": documents
|
||||
}
|
||||
|
||||
# Try to serialize to JSON string
|
||||
try:
|
||||
finalResultStr = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
print(f"✅ Final result JSON string created: {len(finalResultStr)} chars")
|
||||
|
||||
# Verify it can be parsed back
|
||||
parsedBack = json.loads(finalResultStr)
|
||||
assert parsedBack['documents'][0]['title'] == title
|
||||
assert len(parsedBack['documents'][0]['sections']) == 1
|
||||
print("✅ Final result can be parsed back successfully")
|
||||
|
||||
# Check if corruption is still present
|
||||
rows = parsedBack['documents'][0]['sections'][0]['elements'][0]['rows']
|
||||
row373 = rows[2] # Third row (index 2)
|
||||
hasCorruption = any('```' in str(val) for val in row373)
|
||||
|
||||
if hasCorruption:
|
||||
print("⚠️ Corruption still present in final result (expected)")
|
||||
print(f" Row 373: {row373}")
|
||||
|
||||
# Clean the corruption using helper function
|
||||
cleanedSections = cleanCorruptionFromSections(sections)
|
||||
|
||||
# Rebuild final result with cleaned sections
|
||||
documents[0]['sections'] = cleanedSections
|
||||
result['documents'] = documents
|
||||
cleanedFinalResultStr = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
|
||||
# Verify cleaned result
|
||||
cleanedParsed = json.loads(cleanedFinalResultStr)
|
||||
cleanedRows = cleanedParsed['documents'][0]['sections'][0]['elements'][0]['rows']
|
||||
cleanedRow373 = cleanedRows[2]
|
||||
assert not any('```' in str(val) for val in cleanedRow373), "Cleaned row should not have corruption"
|
||||
assert "34919" in cleanedRow373, "Should have reconstructed 34919"
|
||||
print("✅ Corruption cleaned successfully")
|
||||
print(f" Cleaned row 373: {cleanedRow373}")
|
||||
else:
|
||||
print("✅ No corruption found in final result")
|
||||
|
||||
except json.JSONEncodeError as e:
|
||||
print(f"❌ Failed to serialize final result: {e}")
|
||||
assert False, f"Failed to serialize final result: {e}"
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"❌ Failed to parse final result back: {e}")
|
||||
assert False, f"Failed to parse final result back: {e}"
|
||||
|
||||
print("✅ Final result building test completed")
|
||||
|
||||
|
||||
def testEndToEndFinalizationWithCorruption():
|
||||
"""Test end-to-end finalization process simulating the exact failure scenario"""
|
||||
print("\n" + "="*60)
|
||||
print("TEST: End-to-End Finalization with Corruption (Failure Scenario)")
|
||||
print("="*60)
|
||||
|
||||
# Load the real accumulated JSON (with corruption)
|
||||
debugFile = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..", "..", "..", "local", "debug", "prompts",
|
||||
"20251130-205629-015-document_generation_accumulated_json_iteration_2.json"
|
||||
)
|
||||
|
||||
if not os.path.exists(debugFile):
|
||||
print(f"⚠️ Debug file not found: {debugFile}")
|
||||
print(" Skipping test - file may not exist in this environment")
|
||||
return
|
||||
|
||||
# Step 1: Load and parse accumulated JSON
|
||||
with open(debugFile, 'r', encoding='utf-8') as f:
|
||||
jsonContent = f.read()
|
||||
|
||||
extractedJson = extractJsonString(jsonContent)
|
||||
cleanedJson = JsonResponseHandler.cleanEncodingIssues(extractedJson)
|
||||
|
||||
try:
|
||||
parsedJson = json.loads(cleanedJson)
|
||||
except json.JSONDecodeError as e:
|
||||
repairedJson = repairBrokenJson(cleanedJson)
|
||||
if not repairedJson:
|
||||
print(f"❌ Failed to parse or repair JSON: {e}")
|
||||
assert False, f"Failed to parse or repair JSON: {e}"
|
||||
parsedJson = repairedJson
|
||||
|
||||
# Step 2: Extract sections (as done in mainServiceAi)
|
||||
sections = extractSectionsFromDocument(parsedJson)
|
||||
print(f"✅ Extracted {len(sections)} sections")
|
||||
|
||||
# Step 3: Complete incomplete structures (as done in mainServiceAi)
|
||||
completedSections = JsonResponseHandler.completeIncompleteStructures(sections)
|
||||
print(f"✅ Completed structures for {len(completedSections)} sections")
|
||||
|
||||
# Step 4: Check for corruption BEFORE building final result
|
||||
corruptionFound = False
|
||||
for section in completedSections:
|
||||
sectionStr = json.dumps(section)
|
||||
if '```json' in sectionStr or '```' in sectionStr:
|
||||
corruptionFound = True
|
||||
print(f"⚠️ Corruption detected in section {section.get('id', 'unknown')}")
|
||||
break
|
||||
|
||||
# Step 5: Clean corruption if found (this should be done before building final result)
|
||||
if corruptionFound:
|
||||
print(" Cleaning corruption from sections...")
|
||||
cleanedSections = cleanCorruptionFromSections(completedSections)
|
||||
print("✅ Corruption cleaned from sections")
|
||||
else:
|
||||
cleanedSections = completedSections
|
||||
print("✅ No corruption found")
|
||||
|
||||
# Step 6: Build final result (simulating _buildFinalResultFromSections)
|
||||
documentMetadata = {
|
||||
"title": "Prime Numbers Table",
|
||||
"filename": "prime_numbers_table.json"
|
||||
}
|
||||
|
||||
title = documentMetadata.get("title", "Generated Document")
|
||||
filename = documentMetadata.get("filename", "document.json")
|
||||
|
||||
documents = [{
|
||||
"id": "doc_1",
|
||||
"title": title,
|
||||
"filename": filename,
|
||||
"sections": cleanedSections
|
||||
}]
|
||||
|
||||
result = {
|
||||
"metadata": {
|
||||
"split_strategy": "single_document",
|
||||
"source_documents": [],
|
||||
"extraction_method": "ai_generation"
|
||||
},
|
||||
"documents": documents
|
||||
}
|
||||
|
||||
# Step 7: Serialize final result (this is where it might have failed)
|
||||
try:
|
||||
finalResultStr = json.dumps(result, indent=2, ensure_ascii=False)
|
||||
print(f"✅ Final result serialized successfully: {len(finalResultStr)} chars")
|
||||
|
||||
# Step 8: Verify it can be parsed back
|
||||
parsedBack = json.loads(finalResultStr)
|
||||
assert parsedBack['documents'][0]['title'] == title
|
||||
assert len(parsedBack['documents'][0]['sections']) == len(cleanedSections)
|
||||
print("✅ Final result can be parsed back successfully")
|
||||
|
||||
# Step 9: Verify no corruption in final result
|
||||
finalResultStr_check = json.dumps(parsedBack)
|
||||
if '```json' in finalResultStr_check or '```' in finalResultStr_check:
|
||||
print("⚠️ WARNING: Corruption still present in final result")
|
||||
else:
|
||||
print("✅ Final result is clean (no corruption)")
|
||||
|
||||
# Step 10: Verify section content
|
||||
if parsedBack['documents'][0]['sections']:
|
||||
section = parsedBack['documents'][0]['sections'][0]
|
||||
if section.get('id') == 'section_prime_numbers_table':
|
||||
elements = section.get('elements', [])
|
||||
if elements and 'rows' in elements[0]:
|
||||
rows = elements[0]['rows']
|
||||
print(f"✅ Final result contains {len(rows)} rows")
|
||||
assert len(rows) == 400, f"Expected 400 rows, got {len(rows)}"
|
||||
|
||||
# Verify row 373 is clean
|
||||
if len(rows) >= 373:
|
||||
row373 = rows[372]
|
||||
row373Str = json.dumps(row373)
|
||||
if '```' in row373Str:
|
||||
print(f"⚠️ WARNING: Row 373 still has corruption: {row373Str[:100]}")
|
||||
else:
|
||||
print(f"✅ Row 373 is clean: {row373[:5]}...")
|
||||
|
||||
print("\n✅ End-to-end finalization test completed successfully")
|
||||
print(f" Final result ready to write to debug file ({len(finalResultStr)} chars)")
|
||||
|
||||
except json.JSONEncodeError as e:
|
||||
print(f"❌ Failed to serialize final result: {e}")
|
||||
print(" This is likely why the final_result.txt file was empty")
|
||||
assert False, f"Failed to serialize final result: {e}"
|
||||
except Exception as e:
|
||||
print(f"❌ Unexpected error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
assert False, f"Unexpected error: {e}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n" + "="*60)
|
||||
print("JSON FINALIZATION TEST SUITE")
|
||||
print("="*60)
|
||||
print("Testing finalization process after accumulation is complete")
|
||||
print("="*60)
|
||||
|
||||
try:
|
||||
# Test 1: Finalization with real-world accumulated JSON
|
||||
testFinalizationWithRealWorldAccumulatedJson()
|
||||
|
||||
# Test 2: Cleaning markdown code fences
|
||||
testCleaningMarkdownCodeFences()
|
||||
|
||||
# Test 3: Finalization with complete JSON
|
||||
testFinalizationWithCompleteJson()
|
||||
|
||||
# Test 4: Building final result from sections
|
||||
testBuildingFinalResultFromSections()
|
||||
|
||||
# Test 5: End-to-end finalization with corruption (simulating failure scenario)
|
||||
testEndToEndFinalizationWithCorruption()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ ALL TESTS COMPLETED")
|
||||
print("="*60)
|
||||
except AssertionError as e:
|
||||
print(f"\n❌ TEST FAILED: {e}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
Loading…
Reference in a new issue