967 lines
41 KiB
Python
967 lines
41 KiB
Python
import json
|
||
import logging
|
||
from typing import Any, Dict, List, Optional, Tuple, Union, Type, TypeVar
|
||
from pydantic import BaseModel, ValidationError
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
T = TypeVar('T', bound=BaseModel)
|
||
|
||
|
||
def stripCodeFences(text: str) -> str:
|
||
"""Remove ```json / ``` fences and surrounding whitespace if present."""
|
||
if not text:
|
||
return text
|
||
s = text.strip()
|
||
if s.startswith("```") and s.endswith("```"):
|
||
# Remove first/last triple backticks
|
||
# Commonly starts with ```json\n
|
||
# Strip opening backticks
|
||
i = 3
|
||
# Skip optional language tag like 'json'
|
||
while i < len(s) and s[i] != '\n':
|
||
i += 1
|
||
if i < len(s) and s[i] == '\n':
|
||
s = s[i+1:]
|
||
# Strip trailing ```
|
||
if s.endswith("```"):
|
||
s = s[:-3]
|
||
return s.strip()
|
||
return s
|
||
|
||
|
||
def extractFirstBalancedJson(text: str) -> str:
|
||
"""Return the first balanced JSON object/array substring; otherwise return trimmed input."""
|
||
if not text:
|
||
return text
|
||
s = text.strip()
|
||
# Find first '{' or '['
|
||
brace = s.find('{')
|
||
bracket = s.find('[')
|
||
start = -1
|
||
if brace != -1 and (bracket == -1 or brace < bracket):
|
||
start = brace
|
||
elif bracket != -1:
|
||
start = bracket
|
||
if start == -1:
|
||
return s
|
||
# Scan for matching close using a simple stack
|
||
stack: List[str] = []
|
||
for i in range(start, len(s)):
|
||
ch = s[i]
|
||
if ch in '{[':
|
||
stack.append(ch)
|
||
elif ch in '}]':
|
||
if not stack:
|
||
continue
|
||
opener = stack.pop()
|
||
if (opener == '{' and ch != '}') or (opener == '[' and ch != ']'):
|
||
continue
|
||
if not stack:
|
||
return s[start:i+1].strip()
|
||
return s
|
||
|
||
|
||
def normalizeJsonText(text: str) -> str:
|
||
"""Light normalization: remove BOM, normalize smart quotes."""
|
||
if not text:
|
||
return text
|
||
s = text
|
||
# Remove UTF-8 BOM if present
|
||
if s.startswith('\ufeff'):
|
||
s = s.lstrip('\ufeff')
|
||
# Normalize smart quotes to straight quotes
|
||
s = s.replace('“', '"').replace('”', '"').replace('’', "'").replace('‘', "'")
|
||
return s
|
||
|
||
|
||
def extractJsonString(text: str) -> str:
|
||
"""Strip code fences, normalize, then extract first balanced JSON substring."""
|
||
s = normalizeJsonText(text)
|
||
s = stripCodeFences(s)
|
||
s = extractFirstBalancedJson(s)
|
||
return s.strip()
|
||
|
||
|
||
def tryParseJson(text: Union[str, bytes]) -> Tuple[Optional[Union[Dict, List]], Optional[Exception], str]:
|
||
"""Extract and parse JSON; return (obj, error, cleaned_str)."""
|
||
if isinstance(text, bytes):
|
||
try:
|
||
text = text.decode('utf-8', errors='replace')
|
||
except Exception:
|
||
text = str(text)
|
||
cleaned = extractJsonString(text or "")
|
||
try:
|
||
return json.loads(cleaned), None, cleaned
|
||
except Exception as e:
|
||
return None, e, cleaned
|
||
|
||
|
||
def parseJsonOrRaise(text: Union[str, bytes]) -> Union[Dict, List]:
|
||
obj, err, cleaned = tryParseJson(text)
|
||
if err is not None:
|
||
logger.error(f"parse_json_or_raise failed: {err}. Cleaned preview: {cleaned[:200]}...")
|
||
raise err
|
||
return obj
|
||
|
||
|
||
def mergeRootLists(jsonParts: List[Union[str, Dict, List]]) -> Dict[str, Any]:
|
||
"""
|
||
Generic merger for root-level lists: take first dict as base; for each subsequent part:
|
||
- if value is list and same key exists as list, extend it
|
||
- if key absent, add it
|
||
- for non-list keys, keep the original (from the first part)
|
||
Sets continuation=None if present in base.
|
||
"""
|
||
base: Optional[Dict[str, Any]] = None
|
||
parsed: List[Dict[str, Any]] = []
|
||
for part in jsonParts:
|
||
if isinstance(part, (dict, list)):
|
||
obj = part
|
||
else:
|
||
obj, err, _ = tryParseJson(part)
|
||
if err is not None or not isinstance(obj, (dict, list)):
|
||
continue
|
||
if isinstance(obj, dict):
|
||
parsed.append(obj)
|
||
if not parsed:
|
||
return {}
|
||
base = dict(parsed[0])
|
||
for obj in parsed[1:]:
|
||
for k, v in obj.items():
|
||
if isinstance(v, list) and isinstance(base.get(k), list):
|
||
base[k].extend(v)
|
||
elif k not in base:
|
||
base[k] = v
|
||
if 'continuation' in base:
|
||
base['continuation'] = None
|
||
return base
|
||
|
||
|
||
def repairBrokenJson(text: str) -> Optional[Dict[str, Any]]:
|
||
"""
|
||
Attempt to repair broken JSON using multiple strategies.
|
||
Generic solution that works for any content type.
|
||
Returns the best repair attempt or None if all fail.
|
||
"""
|
||
if not text:
|
||
return None
|
||
|
||
# Strategy 1: Try to extract sections from the entire text first
|
||
# This handles cases where the JSON structure is broken but content is intact
|
||
extractedSections = _extractSectionsRegex(text)
|
||
if extractedSections:
|
||
logger.info(f"Extracted {len(extractedSections)} sections using regex")
|
||
return {
|
||
"metadata": {
|
||
"split_strategy": "single_document",
|
||
"source_documents": [],
|
||
"extraction_method": "ai_generation"
|
||
},
|
||
"documents": [{"sections": extractedSections}]
|
||
}
|
||
|
||
# Strategy 2: Progressive parsing - try to find longest valid prefix
|
||
bestResult = None
|
||
bestValidLength = 0
|
||
|
||
# Try different step sizes to find the best valid JSON
|
||
for stepSize in [100, 50, 10, 1]:
|
||
for i in range(len(text), 0, -stepSize):
|
||
testStr = text[:i]
|
||
closedStr = _closeJsonStructures(testStr)
|
||
obj, err, _ = tryParseJson(closedStr)
|
||
if err is None and isinstance(obj, dict):
|
||
bestResult = obj
|
||
bestValidLength = i
|
||
logger.debug(f"Progressive parsing success at length {i} (step: {stepSize})")
|
||
break
|
||
if bestResult:
|
||
break
|
||
|
||
if bestResult:
|
||
logger.info(f"Repaired JSON using progressive parsing (valid length: {bestValidLength})")
|
||
|
||
# Check if we have sections in the result
|
||
sections = extractSectionsFromDocument(bestResult)
|
||
if sections:
|
||
logger.info(f"Progressive parsing found {len(sections)} sections")
|
||
return bestResult
|
||
else:
|
||
# No sections found in progressive parsing, try to extract from broken part
|
||
logger.info("Progressive parsing found no sections, trying to extract from broken part")
|
||
extractedSections = _extractSectionsRegex(text[bestValidLength:])
|
||
if extractedSections:
|
||
logger.info(f"Extracted {len(extractedSections)} sections from broken part")
|
||
# Merge with the valid part
|
||
if "documents" not in bestResult:
|
||
bestResult["documents"] = []
|
||
if not bestResult["documents"]:
|
||
bestResult["documents"] = [{"sections": []}]
|
||
bestResult["documents"][0]["sections"].extend(extractedSections)
|
||
return bestResult
|
||
|
||
# Strategy 3: Structure closing - close incomplete structures
|
||
closedStr = _closeJsonStructures(text)
|
||
obj, err, _ = tryParseJson(closedStr)
|
||
if err is None and isinstance(obj, dict):
|
||
logger.info("Repaired JSON using structure closing")
|
||
return obj
|
||
|
||
logger.warning("All repair strategies failed")
|
||
return None
|
||
|
||
|
||
def _closeJsonStructures(text: str) -> str:
|
||
"""
|
||
Close incomplete JSON structures by adding missing closing brackets.
|
||
"""
|
||
if not text:
|
||
return text
|
||
|
||
# Count open/close brackets and braces
|
||
openBraces = text.count('{')
|
||
closeBraces = text.count('}')
|
||
openBrackets = text.count('[')
|
||
closeBrackets = text.count(']')
|
||
|
||
# Close incomplete structures
|
||
result = text
|
||
for _ in range(openBraces - closeBraces):
|
||
result += '}'
|
||
for _ in range(openBrackets - closeBrackets):
|
||
result += ']'
|
||
|
||
return result
|
||
|
||
|
||
def _extractSectionsRegex(text: str) -> List[Dict[str, Any]]:
|
||
"""
|
||
Extract sections from broken JSON using regex patterns.
|
||
Generic solution that works for any content type.
|
||
"""
|
||
import re
|
||
|
||
sections = []
|
||
|
||
# Pattern to find section objects
|
||
sectionPattern = r'"id"\s*:\s*"(section_\d+)"\s*,?\s*"content_type"\s*:\s*"(\w+)"\s*,?\s*"order"\s*:\s*(\d+)'
|
||
|
||
for match in re.finditer(sectionPattern, text, re.IGNORECASE):
|
||
sectionId = match.group(1)
|
||
contentType = match.group(2)
|
||
order = int(match.group(3))
|
||
|
||
# Try to extract elements array - look for the elements array after this section
|
||
elementsMatch = re.search(
|
||
r'"elements"\s*:\s*\[(.*?)\]',
|
||
text[match.end():match.end()+5000] # Look ahead for elements (large range)
|
||
)
|
||
|
||
elements = []
|
||
if elementsMatch:
|
||
try:
|
||
elementsStr = '[' + elementsMatch.group(1) + ']'
|
||
elements = json.loads(elementsStr)
|
||
except:
|
||
# If JSON parsing fails, try to extract individual items manually
|
||
elementsText = elementsMatch.group(1)
|
||
elements = _extractElementsFromText(elementsText, contentType)
|
||
|
||
sections.append({
|
||
"id": sectionId,
|
||
"content_type": contentType,
|
||
"elements": elements,
|
||
"order": order
|
||
})
|
||
|
||
# If no sections found with the main pattern, try to find any content patterns
|
||
if not sections:
|
||
sections = _extractGenericContent(text)
|
||
|
||
return sections
|
||
|
||
|
||
def _extractElementsFromText(elementsText: str, contentType: str) -> List[Dict[str, Any]]:
|
||
"""
|
||
Extract elements from text when JSON parsing fails.
|
||
Generic approach that works for any content type.
|
||
Handles incomplete strings and corrupted data.
|
||
Excludes the last incomplete item to prevent corrupted data.
|
||
"""
|
||
import re
|
||
|
||
elements = []
|
||
|
||
if contentType == "list":
|
||
# Look for {"text": "..."} patterns, including incomplete ones
|
||
text_items = re.findall(r'\{"text"\s*:\s*"([^"]*)"\}', elementsText)
|
||
# Also look for incomplete patterns like {"text": "36
|
||
incomplete_items = re.findall(r'\{"text"\s*:\s*"([^"]*?)(?:\n|$)', elementsText)
|
||
|
||
# Combine both complete and incomplete items
|
||
all_items = text_items + incomplete_items
|
||
# Remove duplicates and empty strings
|
||
unique_items = list(dict.fromkeys([item for item in all_items if item.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_items:
|
||
unique_items = _removeLastIncompleteItem(unique_items, elementsText)
|
||
|
||
elements = [{"text": item} for item in unique_items]
|
||
|
||
elif contentType == "paragraph":
|
||
# Look for {"text": "..."} patterns, including incomplete ones
|
||
text_items = re.findall(r'\{"text"\s*:\s*"([^"]*)"\}', elementsText)
|
||
incomplete_items = re.findall(r'\{"text"\s*:\s*"([^"]*?)(?:\n|$)', elementsText)
|
||
|
||
all_items = text_items + incomplete_items
|
||
unique_items = list(dict.fromkeys([item for item in all_items if item.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_items:
|
||
unique_items = _removeLastIncompleteItem(unique_items, elementsText)
|
||
|
||
elements = [{"text": item} for item in unique_items]
|
||
|
||
elif contentType == "heading":
|
||
# Look for {"level": X, "text": "..."} patterns, including incomplete ones
|
||
heading_items = re.findall(r'\{"level"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"([^"]*)"\}', elementsText)
|
||
incomplete_heading_items = re.findall(r'\{"level"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"([^"]*?)(?:\n|$)', elementsText)
|
||
|
||
all_items = heading_items + incomplete_heading_items
|
||
unique_items = list(dict.fromkeys([(int(level), text) for level, text in all_items if text.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_items:
|
||
unique_items = _removeLastIncompleteItem(unique_items, elementsText)
|
||
|
||
elements = [{"level": level, "text": text} for level, text in unique_items]
|
||
|
||
elif contentType == "table":
|
||
# Look for table patterns
|
||
table_items = re.findall(r'\{"headers"\s*:\s*\[(.*?)\]\s*,\s*"rows"\s*:\s*\[(.*?)\]\s*,\s*"caption"\s*:\s*"([^"]*)"\}', elementsText)
|
||
for headers_str, rows_str, caption in table_items:
|
||
# Extract headers
|
||
headers = re.findall(r'"([^"]+)"', headers_str)
|
||
# Extract rows (simplified)
|
||
rows = []
|
||
row_matches = re.findall(r'\[(.*?)\]', rows_str)
|
||
for row_match in row_matches:
|
||
row_items = re.findall(r'"([^"]+)"', row_match)
|
||
rows.append(row_items)
|
||
|
||
elements.append({
|
||
"headers": headers,
|
||
"rows": rows,
|
||
"caption": caption
|
||
})
|
||
|
||
elif contentType == "code":
|
||
# Look for {"code": "...", "language": "..."} patterns, including incomplete ones
|
||
code_items = re.findall(r'\{"code"\s*:\s*"([^"]*)"\s*,\s*"language"\s*:\s*"([^"]*)"\}', elementsText)
|
||
incomplete_code_items = re.findall(r'\{"code"\s*:\s*"([^"]*?)(?:\n|$)', elementsText)
|
||
|
||
all_items = code_items + [(code, "unknown") for code in incomplete_code_items]
|
||
unique_items = list(dict.fromkeys([(code, lang) for code, lang in all_items if code.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_items:
|
||
unique_items = _removeLastIncompleteItem(unique_items, elementsText)
|
||
|
||
elements = [{"code": code, "language": lang} for code, lang in unique_items]
|
||
|
||
else:
|
||
# Generic fallback - look for any text content, including incomplete
|
||
text_items = re.findall(r'"text"\s*:\s*"([^"]*)"', elementsText)
|
||
incomplete_text_items = re.findall(r'"text"\s*:\s*"([^"]*?)(?:\n|$)', elementsText)
|
||
|
||
all_items = text_items + incomplete_text_items
|
||
unique_items = list(dict.fromkeys([item for item in all_items if item.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_items:
|
||
unique_items = _removeLastIncompleteItem(unique_items, elementsText)
|
||
|
||
elements = [{"text": item} for item in unique_items]
|
||
|
||
return elements
|
||
|
||
|
||
def _removeLastIncompleteItem(items: List[str], original_text: str) -> List[str]:
|
||
"""
|
||
Remove the last item if it appears to be incomplete/corrupted.
|
||
This prevents corrupted data from being included in the final result.
|
||
"""
|
||
import re
|
||
|
||
if not items:
|
||
return items
|
||
|
||
# Check if the original text ends with incomplete JSON patterns
|
||
# Look for patterns that suggest the last item was cut off
|
||
|
||
# Pattern 1: Text ends with incomplete string like {"text": "36
|
||
if re.search(r'\{"[^"]*"\s*:\s*"[^"]*$', original_text):
|
||
logger.debug("Detected incomplete string at end - removing last item")
|
||
return items[:-1]
|
||
|
||
# Pattern 2: Text ends with incomplete boolean like {"bool_flag": tr
|
||
if re.search(r'\{"[^"]*"\s*:\s*(true|false|tr|fa)$', original_text):
|
||
logger.debug("Detected incomplete boolean at end - removing last item")
|
||
return items[:-1]
|
||
|
||
# Pattern 3: Text ends with incomplete number like {"number": 123
|
||
if re.search(r'\{"[^"]*"\s*:\s*\d+$', original_text):
|
||
logger.debug("Detected incomplete number at end - removing last item")
|
||
return items[:-1]
|
||
|
||
# Pattern 4: Text ends with incomplete array like {"array": [1,2,3
|
||
if re.search(r'\{"[^"]*"\s*:\s*\[[^\]]*$', original_text):
|
||
logger.debug("Detected incomplete array at end - removing last item")
|
||
return items[:-1]
|
||
|
||
# Pattern 5: Text ends with incomplete object like {"obj": {"key": "val
|
||
if re.search(r'\{"[^"]*"\s*:\s*\{[^}]*$', original_text):
|
||
logger.debug("Detected incomplete object at end - removing last item")
|
||
return items[:-1]
|
||
|
||
# Pattern 6: Text ends with trailing comma (common sign of incomplete JSON)
|
||
if original_text.rstrip().endswith(','):
|
||
logger.debug("Detected trailing comma - removing last item")
|
||
return items[:-1]
|
||
|
||
# If no incomplete patterns detected, return all items
|
||
return items
|
||
|
||
|
||
def _extractGenericContent(text: str) -> List[Dict[str, Any]]:
|
||
"""
|
||
Extract generic content when no specific section patterns are found.
|
||
This handles cases where the JSON structure is completely broken.
|
||
Handles incomplete strings and corrupted data.
|
||
Excludes the last incomplete item to prevent corrupted data.
|
||
"""
|
||
import re
|
||
|
||
sections = []
|
||
|
||
# Look for any structured content patterns
|
||
# Pattern 1: Look for list items {"text": "..."}, including incomplete ones
|
||
list_items = re.findall(r'\{"text"\s*:\s*"([^"]*)"\}', text)
|
||
incomplete_list_items = re.findall(r'\{"text"\s*:\s*"([^"]*?)(?:\n|$)', text)
|
||
|
||
all_list_items = list_items + incomplete_list_items
|
||
unique_list_items = list(dict.fromkeys([item for item in all_list_items if item.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_list_items:
|
||
unique_list_items = _removeLastIncompleteItem(unique_list_items, text)
|
||
|
||
if unique_list_items:
|
||
elements = [{"text": item} for item in unique_list_items]
|
||
sections.append({
|
||
"id": "section_1",
|
||
"content_type": "list",
|
||
"elements": elements,
|
||
"order": 1
|
||
})
|
||
|
||
# Pattern 2: Look for paragraph text {"text": "..."}, including incomplete ones
|
||
elif re.search(r'\{"text"\s*:\s*"[^"]*\}', text):
|
||
# Extract all text elements, including incomplete ones
|
||
text_items = re.findall(r'\{"text"\s*:\s*"([^"]*)"\}', text)
|
||
incomplete_text_items = re.findall(r'\{"text"\s*:\s*"([^"]*?)(?:\n|$)', text)
|
||
|
||
all_text_items = text_items + incomplete_text_items
|
||
unique_text_items = list(dict.fromkeys([item for item in all_text_items if item.strip()]))
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if unique_text_items:
|
||
unique_text_items = _removeLastIncompleteItem(unique_text_items, text)
|
||
|
||
if unique_text_items:
|
||
elements = [{"text": item} for item in unique_text_items]
|
||
sections.append({
|
||
"id": "section_1",
|
||
"content_type": "paragraph",
|
||
"elements": elements,
|
||
"order": 1
|
||
})
|
||
|
||
# Pattern 3: Look for any quoted strings that might be content, including incomplete ones
|
||
elif re.search(r'"([^"]{3,})"', text): # Strings longer than 3 chars (reduced threshold)
|
||
# Extract longer quoted strings, including incomplete ones
|
||
text_items = re.findall(r'"([^"]{3,})"', text)
|
||
incomplete_text_items = re.findall(r'"([^"]{3,}?)(?:\n|$)', text)
|
||
|
||
all_text_items = text_items + incomplete_text_items
|
||
# Filter out likely JSON keys
|
||
content_items = [item for item in all_text_items if not item.startswith(('section_', 'doc_', 'metadata', 'split_strategy', 'source_documents', 'extraction_method', 'id', 'content_type', 'elements', 'order', 'title', 'filename'))]
|
||
|
||
# Remove the last item if it appears to be incomplete/corrupted
|
||
if content_items:
|
||
content_items = _removeLastIncompleteItem(content_items, text)
|
||
|
||
if content_items:
|
||
elements = [{"text": item} for item in content_items[:10]] # Limit to first 10 items
|
||
sections.append({
|
||
"id": "section_1",
|
||
"content_type": "paragraph",
|
||
"elements": elements,
|
||
"order": 1
|
||
})
|
||
|
||
return sections
|
||
|
||
|
||
def extractSectionsFromDocument(documentData: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||
"""
|
||
Extract all sections from document data structure.
|
||
Handles both flat and nested document structures.
|
||
"""
|
||
if not isinstance(documentData, dict):
|
||
return []
|
||
|
||
# Try to extract sections from documents array
|
||
if "documents" in documentData:
|
||
all_sections = []
|
||
for doc in documentData.get("documents", []):
|
||
if isinstance(doc, dict) and "sections" in doc:
|
||
sections = doc.get("sections", [])
|
||
if isinstance(sections, list):
|
||
all_sections.extend(sections)
|
||
return all_sections
|
||
|
||
# Try to extract sections directly from root
|
||
if "sections" in documentData:
|
||
sections = documentData.get("sections", [])
|
||
if isinstance(sections, list):
|
||
return sections
|
||
|
||
return []
|
||
|
||
|
||
def extractContentSample(section: Dict[str, Any]) -> str:
|
||
"""
|
||
Extract a sample of content from a section for continuation context.
|
||
Returns a string describing the last content for context.
|
||
"""
|
||
if not isinstance(section, dict):
|
||
return ""
|
||
|
||
content_type = section.get("content_type", "").lower()
|
||
elements = section.get("elements", [])
|
||
|
||
if not elements or not isinstance(elements, list):
|
||
return "Content exists"
|
||
|
||
# Get last elements for sampling
|
||
sample_elements = elements[-5:] if len(elements) > 5 else elements
|
||
|
||
if content_type == "list":
|
||
# Extract last few list items
|
||
items_text = []
|
||
for elem in sample_elements:
|
||
if isinstance(elem, dict) and "text" in elem:
|
||
items_text.append(elem.get("text", ""))
|
||
if items_text:
|
||
return f"Last {len(items_text)} items: {', '.join(items_text[:3])}"
|
||
|
||
elif content_type == "paragraph":
|
||
# Extract text and take last 150 chars
|
||
for elem in sample_elements:
|
||
if isinstance(elem, dict) and "text" in elem:
|
||
text = elem.get("text", "")
|
||
if len(text) > 150:
|
||
text = "..." + text[-150:]
|
||
return f"Last content: {text}"
|
||
|
||
elif content_type == "code":
|
||
# Extract last few lines
|
||
for elem in sample_elements:
|
||
if isinstance(elem, dict) and "code" in elem:
|
||
code = elem.get("code", "")
|
||
lines = code.split('\n')
|
||
if len(lines) > 5:
|
||
return f"Last lines ({len(lines)} total): {', '.join(lines[-3:])}"
|
||
return f"Code ({len(lines)} lines)"
|
||
|
||
elif content_type == "table":
|
||
# Extract last rows
|
||
for elem in sample_elements:
|
||
if isinstance(elem, dict) and "rows" in elem:
|
||
rows = elem.get("rows", [])
|
||
return f"Table with {len(rows)} rows"
|
||
|
||
return "Content exists"
|
||
|
||
|
||
def _buildDetailedContinuationInfo(section: Dict[str, Any], content_type: str) -> Dict[str, Any]:
|
||
"""
|
||
Build detailed continuation information for better AI guidance.
|
||
Completely generic - works for any content type (list, paragraph, code, table, etc.)
|
||
"""
|
||
elements = section.get("elements", [])
|
||
|
||
if not elements:
|
||
return {
|
||
"type": "continue_general",
|
||
"sample": extractContentSample(section),
|
||
"last_item": "",
|
||
"item_count": 0,
|
||
"guidance": "Continue generating content in the same format and style."
|
||
}
|
||
|
||
# Count elements regardless of type
|
||
element_count = len(elements)
|
||
|
||
# Extract sample for context - completely generic
|
||
sample = extractContentSample(section)
|
||
|
||
# Generic continuation guidance - applies to ANY content type
|
||
# Tell AI to generate ALL REMAINING content to complete the user request
|
||
return {
|
||
"type": "continue_general",
|
||
"sample": sample,
|
||
"last_item": "",
|
||
"item_count": element_count,
|
||
"guidance": "Generate ALL remaining content to complete the user's request. Continue from where you left off and finish everything that was requested."
|
||
}
|
||
|
||
|
||
def _extractLastItemsFromFragment(fragment: str, max_items: int = 10) -> str:
|
||
"""
|
||
Extract the last few items from a JSON fragment for continuation context.
|
||
Uses JSON structure (sections -> elements -> items) - fully generic.
|
||
Works with broken/incomplete JSON by trying to parse and extract sections.
|
||
"""
|
||
if not fragment:
|
||
return ""
|
||
|
||
# Strategy 1: Try to parse as JSON and extract from structure
|
||
try:
|
||
# Try to repair and parse the fragment
|
||
parsed = repairBrokenJson(fragment)
|
||
if parsed:
|
||
# Extract sections from parsed JSON using structure
|
||
sections = extractSectionsFromDocument(parsed)
|
||
if sections:
|
||
# Get the last section (likely where continuation should happen)
|
||
sorted_sections = sorted(sections, key=lambda s: s.get("order", 0))
|
||
last_section = sorted_sections[-1]
|
||
elements = last_section.get("elements", [])
|
||
|
||
if elements and isinstance(elements, list):
|
||
content_type = last_section.get("content_type", "").lower()
|
||
|
||
# For list content_type, extract from items array
|
||
if content_type == "list" and len(elements) > 0:
|
||
last_element = elements[-1]
|
||
if isinstance(last_element, dict):
|
||
# Check if it has an "items" array (list structure)
|
||
if "items" in last_element and isinstance(last_element["items"], list):
|
||
items_list = last_element["items"]
|
||
if items_list:
|
||
# Get last max_items from this items array
|
||
last_items = items_list[-max_items:] if len(items_list) > max_items else items_list
|
||
# Extract text from each item
|
||
texts = []
|
||
for item in last_items:
|
||
if isinstance(item, dict) and "text" in item:
|
||
texts.append(str(item["text"]))
|
||
if texts:
|
||
return ', '.join(texts)
|
||
|
||
# Or if elements themselves are items (alternative structure)
|
||
elif "text" in last_element:
|
||
# Get last max_items elements that have text
|
||
elements_with_text = [e for e in elements if isinstance(e, dict) and "text" in e]
|
||
if elements_with_text:
|
||
last_elements = elements_with_text[-max_items:] if len(elements_with_text) > max_items else elements_with_text
|
||
texts = [str(e.get("text", "")) for e in last_elements]
|
||
if texts:
|
||
return ', '.join(texts)
|
||
|
||
# For other content types, extract from elements
|
||
elif len(elements) > 0:
|
||
# Get last max_items elements that have text/code
|
||
valid_elements = [e for e in elements if isinstance(e, dict) and ("text" in e or "code" in e)]
|
||
if valid_elements:
|
||
last_elements = valid_elements[-max_items:] if len(valid_elements) > max_items else valid_elements
|
||
texts = []
|
||
for elem in last_elements:
|
||
if "text" in elem:
|
||
texts.append(str(elem["text"]))
|
||
elif "code" in elem:
|
||
# For code, show snippet
|
||
code = str(elem["code"])
|
||
texts.append(code[:50] + "..." if len(code) > 50 else code)
|
||
if texts:
|
||
return ', '.join(texts)
|
||
except Exception as e:
|
||
logger.debug(f"Could not extract items from fragment using JSON structure: {e}")
|
||
|
||
# Strategy 2: If parsing failed, try progressive parsing from the end
|
||
# Look for the last complete JSON structures near the end
|
||
try:
|
||
# Try parsing different lengths from the end
|
||
for length in [3000, 2000, 1000, 500]:
|
||
if len(fragment) > length:
|
||
end_portion = fragment[-length:]
|
||
closed = _closeJsonStructures(end_portion)
|
||
obj, err, _ = tryParseJson(closed)
|
||
if err is None and isinstance(obj, dict):
|
||
# Successfully parsed - extract sections
|
||
sections = extractSectionsFromDocument(obj)
|
||
if sections:
|
||
# Same extraction logic as above
|
||
sorted_sections = sorted(sections, key=lambda s: s.get("order", 0))
|
||
if sorted_sections:
|
||
last_section = sorted_sections[-1]
|
||
elements = last_section.get("elements", [])
|
||
if elements:
|
||
# Extract texts using same logic as Strategy 1
|
||
texts = []
|
||
for elem in elements[-max_items:]:
|
||
if isinstance(elem, dict):
|
||
if "items" in elem and isinstance(elem["items"], list):
|
||
# Get last item from items array
|
||
if elem["items"]:
|
||
last_item = elem["items"][-1]
|
||
if isinstance(last_item, dict) and "text" in last_item:
|
||
texts.append(str(last_item["text"]))
|
||
elif "text" in elem:
|
||
texts.append(str(elem["text"]))
|
||
if texts:
|
||
return ', '.join(texts[-max_items:])
|
||
except Exception as e:
|
||
logger.debug(f"Progressive parsing from end failed: {e}")
|
||
|
||
# Strategy 3: If all parsing fails, try simple extraction from raw fragment
|
||
# Look for last complete {"text": "..."} pattern near the end
|
||
try:
|
||
# Look at last 2000 chars for the pattern
|
||
end_portion = fragment[-2000:] if len(fragment) > 2000 else fragment
|
||
# Find all {"text": "value"} patterns
|
||
import re
|
||
# Pattern to match {"text": "..."} with escaped quotes
|
||
pattern = r'\{"text"\s*:\s*"([^"]+)"\}'
|
||
matches = re.findall(pattern, end_portion)
|
||
if matches:
|
||
# Get last max_items
|
||
last_matches = matches[-max_items:] if len(matches) > max_items else matches
|
||
return ', '.join(last_matches)
|
||
except Exception as e:
|
||
logger.debug(f"Simple pattern extraction failed: {e}")
|
||
|
||
# Strategy 4: If all fails, return empty (will use last_item_from_sections)
|
||
return ""
|
||
|
||
|
||
def buildContinuationContext(allSections: List[Dict[str, Any]], lastRawResponse: Optional[str] = None) -> Dict[str, Any]:
|
||
"""
|
||
Build context information from accumulated sections for continuation prompt.
|
||
Extracts last items and provides clear continuation point.
|
||
|
||
Args:
|
||
allSections: List of sections already generated
|
||
lastRawResponse: Raw JSON response from last iteration (can be broken/incomplete)
|
||
|
||
Returns:
|
||
Dict with section_count, last_raw_json, last_items, and continuation point
|
||
"""
|
||
context = {
|
||
"section_count": len(allSections),
|
||
}
|
||
|
||
# Extract last COMPLETE object directly from raw response (generic - works for any structure)
|
||
# This is extracted BEFORE any merging/accumulation happens
|
||
# Returns the full last complete object like {"text": "..."} or {"code": "...", "language": "..."} etc.
|
||
# Logic: find the last complete {...} where there are no nested { inside (flat object)
|
||
last_complete_object = "" # Full object as JSON string
|
||
total_items_count = 0
|
||
|
||
if lastRawResponse:
|
||
raw_json = stripCodeFences(lastRawResponse.strip())
|
||
if raw_json and raw_json.strip() != "{}":
|
||
# Find last complete flat object (no nested objects inside)
|
||
# Scan from the end backwards to find the last complete {...} object
|
||
# A flat object is complete if: starts with {, ends with }, and has no nested { inside
|
||
|
||
# Work backwards from the end, find last }
|
||
for i in range(len(raw_json) - 1, -1, -1):
|
||
if raw_json[i] == '}':
|
||
# Found a closing brace, work backwards to find its opening brace
|
||
depth = 1
|
||
opening_pos = -1
|
||
|
||
for j in range(i - 1, -1, -1):
|
||
if raw_json[j] == '}':
|
||
depth += 1
|
||
elif raw_json[j] == '{':
|
||
depth -= 1
|
||
if depth == 0:
|
||
# Found matching opening brace
|
||
opening_pos = j
|
||
# Check if this is a flat object (no nested { inside)
|
||
obj_content = raw_json[j + 1:i]
|
||
if '{' not in obj_content:
|
||
# This is a flat object (no nested objects inside)
|
||
last_complete_object = raw_json[j:i + 1]
|
||
break
|
||
|
||
if last_complete_object:
|
||
break
|
||
|
||
# Also try structure-based parsing for item count
|
||
try:
|
||
parsed = repairBrokenJson(raw_json)
|
||
if parsed:
|
||
sections = extractSectionsFromDocument(parsed)
|
||
if sections:
|
||
sorted_sections = sorted(sections, key=lambda s: s.get("order", 0))
|
||
last_section = sorted_sections[-1]
|
||
elements = last_section.get("elements", [])
|
||
|
||
if elements and isinstance(elements, list) and len(elements) > 0:
|
||
if last_section.get("content_type") == "list":
|
||
last_element = elements[-1]
|
||
if isinstance(last_element, dict):
|
||
if "items" in last_element and isinstance(last_element["items"], list):
|
||
items_list = last_element["items"]
|
||
# Only count complete items (those successfully extracted)
|
||
total_items_count = len(items_list)
|
||
except Exception as e:
|
||
logger.debug(f"Could not extract item count from raw response structure: {e}")
|
||
|
||
# Also extract last items for display (fragment extraction)
|
||
last_items_from_fragment = _extractLastItemsFromFragment(raw_json, max_items=10)
|
||
|
||
context["last_raw_json"] = raw_json
|
||
context["last_item_object"] = last_complete_object # Full last complete object (generic - any structure)
|
||
context["last_items_from_fragment"] = last_items_from_fragment
|
||
context["total_items_count"] = total_items_count # Count from raw response
|
||
|
||
logger.debug(f"Included previous JSON response in continuation context ({len(raw_json)} chars, {total_items_count} items in response, last complete object: {last_complete_object})")
|
||
else:
|
||
logger.warning("lastRawResponse was empty or just '{}' - continuation may not work correctly")
|
||
else:
|
||
# No raw response - fallback to extracting from accumulated sections
|
||
# Extract the last complete object from the last element
|
||
last_item_object_from_sections = ""
|
||
if allSections:
|
||
sorted_sections = sorted(allSections, key=lambda s: s.get("order", 0))
|
||
last_section = sorted_sections[-1]
|
||
elements = last_section.get("elements", [])
|
||
|
||
if elements and isinstance(elements, list) and len(elements) > 0:
|
||
# Get the last element (could be any structure - generic)
|
||
last_element = elements[-1]
|
||
if isinstance(last_element, dict):
|
||
# Try to get items if it's a list structure
|
||
if "items" in last_element and isinstance(last_element["items"], list):
|
||
items_list = last_element["items"]
|
||
total_items_count = len(items_list)
|
||
if items_list:
|
||
# Get last item (any structure)
|
||
last_item = items_list[-1]
|
||
if isinstance(last_item, dict):
|
||
# Convert to JSON string (generic - works for any object structure)
|
||
import json
|
||
try:
|
||
last_item_object_from_sections = json.dumps(last_item)
|
||
except:
|
||
pass
|
||
else:
|
||
# Element itself is the object (no items array)
|
||
total_items_count = len(elements)
|
||
# Convert to JSON string (generic)
|
||
import json
|
||
try:
|
||
last_item_object_from_sections = json.dumps(last_element)
|
||
except:
|
||
pass
|
||
|
||
context["last_item_object"] = last_item_object_from_sections
|
||
context["total_items_count"] = total_items_count
|
||
logger.debug(f"No previous raw response available for continuation context (but have {total_items_count} items accumulated, last item object: {last_item_object_from_sections})")
|
||
|
||
return context
|
||
|
||
|
||
def parseJsonWithModel(jsonString: str, modelClass: Type[T]) -> T:
|
||
"""
|
||
Parse JSON string using Pydantic model with error handling.
|
||
|
||
Uses existing jsonUtils methods:
|
||
- extractJsonString() - Extracts JSON from text with code fences
|
||
- tryParseJson() - Safe parsing with error handling
|
||
- repairBrokenJson() - Repairs broken/incomplete JSON
|
||
|
||
Args:
|
||
jsonString: JSON string to parse (may contain code fences, extra text, etc.)
|
||
modelClass: Pydantic model class to parse into
|
||
|
||
Returns:
|
||
Parsed Pydantic model instance
|
||
|
||
Raises:
|
||
ValueError: If JSON cannot be parsed or validated
|
||
"""
|
||
if not jsonString:
|
||
raise ValueError(f"Cannot parse empty JSON string for {modelClass.__name__}")
|
||
|
||
# Step 1: Extract JSON string (handles code fences, extra text)
|
||
extractedJson = extractJsonString(jsonString)
|
||
|
||
if not extractedJson or extractedJson.strip() == "":
|
||
raise ValueError(f"No JSON found in string for {modelClass.__name__}")
|
||
|
||
# Step 2: Try to parse as JSON
|
||
parsedJson, error, cleaned = tryParseJson(extractedJson)
|
||
|
||
if error is None and parsedJson is not None:
|
||
# Successfully parsed - try to create model
|
||
try:
|
||
if isinstance(parsedJson, dict):
|
||
return modelClass(**parsedJson)
|
||
elif isinstance(parsedJson, list):
|
||
# If model expects a list, try to parse first item
|
||
if parsedJson:
|
||
return modelClass(**parsedJson[0])
|
||
else:
|
||
raise ValueError(f"Empty list cannot be parsed as {modelClass.__name__}")
|
||
else:
|
||
raise ValueError(f"Parsed JSON is not a dict or list: {type(parsedJson)}")
|
||
except ValidationError as e:
|
||
logger.error(f"Validation error parsing {modelClass.__name__}: {e}")
|
||
raise ValueError(f"Invalid data for {modelClass.__name__}: {e}")
|
||
except Exception as e:
|
||
logger.error(f"Error creating {modelClass.__name__} instance: {e}")
|
||
raise ValueError(f"Failed to create {modelClass.__name__} instance: {e}")
|
||
|
||
# Step 3: Try to repair broken JSON
|
||
logger.warning(f"Initial JSON parsing failed, attempting repair for {modelClass.__name__}")
|
||
repairedJson = repairBrokenJson(extractedJson)
|
||
|
||
if repairedJson:
|
||
# Try parsing repaired JSON
|
||
parsedRepaired, errorRepaired, _ = tryParseJson(json.dumps(repairedJson))
|
||
|
||
if errorRepaired is None and parsedRepaired is not None:
|
||
try:
|
||
if isinstance(parsedRepaired, dict):
|
||
return modelClass(**parsedRepaired)
|
||
elif isinstance(parsedRepaired, list) and parsedRepaired:
|
||
return modelClass(**parsedRepaired[0])
|
||
except ValidationError as e:
|
||
logger.error(f"Validation error parsing repaired {modelClass.__name__}: {e}")
|
||
raise ValueError(f"Invalid repaired data for {modelClass.__name__}: {e}")
|
||
except Exception as e:
|
||
logger.error(f"Error creating {modelClass.__name__} from repaired JSON: {e}")
|
||
|
||
# Step 4: All parsing failed
|
||
logger.error(f"Failed to parse JSON for {modelClass.__name__}. Cleaned JSON preview: {cleaned[:200]}...")
|
||
raise ValueError(f"Failed to parse or validate JSON for {modelClass.__name__}. JSON may be malformed or incomplete.")
|
||
|