PowerOn/gateway
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

full refactory of core json cut handling system

Browse source
This commit is contained in:
ValueOn AG 2026-01-05 21:16:10 +01:00
parent ecababe600
commit d747054976
14 changed files with 2387 additions and 4087 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
modules/datamodels/datamodelAi.py
View file

@ -261,13 +261,26 @@ class ContinuationContext(BaseModel):
"""Pydantic model for continuation context information.""" """Pydantic model for continuation context information."""
section_count: int section_count: int
delivered_summary: str delivered_summary: str
cut_off_element: Optional[str] = None
element_before_cutoff: Optional[str] = None
template_structure: Optional[str] = None template_structure: Optional[str] = None
last_complete_part: Optional[str] = None last_complete_part: Optional[str] = None
incomplete_part: Optional[str] = None incomplete_part: Optional[str] = None
structure_context: Optional[str] = None
last_raw_json: Optional[str] = None last_raw_json: Optional[str] = None
overlap_context: Optional[str] = None # From jsonContinuation.getContexts() - innermost element containing cut
hierarchy_context: Optional[str] = None # From jsonContinuation.getContexts() - full structure from root to cut
class JsonContinuationContexts(BaseModel):
"""
Pydantic model for JSON continuation contexts.
Contains three contexts for truncated JSON strings:
- overlapContext: The innermost object/array element containing the cut point (for merging)
- hierarchyContext: Full structure from root to cut with budget-limited values
- completePart: Valid JSON with all structures properly closed
"""
overlapContext: str = Field(description="The innermost object/array element containing the cut point (for merging)")
hierarchyContext: str = Field(description="Full structure from root to cut with budget-limited values")
completePart: str = Field(description="Valid JSON with all structures properly closed")
class SectionPromptArgs(BaseModel): class SectionPromptArgs(BaseModel):

921
modules/services/serviceAi/merge_1.txt
View file

@ -1,64 +1,57 @@
================================================================================ ================================================================================
JSON MERGE OPERATION #1 JSON MERGE OPERATION #1
================================================================================ ================================================================================
Timestamp: 2026-01-04T23:08:13.252204 Timestamp: 2026-01-05T08:30:55.469646
INPUT: INPUT:
Accumulated length: 31737 chars Accumulated length: 419 chars
New Fragment length: 10178 chars New Fragment length: 120 chars
Accumulated: 409 lines (showing first 5 and last 5) Accumulated: 20 lines (showing first 5 and last 5)
{ {
"elements": [ "metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
... (10 lines omitted) ...
{ {
"type": "table", "type": "heading",
"content": { "content": {
... (399 lines omitted) ... "text": "Introduction",
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691], New Fragment: 8 lines (showing first 5 and last 5)
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831], "level": 1
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039
New Fragment: 135 lines (showing first 5 and last 5)
```json
{
"elements": [
{
"type": "table",
... (125 lines omitted) ...
} }
} }
] ]
} },
```
Normalized Accumulated (31737 chars)
(showing first 5 and last 5 of 409 lines)
{ {
"id": "sec2",
"conten
Normalized Accumulated (407 chars)
(showing first 5 and last 5 of 19 lines)
{
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
... (9 lines omitted) ...
"elements": [ "elements": [
{ {
"type": "table", "type": "heading",
"content": { "content": {
... (399 lines omitted) ... "text": "Introduction",
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039
Normalized New Fragment (10166 chars) Normalized New Fragment (115 chars)
(showing first 5 and last 5 of 133 lines) "level": 1
{
"elements": [
{
"type": "table",
"content": {
... (123 lines omitted) ...
]
} }
} }
] ]
} },
{
"id": "sec2",
"conten
STEP: PHASE 1 STEP: PHASE 1
Description: Finding overlap between JSON strings Description: Finding overlap between JSON strings
⏳ In progress... ⏳ In progress...
@ -70,832 +63,52 @@ STEP: PHASE 1
⚠️ NO OVERLAP FOUND - This indicates iterations should stop ⚠️ NO OVERLAP FOUND - This indicates iterations should stop
Closing JSON and returning final result Closing JSON and returning final result
Closed JSON (31743 chars): Closed JSON (414 chars):
============================================================================== ==============================================================================
{ {
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
},
"documents": [
{
"id": "doc1",
"title": "Document 1",
"sections": [
{
"id": "sec1",
"content_type": "heading",
"elements": [ "elements": [
{ {
"type": "table", "type": "heading",
"content": { "content": {
"headers": ["Spalte1", "Spalte2", "Spalte3", "Spalte4", "Spalte5", "Spalte6", "Spalte7", "Spalte8", "Spalte9", "Spalte10"], "text": "Introduction"}}]}]}]}
"rows": [
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29],
[31, 37, 41, 43, 47, 53, 59, 61, 67, 71],
[73, 79, 83, 89, 97, 101, 103, 107, 109, 113],
[127, 131, 137, 139, 149, 151, 157, 163, 167, 173],
[179, 181, 191, 193, 197, 199, 211, 223, 227, 229],
[233, 239, 241, 251, 257, 263, 269, 271, 277, 281],
[283, 293, 307, 311, 313, 317, 331, 337, 347, 349],
[353, 359, 367, 373, 379, 383, 389, 397, 401, 409],
[419, 421, 431, 433, 439, 443, 449, 457, 461, 463],
[467, 479, 487, 491, 499, 503, 509, 521, 523, 541],
[547, 557, 563, 569, 571, 577, 587, 593, 599, 601],
[607, 613, 617, 619, 631, 641, 643, 647, 653, 659],
[661, 673, 677, 683, 691, 701, 709, 719, 727, 733],
[739, 743, 751, 757, 761, 769, 773, 787, 797, 809],
[811, 821, 823, 827, 829, 839, 853, 857, 859, 863],
[877, 881, 883, 887, 907, 911, 919, 929, 937, 941],
[947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013],
[1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069],
[1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151],
[1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223],
[1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291],
[1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373],
[1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451],
[1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511],
[1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583],
[1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657],
[1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733],
[1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811],
[1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889],
[1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987],
[1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053],
[2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129],
[2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213],
[2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287],
[2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357],
[2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423],
[2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531],
[2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617],
[2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687],
[2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741],
[2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819],
[2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903],
[2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999],
[3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079],
[3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181],
[3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257],
[3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331],
[3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413],
[3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511],
[3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571],
[3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643],
[3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727],
[3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821],
[3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907],
[3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989],
[4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057],
[4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139],
[4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231],
[4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297],
[4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409],
[4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493],
[4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583],
[4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657],
[4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751],
[4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831],
[4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937],
[4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003],
[5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087],
[5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179],
[5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279],
[5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387],
[5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443],
[5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521],
[5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639],
[5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693],
[5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791],
[5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857],
[5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939],
[5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053],
[6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133],
[6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221],
[6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301],
[6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367],
[6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473],
[6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571],
[6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673],
[6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761],
[6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833],
[6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917],
[6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997],
[7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103],
[7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207],
[7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297],
[7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411],
[7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499],
[7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561],
[7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643],
[7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723],
[7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829],
[7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919],
[7927, 7933, 7937, 7949, 7951, 7963, 7993, 8009, 8011, 8017],
[8039, 8053, 8059, 8069, 8081, 8087, 8089, 8093, 8101, 8111],
[8117, 8123, 8147, 8161, 8167, 8171, 8179, 8191, 8209, 8219],
[8221, 8231, 8233, 8237, 8243, 8263, 8269, 8273, 8287, 8291],
[8293, 8297, 8311, 8317, 8329, 8353, 8363, 8369, 8377, 8387],
[8389, 8419, 8423, 8429, 8431, 8443, 8447, 8461, 8467, 8501],
[8513, 8521, 8527, 8537, 8539, 8543, 8563, 8573, 8581, 8597],
[8599, 8609, 8623, 8627, 8629, 8641, 8647, 8663, 8669, 8677],
[8681, 8689, 8693, 8699, 8707, 8713, 8719, 8731, 8737, 8741],
[8747, 8753, 8761, 8779, 8783, 8803, 8807, 8819, 8821, 8831],
[8837, 8839, 8849, 8861, 8863, 8867, 8887, 8893, 8923, 8929],
[8933, 8941, 8951, 8963, 8969, 8971, 8999, 9001, 9007, 9011],
[9013, 9029, 9041, 9043, 9049, 9059, 9067, 9091, 9103, 9109],
[9127, 9133, 9137, 9151, 9157, 9161, 9173, 9181, 9187, 9199],
[9203, 9209, 9221, 9227, 9239, 9241, 9257, 9277, 9281, 9283],
[9293, 9311, 9319, 9323, 9337, 9341, 9343, 9349, 9371, 9377],
[9391, 9397, 9403, 9413, 9419, 9421, 9431, 9433, 9437, 9439],
[9461, 9463, 9467, 9473, 9479, 9491, 9497, 9511, 9521, 9533],
[9539, 9547, 9551, 9587, 9601, 9613, 9619, 9623, 9629, 9631],
[9643, 9649, 9661, 9677, 9679, 9689, 9697, 9719, 9721, 9733],
[9739, 9743, 9749, 9767, 9769, 9781, 9787, 9791, 9803, 9811],
[9817, 9829, 9833, 9839, 9851, 9857, 9859, 9871, 9883, 9887],
[9901, 9907, 9923, 9929, 9931, 9941, 9949, 9967, 9973, 10007],
[10009, 10037, 10039, 10061, 10067, 10069, 10079, 10091, 10093, 10099],
[10103, 10111, 10133, 10139, 10141, 10151, 10159, 10163, 10169, 10177],
[10181, 10193, 10211, 10223, 10243, 10247, 10253, 10259, 10267, 10271],
[10273, 10289, 10301, 10303, 10313, 10321, 10331, 10333, 10337, 10343],
[10357, 10369, 10391, 10399, 10427, 10429, 10433, 10453, 10457, 10459],
[10463, 10477, 10487, 10499, 10501, 10513, 10529, 10531, 10559, 10567],
[10589, 10597, 10601, 10607, 10613, 10627, 10631, 10639, 10651, 10657],
[10663, 10667, 10687, 10691, 10709, 10711, 10723, 10729, 10733, 10739],
[10753, 10771, 10781, 10789, 10799, 10831, 10837, 10847, 10853, 10859],
[10861, 10867, 10883, 10889, 10891, 10903, 10909, 10937, 10939, 10949],
[10957, 10973, 10979, 10987, 10993, 11003, 11027, 11047, 11057, 11059],
[11069, 11071, 11083, 11087, 11093, 11113, 11117, 11119, 11131, 11149],
[11159, 11161, 11171, 11173, 11177, 11197, 11213, 11239, 11243, 11251],
[11257, 11261, 11273, 11279, 11287, 11299, 11311, 11317, 11321, 11329],
[11351, 11353, 11369, 11383, 11393, 11399, 11411, 11423, 11437, 11443],
[11447, 11467, 11471, 11483, 11489, 11491, 11497, 11503, 11519, 11527],
[11549, 11551, 11579, 11587, 11593, 11597, 11617, 11621, 11633, 11657],
[11677, 11681, 11689, 11699, 11701, 11717, 11719, 11731, 11743, 11777],
[11779, 11783, 11789, 11801, 11807, 11813, 11821, 11827, 11831, 11833],
[11839, 11863, 11867, 11887, 11897, 11903, 11909, 11923, 11927, 11933],
[11939, 11941, 11953, 11959, 11969, 11971, 11981, 11987, 12007, 12011],
[12037, 12041, 12043, 12049, 12071, 12073, 12097, 12101, 12107, 12109],
[12113, 12119, 12143, 12149, 12157, 12161, 12163, 12197, 12203, 12211],
[12227, 12239, 12241, 12251, 12253, 12263, 12269, 12277, 12281, 12289],
[12301, 12323, 12329, 12343, 12347, 12373, 12377, 12379, 12391, 12401],
[12409, 12413, 12421, 12433, 12437, 12451, 12457, 12473, 12479, 12487],
[12491, 12497, 12503, 12511, 12517, 12527, 12539, 12541, 12547, 12553],
[12569, 12577, 12583, 12589, 12601, 12611, 12613, 12619, 12637, 12641],
[12647, 12653, 12659, 12671, 12689, 12697, 12703, 12713, 12721, 12739],
[12743, 12757, 12763, 12781, 12791, 12799, 12809, 12821, 12823, 12829],
[12841, 12853, 12889, 12893, 12899, 12907, 12911, 12917, 12919, 12923],
[12941, 12953, 12959, 12967, 12973, 12979, 12983, 13001, 13003, 13007],
[13009, 13033, 13037, 13043, 13049, 13063, 13093, 13099, 13103, 13109],
[13121, 13127, 13147, 13151, 13159, 13163, 13171, 13177, 13183, 13187],
[13217, 13219, 13229, 13241, 13249, 13259, 13267, 13291, 13297, 13309],
[13313, 13327, 13331, 13337, 13339, 13367, 13381, 13397, 13399, 13411],
[13417, 13421, 13441, 13451, 13457, 13463, 13469, 13477, 13487, 13499],
[13513, 13523, 13537, 13553, 13567, 13577, 13591, 13597, 13613, 13619],
[13627, 13633, 13649, 13669, 13679, 13681, 13687, 13691, 13693, 13697],
[13709, 13711, 13721, 13723, 13729, 13751, 13757, 13759, 13763, 13781],
[13789, 13799, 13807, 13829, 13831, 13841, 13859, 13873, 13877, 13879],
[13883, 13901, 13903, 13907, 13913, 13921, 13931, 13933, 13963, 13967],
[13997, 13999, 14009, 14011, 14029, 14033, 14051, 14057, 14071, 14081],
[14083, 14087, 14107, 14143, 14149, 14153, 14159, 14173, 14177, 14197],
[14207, 14221, 14243, 14249, 14251, 14281, 14293, 14303, 14321, 14323],
[14327, 14341, 14347, 14369, 14387, 14389, 14401, 14407, 14411, 14419],
[14423, 14431, 14437, 14447, 14449, 14461, 14479, 14489, 14503, 14519],
[14533, 14537, 14543, 14549, 14551, 14557, 14561, 14563, 14591, 14593],
[14621, 14627, 14629, 14633, 14639, 14653, 14657, 14669, 14683, 14699],
[14713, 14717, 14723, 14731, 14737, 14741, 14747, 14753, 14759, 14767],
[14771, 14779, 14783, 14797, 14813, 14821, 14827, 14831, 14843, 14851],
[14867, 14869, 14879, 14887, 14891, 14897, 14923, 14929, 14939, 14947],
[14951, 14957, 14969, 14983, 15013, 15017, 15031, 15053, 15061, 15073],
[15077, 15083, 15091, 15101, 15107, 15121, 15131, 15137, 15139, 15149],
[15161, 15173, 15187, 15193, 15199, 15217, 15227, 15233, 15241, 15259],
[15263, 15269, 15271, 15277, 15287, 15289, 15299, 15307, 15313, 15319],
[15329, 15331, 15349, 15359, 15361, 15373, 15377, 15383, 15391, 15401],
[15413, 15427, 15439, 15443, 15451, 15461, 15467, 15473, 15493, 15497],
[15511, 15527, 15541, 15551, 15559, 15569, 15581, 15583, 15601, 15607],
[15619, 15629, 15641, 15643, 15647, 15649, 15661, 15667, 15671, 15679],
[15683, 15727, 15731, 15733, 15737, 15739, 15749, 15761, 15767, 15773],
[15787, 15791, 15797, 15803, 15809, 15817, 15823, 15859, 15877, 15881],
[15887, 15889, 15901, 15907, 15913, 15919, 15923, 15937, 15959, 15971],
[15973, 15991, 16001, 16007, 16033, 16057, 16061, 16063, 16067, 16069],
[16073, 16087, 16091, 16097, 16103, 16111, 16127, 16139, 16141, 16183],
[16187, 16189, 16193, 16217, 16223, 16229, 16231, 16249, 16253, 16267],
[16273, 16301, 16319, 16333, 16339, 16349, 16361, 16363, 16369, 16381],
[16411, 16417, 16421, 16427, 16433, 16447, 16451, 16453, 16477, 16481],
[16487, 16493, 16519, 16529, 16547, 16553, 16561, 16567, 16573, 16603],
[16607, 16619, 16631, 16633, 16649, 16651, 16657, 16661, 16673, 16691],
[16693, 16699, 16703, 16729, 16741, 16747, 16759, 16763, 16787, 16811],
[16823, 16829, 16831, 16843, 16871, 16879, 16883, 16889, 16901, 16903],
[16921, 16927, 16931, 16937, 16943, 16963, 16979, 16981, 16987, 16993],
[17011, 17021, 17027, 17029, 17033, 17041, 17047, 17053, 17077, 17093],
[17099, 17107, 17117, 17123, 17137, 17159, 17167, 17183, 17189, 17191],
[17203, 17207, 17209, 17231, 17239, 17257, 17291, 17293, 17299, 17317],
[17321, 17327, 17333, 17341, 17351, 17359, 17377, 17383, 17387, 17389],
[17393, 17401, 17417, 17419, 17431, 17443, 17449, 17467, 17471, 17477],
[17483, 17489, 17491, 17497, 17509, 17519, 17539, 17551, 17569, 17573],
[17579, 17581, 17597, 17599, 17609, 17623, 17627, 17657, 17659, 17669],
[17681, 17683, 17707, 17713, 17729, 17737, 17747, 17749, 17761, 17783],
[17789, 17791, 17807, 17827, 17837, 17839, 17851, 17863, 17881, 17891],
[17903, 17909, 17911, 17921, 17923, 17929, 17939, 17957, 17959, 17971],
[17977, 17981, 17987, 17989, 18013, 18041, 18043, 18047, 18049, 18059],
[18061, 18077, 18089, 18097, 18119, 18121, 18127, 18131, 18133, 18143],
[18149, 18169, 18181, 18191, 18199, 18211, 18217, 18223, 18229, 18233],
[18251, 18253, 18257, 18269, 18287, 18289, 18301, 18307, 18311, 18313],
[18329, 18341, 18353, 18367, 18371, 18379, 18397, 18401, 18413, 18427],
[18433, 18439, 18443, 18451, 18457, 18461, 18481, 18493, 18503, 18517],
[18521, 18523, 18539, 18541, 18553, 18583, 18587, 18593, 18617, 18637],
[18661, 18671, 18679, 18691, 18701, 18713, 18719, 18731, 18743, 18749],
[18757, 18773, 18787, 18793, 18797, 18803, 18839, 18859, 18869, 18899],
[18911, 18913, 18917, 18919, 18947, 18959, 18973, 18979, 19001, 19009],
[19013, 19031, 19037, 19051, 19069, 19073, 19079, 19081, 19087, 19121],
[19139, 19141, 19157, 19163, 19181, 19183, 19207, 19211, 19213, 19219],
[19231, 19237, 19249, 19259, 19267, 19273, 19289, 19301, 19309, 19319],
[19333, 19373, 19379, 19381, 19387, 19391, 19403, 19417, 19421, 19423],
[19427, 19429, 19433, 19441, 19447, 19457, 19463, 19469, 19471, 19477],
[19483, 19501, 19507, 19531, 19541, 19543, 19553, 19559, 19571, 19577],
[19583, 19597, 19603, 19609, 19661, 19681, 19687, 19697, 19699, 19709],
[19717, 19727, 19739, 19751, 19753, 19759, 19763, 19777, 19793, 19801],
[19813, 19819, 19841, 19843, 19853, 19861, 19867, 19889, 19891, 19913],
[19919, 19927, 19937, 19949, 19961, 19963, 19973, 19979, 19991, 19993],
[19997, 20011, 20021, 20023, 20029, 20047, 20051, 20063, 20071, 20089],
[20101, 20107, 20113, 20117, 20123, 20129, 20143, 20147, 20149, 20161],
[20173, 20177, 20183, 20201, 20219, 20231, 20233, 20249, 20261, 20269],
[20287, 20297, 20323, 20327, 20333, 20341, 20347, 20353, 20357, 20359],
[20369, 20389, 20393, 20399, 20407, 20411, 20431, 20441, 20443, 20477],
[20479, 20483, 20507, 20509, 20521, 20533, 20543, 20549, 20551, 20563],
[20593, 20599, 20611, 20627, 20639, 20641, 20663, 20681, 20693, 20707],
[20717, 20719, 20731, 20743, 20747, 20749, 20753, 20759, 20771, 20773],
[20789, 20807, 20809, 20849, 20857, 20873, 20879, 20887, 20897, 20899],
[20903, 20921, 20929, 20939, 20947, 20959, 20963, 20981, 20983, 21001],
[21011, 21013, 21017, 21019, 21023, 21031, 21059, 21061, 21067, 21089],
[21101, 21107, 21121, 21139, 21143, 21149, 21157, 21163, 21169, 21179],
[21187, 21191, 21193, 21211, 21221, 21227, 21247, 21269, 21277, 21283],
[21313, 21317, 21319, 21323, 21341, 21347, 21377, 21379, 21383, 21391],
[21397, 21401, 21407, 21419, 21433, 21467, 21481, 21487, 21491, 21493],
[21499, 21503, 21517, 21521, 21523, 21529, 21557, 21559, 21563, 21569],
[21577, 21587, 21589, 21599, 21601, 21611, 21613, 21617, 21647, 21649],
[21661, 21673, 21683, 21701, 21713, 21727, 21737, 21739, 21751, 21757],
[21767, 21773, 21787, 21799, 21803, 21817, 21821, 21839, 21841, 21851],
[21859, 21863, 21871, 21881, 21893, 21911, 21929, 21937, 21943, 21961],
[21977, 21991, 21997, 22003, 22013, 22027, 22031, 22037, 22039, 22051],
[22063, 22067, 22073, 22079, 22091, 22093, 22109, 22111, 22123, 22129],
[22133, 22147, 22153, 22157, 22159, 22171, 22189, 22193, 22229, 22247],
[22259, 22271, 22273, 22277, 22279, 22283, 22291, 22303, 22307, 22343],
[22349, 22367, 22369, 22381, 22391, 22397, 22409, 22433, 22441, 22447],
[22453, 22469, 22481, 22483, 22501, 22511, 22531, 22541, 22543, 22549],
[22567, 22571, 22573, 22613, 22619, 22621, 22637, 22639, 22643, 22651],
[22669, 22679, 22691, 22697, 22699, 22709, 22717, 22721, 22727, 22739],
[22741, 22751, 22769, 22777, 22783, 22787, 22807, 22811, 22817, 22853],
[22859, 22861, 22871, 22877, 22901, 22907, 22921, 22937, 22943, 22961],
[22963, 22973, 22993, 23003, 23011, 23017, 23021, 23027, 23029, 23039],
[23041, 23053, 23057, 23059, 23063, 23071, 23081, 23087, 23099, 23117],
[23131, 23143, 23159, 23167, 23173, 23189, 23197, 23201, 23203, 23209],
[23227, 23251, 23269, 23279, 23291, 23293, 23297, 23311, 23321, 23327],
[23333, 23339, 23357, 23369, 23371, 23399, 23417, 23431, 23447, 23459],
[23473, 23497, 23509, 23531, 23537, 23539, 23549, 23557, 23561, 23563],
[23567, 23581, 23593, 23599, 23603, 23609, 23623, 23627, 23629, 23633],
[23663, 23669, 23671, 23677, 23687, 23689, 23719, 23741, 23743, 23747],
[23753, 23761, 23767, 23773, 23789, 23801, 23813, 23819, 23827, 23831],
[23833, 23857, 23869, 23873, 23879, 23887, 23893, 23899, 23909, 23911],
[23917, 23929, 23957, 23971, 23977, 23981, 23993, 24001, 24007, 24019],
[24023, 24029, 24043, 24049, 24061, 24071, 24077, 24083, 24091, 24097],
[24103, 24107, 24109, 24113, 24121, 24133, 24137, 24151, 24169, 24179],
[24181, 24197, 24203, 24223, 24229, 24239, 24247, 24251, 24281, 24317],
[24329, 24337, 24359, 24371, 24373, 24379, 24391, 24407, 24413, 24419],
[24421, 24439, 24443, 24469, 24473, 24481, 24499, 24509, 24517, 24527],
[24533, 24547, 24551, 24571, 24593, 24611, 24623, 24631, 24659, 24671],
[24677, 24683, 24691, 24697, 24709, 24733, 24749, 24763, 24767, 24781],
[24793, 24799, 24809, 24821, 24841, 24847, 24851, 24859, 24877, 24889],
[24907, 24917, 24919, 24923, 24943, 24953, 24967, 24971, 24977, 24979],
[24989, 25013, 25031, 25033, 25037, 25057, 25073, 25087, 25097, 25111],
[25117, 25121, 25127, 25147, 25153, 25163, 25169, 25171, 25183, 25189],
[25219, 25229, 25237, 25243, 25247, 25253, 25261, 25301, 25303, 25307],
[25309, 25321, 25339, 25343, 25349, 25357, 25367, 25373, 25391, 25409],
[25411, 25423, 25439, 25447, 25453, 25457, 25463, 25469, 25471, 25523],
[25537, 25541, 25561, 25577, 25579, 25583, 25589, 25601, 25603, 25609],
[25621, 25633, 25639, 25643, 25657, 25667, 25673, 25679, 25693, 25703],
[25717, 25733, 25741, 25747, 25759, 25763, 25771, 25793, 25799, 25801],
[25819, 25841, 25847, 25849, 25867, 25873, 25889, 25903, 25913, 25919],
[25931, 25933, 25939, 25943, 25951, 25969, 25981, 25997, 25999, 26003],
[26017, 26021, 26029, 26041, 26053, 26083, 26099, 26107, 26111, 26113],
[26119, 26141, 26153, 26161, 26171, 26177, 26183, 26189, 26203, 26209],
[26227, 26237, 26249, 26251, 26261, 26263, 26267, 26293, 26297, 26309],
[26317, 26321, 26339, 26347, 26357, 26371, 26387, 26393, 26399, 26407],
[26417, 26423, 26431, 26437, 26449, 26459, 26479, 26489, 26497, 26501],
[26513, 26539, 26557, 26561, 26573, 26591, 26597, 26627, 26633, 26641],
[26647, 26669, 26681, 26683, 26687, 26693, 26699, 26701, 26711, 26713],
[26717, 26723, 26729, 26731, 26737, 26759, 26777, 26783, 26801, 26813],
[26821, 26833, 26839, 26849, 26861, 26863, 26879, 26881, 26891, 26893],
[26903, 26921, 26927, 26947, 26951, 26953, 26959, 26981, 26987, 26993],
[27011, 27017, 27031, 27043, 27059, 27061, 27067, 27073, 27077, 27091],
[27103, 27107, 27109, 27127, 27143, 27179, 27191, 27197, 27211, 27239],
[27241, 27253, 27259, 27271, 27277, 27281, 27283, 27299, 27329, 27337],
[27361, 27367, 27397, 27407, 27409, 27427, 27431, 27437, 27449, 27457],
[27479, 27481, 27487, 27509, 27527, 27529, 27539, 27541, 27551, 27581],
[27583, 27611, 27617, 27631, 27647, 27653, 27673, 27689, 27691, 27697],
[27701, 27733, 27737, 27739, 27743, 27749, 27751, 27763, 27767, 27773],
[27779, 27791, 27793, 27799, 27803, 27809, 27817, 27823, 27827, 27847],
[27851, 27883, 27893, 27901, 27917, 27919, 27941, 27943, 27947, 27953],
[27961, 27967, 27983, 27997, 28001, 28019, 28027, 28031, 28051, 28057],
[28069, 28081, 28087, 28097, 28099, 28109, 28111, 28123, 28151, 28163],
[28181, 28183, 28201, 28211, 28219, 28229, 28277, 28279, 28283, 28289],
[28297, 28307, 28309, 28319, 28349, 28351, 28387, 28393, 28403, 28409],
[28411, 28429, 28433, 28439, 28447, 28463, 28477, 28493, 28499, 28513],
[28517, 28537, 28541, 28547, 28549, 28559, 28571, 28573, 28579, 28591],
[28597, 28603, 28607, 28619, 28621, 28627, 28631, 28643, 28649, 28657],
[28661, 28663, 28669, 28687, 28697, 28703, 28711, 28723, 28729, 28751],
[28753, 28759, 28771, 28789, 28793, 28807, 28813, 28817, 28837, 28843],
[28859, 28867, 28871, 28879, 28901, 28909, 28921, 28927, 28933, 28949],
[28961, 28979, 29009, 29017, 29021, 29023, 29027, 29033, 29059, 29063],
[29077, 29101, 29123, 29129, 29131, 29137, 29147, 29153, 29167, 29173],
[29179, 29191, 29201, 29207, 29209, 29221, 29231, 29243, 29251, 29269],
[29287, 29297, 29303, 29311, 29327, 29333, 29339, 29347, 29363, 29383],
[29387, 29389, 29399, 29401, 29411, 29423, 29429, 29437, 29443, 29453],
[29473, 29483, 29501, 29527, 29531, 29537, 29567, 29569, 29573, 29581],
[29587, 29599, 29611, 29629, 29633, 29641, 29663, 29669, 29671, 29683],
[29717, 29723, 29741, 29753, 29759, 29761, 29789, 29803, 29819, 29833],
[29837, 29851, 29863, 29867, 29873, 29879, 29881, 29917, 29921, 29927],
[29947, 29959, 29983, 29989, 30011, 30013, 30029, 30047, 30059, 30071],
[30089, 30091, 30097, 30103, 30109, 30113, 30119, 30133, 30137, 30139],
[30161, 30169, 30181, 30187, 30197, 30203, 30211, 30223, 30241, 30253],
[30259, 30269, 30271, 30293, 30307, 30313, 30319, 30323, 30341, 30347],
[30367, 30389, 30391, 30403, 30427, 30431, 30449, 30467, 30469, 30491],
[30493, 30497, 30509, 30517, 30529, 30539, 30553, 30557, 30559, 30577],
[30593, 30631, 30637, 30643, 30649, 30661, 30671, 30677, 30689, 30697],
[30703, 30707, 30713, 30727, 30757, 30763, 30773, 30781, 30803, 30809],
[30817, 30829, 30839, 30841, 30851, 30853, 30859, 30869, 30871, 30881],
[30893, 30911, 30931, 30937, 30941, 30949, 30971, 30977, 30983, 31013],
[31019, 31033, 31039, 31051, 31063, 31069, 31079, 31081, 31091, 31121],
[31123, 31139, 31147, 31151, 31153, 31159, 31177, 31181, 31183, 31189],
[31193, 31219, 31223, 31231, 31237, 31247, 31249, 31253, 31259, 31267],
[31271, 31277, 31307, 31319, 31321, 31327, 31333, 31337, 31357, 31379],
[31387, 31391, 31393, 31397, 31469, 31477, 31481, 31489, 31511, 31513],
[31517, 31531, 31541, 31543, 31547, 31567, 31573, 31583, 31601, 31607],
[31627, 31643, 31649, 31657, 31663, 31667, 31687, 31699, 31721, 31723],
[31727, 31729, 31741, 31751, 31769, 31771, 31793, 31799, 31817, 31847],
[31849, 31859, 31873, 31883, 31891, 31907, 31957, 31963, 31973, 31981],
[31991, 32003, 32009, 32027, 32029, 32051, 32057, 32059, 32063, 32069],
[32077, 32083, 32089, 32099, 32117, 32119, 32141, 32143, 32159, 32173],
[32183, 32189, 32191, 32203, 32213, 32233, 32237, 32251, 32257, 32261],
[32297, 32299, 32303, 32309, 32321, 32323, 32327, 32341, 32353, 32359],
[32363, 32369, 32371, 32377, 32381, 32401, 32411, 32413, 32423, 32429],
[32441, 32443, 32467, 32479, 32491, 32497, 32503, 32507, 32531, 32533],
[32537, 32561, 32563, 32569, 32573, 32579, 32587, 32603, 32609, 32611],
[32621, 32633, 32647, 32653, 32687, 32693, 32707, 32713, 32717, 32719],
[32749, 32771, 32779, 32783, 32789, 32797, 32801, 32803, 32831, 32833],
[32839, 32843, 32869, 32887, 32909, 32911, 32917, 32933, 32939, 32941],
[32957, 32969, 32971, 32983, 32987, 32993, 32999, 33013, 33023, 33029],
[33037, 33049, 33053, 33071, 33073, 33083, 33091, 33107, 33113, 33119],
[33149, 33151, 33161, 33179, 33181, 33191, 33199, 33203, 33211, 33223],
[33247, 33287, 33289, 33301, 33311, 33317, 33329, 33331, 33343, 33347],
[33349, 33353, 33359, 33377, 33391, 33403, 33409, 33413, 33427, 33457],
[33461, 33469, 33479, 33487, 33493, 33503, 33521, 33529, 33533, 33547],
[33563, 33569, 33577, 33581, 33587, 33589, 33599, 33601, 33613, 33617],
[33619, 33623, 33629, 33637, 33641, 33647, 33679, 33703, 33713, 33721],
[33739, 33749, 33751, 33757, 33767, 33769, 33773, 33791, 33797, 33809],
[33811, 33827, 33829, 33851, 33857, 33863, 33871, 33889, 33893, 33911],
[33923, 33931, 33937, 33941, 33961, 33967, 33997, 34019, 34031, 34033],
[34039, 34057, 34061, 34123, 34127, 34129, 34141, 34147, 34157, 34159],
[34171, 34183, 34211, 34213, 34217, 34231, 34253, 34259, 34261, 34267],
[34273, 34283, 34297, 34301, 34303, 34313, 34319, 34327, 34337, 34351],
[34361, 34367, 34369, 34381, 34403, 34421, 34429, 34439, 34457, 34469],
[34471, 34483, 34487, 34499, 34501, 34511, 34513, 34519, 34537, 34543],
[34549, 34583, 34589, 34591, 34603, 34607, 34613, 34631, 34649, 34651],
[34667, 34673, 34679, 34687, 34693, 34703, 34721, 34729, 34739, 34747],
[34757, 34759, 34763, 34781, 34807, 34819, 34841, 34843, 34847, 34849],
[34871, 34877, 34883, 34897, 34913, 34919, 34939, 34949, 34961, 34963],
[34981, 35023, 35027, 35051, 35053, 35059, 35069, 35081, 35083, 35089],
[35099, 35107, 35111, 35117, 35129, 35141, 35149, 35153, 35159, 35171],
[35201, 35221, 35227, 35251, 35257, 35267, 35279, 35281, 35291, 35311],
[35317, 35323, 35327, 35339, 35353, 35363, 35381, 35393, 35401, 35407],
[35419, 35423, 35437, 35447, 35449, 35461, 35491, 35507, 35509, 35521],
[35527, 35531, 35533, 35537, 35543, 35569, 35573, 35591, 35593, 35597],
[35603, 35617, 35671, 35677, 35729, 35731, 35747, 35753, 35759, 35771],
[35797, 35801, 35803, 35809, 35831, 35837, 35839, 35851, 35863, 35869],
[35879, 35897, 35899, 35911, 35923, 35933, 35951, 35963, 35969, 35977],
[35983, 35993, 35999, 36007, 36011, 36013, 36017, 36037, 36061, 36067],
[36073, 36083, 36097, 36107, 36109, 36131, 36137, 36151, 36161, 36187],
[36191, 36209, 36217, 36229, 36241, 36251, 36263, 36269, 36277, 36293],
[36299, 36307, 36313, 36319, 36341, 36343, 36353, 36373, 36383, 36389],
[36433, 36451, 36457, 36467, 36469, 36473, 36479, 36493, 36497, 36523],
[36527, 36529, 36541, 36551, 36559, 36563, 36571, 36583, 36587, 36599],
[36607, 36629, 36637, 36643, 36653, 36671, 36677, 36683, 36691, 36697],
[36709, 36713, 36721, 36739, 36749, 36761, 36767, 36779, 36781, 36787],
[36791, 36793, 36809, 36821, 36833, 36847, 36857, 36871, 36877, 36887],
[36899, 36901, 36913, 36919, 36923, 36929, 36931, 36943, 36947, 36973],
[36979, 36997, 37003, 37013, 37019, 37021, 37039, 37049, 37057, 37061],
[37087, 37097, 37117, 37123, 37139, 37159, 37171, 37181, 37189, 37199],
[37201, 37217, 37223, 37243, 37253, 37273, 37277, 37307, 37309, 37313],
[37321, 37337, 37339, 37357, 37361, 37363, 37369, 37379, 37397, 37409],
[37423, 37441, 37447, 37463, 37483, 37489, 37493, 37501, 37507, 37511],
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039]]}}]}
============================================================================== ==============================================================================
================================================================================ ================================================================================
MERGE RESULT: ✅ SUCCESS MERGE RESULT: ✅ SUCCESS
================================================================================ ================================================================================
Final result length: 31743 chars Final result length: 414 chars
Final result (COMPLETE): Final result (COMPLETE):
================================================================================ ================================================================================
{ {
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
},
"documents": [
{
"id": "doc1",
"title": "Document 1",
"sections": [
{
"id": "sec1",
"content_type": "heading",
"elements": [ "elements": [
{ {
"type": "table", "type": "heading",
"content": { "content": {
"headers": ["Spalte1", "Spalte2", "Spalte3", "Spalte4", "Spalte5", "Spalte6", "Spalte7", "Spalte8", "Spalte9", "Spalte10"], "text": "Introduction"}}]}]}]}
"rows": [
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29],
[31, 37, 41, 43, 47, 53, 59, 61, 67, 71],
[73, 79, 83, 89, 97, 101, 103, 107, 109, 113],
[127, 131, 137, 139, 149, 151, 157, 163, 167, 173],
[179, 181, 191, 193, 197, 199, 211, 223, 227, 229],
[233, 239, 241, 251, 257, 263, 269, 271, 277, 281],
[283, 293, 307, 311, 313, 317, 331, 337, 347, 349],
[353, 359, 367, 373, 379, 383, 389, 397, 401, 409],
[419, 421, 431, 433, 439, 443, 449, 457, 461, 463],
[467, 479, 487, 491, 499, 503, 509, 521, 523, 541],
[547, 557, 563, 569, 571, 577, 587, 593, 599, 601],
[607, 613, 617, 619, 631, 641, 643, 647, 653, 659],
[661, 673, 677, 683, 691, 701, 709, 719, 727, 733],
[739, 743, 751, 757, 761, 769, 773, 787, 797, 809],
[811, 821, 823, 827, 829, 839, 853, 857, 859, 863],
[877, 881, 883, 887, 907, 911, 919, 929, 937, 941],
[947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013],
[1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069],
[1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151],
[1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223],
[1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291],
[1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373],
[1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451],
[1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511],
[1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583],
[1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657],
[1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733],
[1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811],
[1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889],
[1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987],
[1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053],
[2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129],
[2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213],
[2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287],
[2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357],
[2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423],
[2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531],
[2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617],
[2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687],
[2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741],
[2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819],
[2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903],
[2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999],
[3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079],
[3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181],
[3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257],
[3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331],
[3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413],
[3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511],
[3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571],
[3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643],
[3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727],
[3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821],
[3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907],
[3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989],
[4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057],
[4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139],
[4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231],
[4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297],
[4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409],
[4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493],
[4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583],
[4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657],
[4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751],
[4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831],
[4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937],
[4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003],
[5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087],
[5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179],
[5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279],
[5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387],
[5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443],
[5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521],
[5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639],
[5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693],
[5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791],
[5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857],
[5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939],
[5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053],
[6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133],
[6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221],
[6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301],
[6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367],
[6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473],
[6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571],
[6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673],
[6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761],
[6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833],
[6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917],
[6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997],
[7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103],
[7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207],
[7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297],
[7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411],
[7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499],
[7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561],
[7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643],
[7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723],
[7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829],
[7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919],
[7927, 7933, 7937, 7949, 7951, 7963, 7993, 8009, 8011, 8017],
[8039, 8053, 8059, 8069, 8081, 8087, 8089, 8093, 8101, 8111],
[8117, 8123, 8147, 8161, 8167, 8171, 8179, 8191, 8209, 8219],
[8221, 8231, 8233, 8237, 8243, 8263, 8269, 8273, 8287, 8291],
[8293, 8297, 8311, 8317, 8329, 8353, 8363, 8369, 8377, 8387],
[8389, 8419, 8423, 8429, 8431, 8443, 8447, 8461, 8467, 8501],
[8513, 8521, 8527, 8537, 8539, 8543, 8563, 8573, 8581, 8597],
[8599, 8609, 8623, 8627, 8629, 8641, 8647, 8663, 8669, 8677],
[8681, 8689, 8693, 8699, 8707, 8713, 8719, 8731, 8737, 8741],
[8747, 8753, 8761, 8779, 8783, 8803, 8807, 8819, 8821, 8831],
[8837, 8839, 8849, 8861, 8863, 8867, 8887, 8893, 8923, 8929],
[8933, 8941, 8951, 8963, 8969, 8971, 8999, 9001, 9007, 9011],
[9013, 9029, 9041, 9043, 9049, 9059, 9067, 9091, 9103, 9109],
[9127, 9133, 9137, 9151, 9157, 9161, 9173, 9181, 9187, 9199],
[9203, 9209, 9221, 9227, 9239, 9241, 9257, 9277, 9281, 9283],
[9293, 9311, 9319, 9323, 9337, 9341, 9343, 9349, 9371, 9377],
[9391, 9397, 9403, 9413, 9419, 9421, 9431, 9433, 9437, 9439],
[9461, 9463, 9467, 9473, 9479, 9491, 9497, 9511, 9521, 9533],
[9539, 9547, 9551, 9587, 9601, 9613, 9619, 9623, 9629, 9631],
[9643, 9649, 9661, 9677, 9679, 9689, 9697, 9719, 9721, 9733],
[9739, 9743, 9749, 9767, 9769, 9781, 9787, 9791, 9803, 9811],
[9817, 9829, 9833, 9839, 9851, 9857, 9859, 9871, 9883, 9887],
[9901, 9907, 9923, 9929, 9931, 9941, 9949, 9967, 9973, 10007],
[10009, 10037, 10039, 10061, 10067, 10069, 10079, 10091, 10093, 10099],
[10103, 10111, 10133, 10139, 10141, 10151, 10159, 10163, 10169, 10177],
[10181, 10193, 10211, 10223, 10243, 10247, 10253, 10259, 10267, 10271],
[10273, 10289, 10301, 10303, 10313, 10321, 10331, 10333, 10337, 10343],
[10357, 10369, 10391, 10399, 10427, 10429, 10433, 10453, 10457, 10459],
[10463, 10477, 10487, 10499, 10501, 10513, 10529, 10531, 10559, 10567],
[10589, 10597, 10601, 10607, 10613, 10627, 10631, 10639, 10651, 10657],
[10663, 10667, 10687, 10691, 10709, 10711, 10723, 10729, 10733, 10739],
[10753, 10771, 10781, 10789, 10799, 10831, 10837, 10847, 10853, 10859],
[10861, 10867, 10883, 10889, 10891, 10903, 10909, 10937, 10939, 10949],
[10957, 10973, 10979, 10987, 10993, 11003, 11027, 11047, 11057, 11059],
[11069, 11071, 11083, 11087, 11093, 11113, 11117, 11119, 11131, 11149],
[11159, 11161, 11171, 11173, 11177, 11197, 11213, 11239, 11243, 11251],
[11257, 11261, 11273, 11279, 11287, 11299, 11311, 11317, 11321, 11329],
[11351, 11353, 11369, 11383, 11393, 11399, 11411, 11423, 11437, 11443],
[11447, 11467, 11471, 11483, 11489, 11491, 11497, 11503, 11519, 11527],
[11549, 11551, 11579, 11587, 11593, 11597, 11617, 11621, 11633, 11657],
[11677, 11681, 11689, 11699, 11701, 11717, 11719, 11731, 11743, 11777],
[11779, 11783, 11789, 11801, 11807, 11813, 11821, 11827, 11831, 11833],
[11839, 11863, 11867, 11887, 11897, 11903, 11909, 11923, 11927, 11933],
[11939, 11941, 11953, 11959, 11969, 11971, 11981, 11987, 12007, 12011],
[12037, 12041, 12043, 12049, 12071, 12073, 12097, 12101, 12107, 12109],
[12113, 12119, 12143, 12149, 12157, 12161, 12163, 12197, 12203, 12211],
[12227, 12239, 12241, 12251, 12253, 12263, 12269, 12277, 12281, 12289],
[12301, 12323, 12329, 12343, 12347, 12373, 12377, 12379, 12391, 12401],
[12409, 12413, 12421, 12433, 12437, 12451, 12457, 12473, 12479, 12487],
[12491, 12497, 12503, 12511, 12517, 12527, 12539, 12541, 12547, 12553],
[12569, 12577, 12583, 12589, 12601, 12611, 12613, 12619, 12637, 12641],
[12647, 12653, 12659, 12671, 12689, 12697, 12703, 12713, 12721, 12739],
[12743, 12757, 12763, 12781, 12791, 12799, 12809, 12821, 12823, 12829],
[12841, 12853, 12889, 12893, 12899, 12907, 12911, 12917, 12919, 12923],
[12941, 12953, 12959, 12967, 12973, 12979, 12983, 13001, 13003, 13007],
[13009, 13033, 13037, 13043, 13049, 13063, 13093, 13099, 13103, 13109],
[13121, 13127, 13147, 13151, 13159, 13163, 13171, 13177, 13183, 13187],
[13217, 13219, 13229, 13241, 13249, 13259, 13267, 13291, 13297, 13309],
[13313, 13327, 13331, 13337, 13339, 13367, 13381, 13397, 13399, 13411],
[13417, 13421, 13441, 13451, 13457, 13463, 13469, 13477, 13487, 13499],
[13513, 13523, 13537, 13553, 13567, 13577, 13591, 13597, 13613, 13619],
[13627, 13633, 13649, 13669, 13679, 13681, 13687, 13691, 13693, 13697],
[13709, 13711, 13721, 13723, 13729, 13751, 13757, 13759, 13763, 13781],
[13789, 13799, 13807, 13829, 13831, 13841, 13859, 13873, 13877, 13879],
[13883, 13901, 13903, 13907, 13913, 13921, 13931, 13933, 13963, 13967],
[13997, 13999, 14009, 14011, 14029, 14033, 14051, 14057, 14071, 14081],
[14083, 14087, 14107, 14143, 14149, 14153, 14159, 14173, 14177, 14197],
[14207, 14221, 14243, 14249, 14251, 14281, 14293, 14303, 14321, 14323],
[14327, 14341, 14347, 14369, 14387, 14389, 14401, 14407, 14411, 14419],
[14423, 14431, 14437, 14447, 14449, 14461, 14479, 14489, 14503, 14519],
[14533, 14537, 14543, 14549, 14551, 14557, 14561, 14563, 14591, 14593],
[14621, 14627, 14629, 14633, 14639, 14653, 14657, 14669, 14683, 14699],
[14713, 14717, 14723, 14731, 14737, 14741, 14747, 14753, 14759, 14767],
[14771, 14779, 14783, 14797, 14813, 14821, 14827, 14831, 14843, 14851],
[14867, 14869, 14879, 14887, 14891, 14897, 14923, 14929, 14939, 14947],
[14951, 14957, 14969, 14983, 15013, 15017, 15031, 15053, 15061, 15073],
[15077, 15083, 15091, 15101, 15107, 15121, 15131, 15137, 15139, 15149],
[15161, 15173, 15187, 15193, 15199, 15217, 15227, 15233, 15241, 15259],
[15263, 15269, 15271, 15277, 15287, 15289, 15299, 15307, 15313, 15319],
[15329, 15331, 15349, 15359, 15361, 15373, 15377, 15383, 15391, 15401],
[15413, 15427, 15439, 15443, 15451, 15461, 15467, 15473, 15493, 15497],
[15511, 15527, 15541, 15551, 15559, 15569, 15581, 15583, 15601, 15607],
[15619, 15629, 15641, 15643, 15647, 15649, 15661, 15667, 15671, 15679],
[15683, 15727, 15731, 15733, 15737, 15739, 15749, 15761, 15767, 15773],
[15787, 15791, 15797, 15803, 15809, 15817, 15823, 15859, 15877, 15881],
[15887, 15889, 15901, 15907, 15913, 15919, 15923, 15937, 15959, 15971],
[15973, 15991, 16001, 16007, 16033, 16057, 16061, 16063, 16067, 16069],
[16073, 16087, 16091, 16097, 16103, 16111, 16127, 16139, 16141, 16183],
[16187, 16189, 16193, 16217, 16223, 16229, 16231, 16249, 16253, 16267],
[16273, 16301, 16319, 16333, 16339, 16349, 16361, 16363, 16369, 16381],
[16411, 16417, 16421, 16427, 16433, 16447, 16451, 16453, 16477, 16481],
[16487, 16493, 16519, 16529, 16547, 16553, 16561, 16567, 16573, 16603],
[16607, 16619, 16631, 16633, 16649, 16651, 16657, 16661, 16673, 16691],
[16693, 16699, 16703, 16729, 16741, 16747, 16759, 16763, 16787, 16811],
[16823, 16829, 16831, 16843, 16871, 16879, 16883, 16889, 16901, 16903],
[16921, 16927, 16931, 16937, 16943, 16963, 16979, 16981, 16987, 16993],
[17011, 17021, 17027, 17029, 17033, 17041, 17047, 17053, 17077, 17093],
[17099, 17107, 17117, 17123, 17137, 17159, 17167, 17183, 17189, 17191],
[17203, 17207, 17209, 17231, 17239, 17257, 17291, 17293, 17299, 17317],
[17321, 17327, 17333, 17341, 17351, 17359, 17377, 17383, 17387, 17389],
[17393, 17401, 17417, 17419, 17431, 17443, 17449, 17467, 17471, 17477],
[17483, 17489, 17491, 17497, 17509, 17519, 17539, 17551, 17569, 17573],
[17579, 17581, 17597, 17599, 17609, 17623, 17627, 17657, 17659, 17669],
[17681, 17683, 17707, 17713, 17729, 17737, 17747, 17749, 17761, 17783],
[17789, 17791, 17807, 17827, 17837, 17839, 17851, 17863, 17881, 17891],
[17903, 17909, 17911, 17921, 17923, 17929, 17939, 17957, 17959, 17971],
[17977, 17981, 17987, 17989, 18013, 18041, 18043, 18047, 18049, 18059],
[18061, 18077, 18089, 18097, 18119, 18121, 18127, 18131, 18133, 18143],
[18149, 18169, 18181, 18191, 18199, 18211, 18217, 18223, 18229, 18233],
[18251, 18253, 18257, 18269, 18287, 18289, 18301, 18307, 18311, 18313],
[18329, 18341, 18353, 18367, 18371, 18379, 18397, 18401, 18413, 18427],
[18433, 18439, 18443, 18451, 18457, 18461, 18481, 18493, 18503, 18517],
[18521, 18523, 18539, 18541, 18553, 18583, 18587, 18593, 18617, 18637],
[18661, 18671, 18679, 18691, 18701, 18713, 18719, 18731, 18743, 18749],
[18757, 18773, 18787, 18793, 18797, 18803, 18839, 18859, 18869, 18899],
[18911, 18913, 18917, 18919, 18947, 18959, 18973, 18979, 19001, 19009],
[19013, 19031, 19037, 19051, 19069, 19073, 19079, 19081, 19087, 19121],
[19139, 19141, 19157, 19163, 19181, 19183, 19207, 19211, 19213, 19219],
[19231, 19237, 19249, 19259, 19267, 19273, 19289, 19301, 19309, 19319],
[19333, 19373, 19379, 19381, 19387, 19391, 19403, 19417, 19421, 19423],
[19427, 19429, 19433, 19441, 19447, 19457, 19463, 19469, 19471, 19477],
[19483, 19501, 19507, 19531, 19541, 19543, 19553, 19559, 19571, 19577],
[19583, 19597, 19603, 19609, 19661, 19681, 19687, 19697, 19699, 19709],
[19717, 19727, 19739, 19751, 19753, 19759, 19763, 19777, 19793, 19801],
[19813, 19819, 19841, 19843, 19853, 19861, 19867, 19889, 19891, 19913],
[19919, 19927, 19937, 19949, 19961, 19963, 19973, 19979, 19991, 19993],
[19997, 20011, 20021, 20023, 20029, 20047, 20051, 20063, 20071, 20089],
[20101, 20107, 20113, 20117, 20123, 20129, 20143, 20147, 20149, 20161],
[20173, 20177, 20183, 20201, 20219, 20231, 20233, 20249, 20261, 20269],
[20287, 20297, 20323, 20327, 20333, 20341, 20347, 20353, 20357, 20359],
[20369, 20389, 20393, 20399, 20407, 20411, 20431, 20441, 20443, 20477],
[20479, 20483, 20507, 20509, 20521, 20533, 20543, 20549, 20551, 20563],
[20593, 20599, 20611, 20627, 20639, 20641, 20663, 20681, 20693, 20707],
[20717, 20719, 20731, 20743, 20747, 20749, 20753, 20759, 20771, 20773],
[20789, 20807, 20809, 20849, 20857, 20873, 20879, 20887, 20897, 20899],
[20903, 20921, 20929, 20939, 20947, 20959, 20963, 20981, 20983, 21001],
[21011, 21013, 21017, 21019, 21023, 21031, 21059, 21061, 21067, 21089],
[21101, 21107, 21121, 21139, 21143, 21149, 21157, 21163, 21169, 21179],
[21187, 21191, 21193, 21211, 21221, 21227, 21247, 21269, 21277, 21283],
[21313, 21317, 21319, 21323, 21341, 21347, 21377, 21379, 21383, 21391],
[21397, 21401, 21407, 21419, 21433, 21467, 21481, 21487, 21491, 21493],
[21499, 21503, 21517, 21521, 21523, 21529, 21557, 21559, 21563, 21569],
[21577, 21587, 21589, 21599, 21601, 21611, 21613, 21617, 21647, 21649],
[21661, 21673, 21683, 21701, 21713, 21727, 21737, 21739, 21751, 21757],
[21767, 21773, 21787, 21799, 21803, 21817, 21821, 21839, 21841, 21851],
[21859, 21863, 21871, 21881, 21893, 21911, 21929, 21937, 21943, 21961],
[21977, 21991, 21997, 22003, 22013, 22027, 22031, 22037, 22039, 22051],
[22063, 22067, 22073, 22079, 22091, 22093, 22109, 22111, 22123, 22129],
[22133, 22147, 22153, 22157, 22159, 22171, 22189, 22193, 22229, 22247],
[22259, 22271, 22273, 22277, 22279, 22283, 22291, 22303, 22307, 22343],
[22349, 22367, 22369, 22381, 22391, 22397, 22409, 22433, 22441, 22447],
[22453, 22469, 22481, 22483, 22501, 22511, 22531, 22541, 22543, 22549],
[22567, 22571, 22573, 22613, 22619, 22621, 22637, 22639, 22643, 22651],
[22669, 22679, 22691, 22697, 22699, 22709, 22717, 22721, 22727, 22739],
[22741, 22751, 22769, 22777, 22783, 22787, 22807, 22811, 22817, 22853],
[22859, 22861, 22871, 22877, 22901, 22907, 22921, 22937, 22943, 22961],
[22963, 22973, 22993, 23003, 23011, 23017, 23021, 23027, 23029, 23039],
[23041, 23053, 23057, 23059, 23063, 23071, 23081, 23087, 23099, 23117],
[23131, 23143, 23159, 23167, 23173, 23189, 23197, 23201, 23203, 23209],
[23227, 23251, 23269, 23279, 23291, 23293, 23297, 23311, 23321, 23327],
[23333, 23339, 23357, 23369, 23371, 23399, 23417, 23431, 23447, 23459],
[23473, 23497, 23509, 23531, 23537, 23539, 23549, 23557, 23561, 23563],
[23567, 23581, 23593, 23599, 23603, 23609, 23623, 23627, 23629, 23633],
[23663, 23669, 23671, 23677, 23687, 23689, 23719, 23741, 23743, 23747],
[23753, 23761, 23767, 23773, 23789, 23801, 23813, 23819, 23827, 23831],
[23833, 23857, 23869, 23873, 23879, 23887, 23893, 23899, 23909, 23911],
[23917, 23929, 23957, 23971, 23977, 23981, 23993, 24001, 24007, 24019],
[24023, 24029, 24043, 24049, 24061, 24071, 24077, 24083, 24091, 24097],
[24103, 24107, 24109, 24113, 24121, 24133, 24137, 24151, 24169, 24179],
[24181, 24197, 24203, 24223, 24229, 24239, 24247, 24251, 24281, 24317],
[24329, 24337, 24359, 24371, 24373, 24379, 24391, 24407, 24413, 24419],
[24421, 24439, 24443, 24469, 24473, 24481, 24499, 24509, 24517, 24527],
[24533, 24547, 24551, 24571, 24593, 24611, 24623, 24631, 24659, 24671],
[24677, 24683, 24691, 24697, 24709, 24733, 24749, 24763, 24767, 24781],
[24793, 24799, 24809, 24821, 24841, 24847, 24851, 24859, 24877, 24889],
[24907, 24917, 24919, 24923, 24943, 24953, 24967, 24971, 24977, 24979],
[24989, 25013, 25031, 25033, 25037, 25057, 25073, 25087, 25097, 25111],
[25117, 25121, 25127, 25147, 25153, 25163, 25169, 25171, 25183, 25189],
[25219, 25229, 25237, 25243, 25247, 25253, 25261, 25301, 25303, 25307],
[25309, 25321, 25339, 25343, 25349, 25357, 25367, 25373, 25391, 25409],
[25411, 25423, 25439, 25447, 25453, 25457, 25463, 25469, 25471, 25523],
[25537, 25541, 25561, 25577, 25579, 25583, 25589, 25601, 25603, 25609],
[25621, 25633, 25639, 25643, 25657, 25667, 25673, 25679, 25693, 25703],
[25717, 25733, 25741, 25747, 25759, 25763, 25771, 25793, 25799, 25801],
[25819, 25841, 25847, 25849, 25867, 25873, 25889, 25903, 25913, 25919],
[25931, 25933, 25939, 25943, 25951, 25969, 25981, 25997, 25999, 26003],
[26017, 26021, 26029, 26041, 26053, 26083, 26099, 26107, 26111, 26113],
[26119, 26141, 26153, 26161, 26171, 26177, 26183, 26189, 26203, 26209],
[26227, 26237, 26249, 26251, 26261, 26263, 26267, 26293, 26297, 26309],
[26317, 26321, 26339, 26347, 26357, 26371, 26387, 26393, 26399, 26407],
[26417, 26423, 26431, 26437, 26449, 26459, 26479, 26489, 26497, 26501],
[26513, 26539, 26557, 26561, 26573, 26591, 26597, 26627, 26633, 26641],
[26647, 26669, 26681, 26683, 26687, 26693, 26699, 26701, 26711, 26713],
[26717, 26723, 26729, 26731, 26737, 26759, 26777, 26783, 26801, 26813],
[26821, 26833, 26839, 26849, 26861, 26863, 26879, 26881, 26891, 26893],
[26903, 26921, 26927, 26947, 26951, 26953, 26959, 26981, 26987, 26993],
[27011, 27017, 27031, 27043, 27059, 27061, 27067, 27073, 27077, 27091],
[27103, 27107, 27109, 27127, 27143, 27179, 27191, 27197, 27211, 27239],
[27241, 27253, 27259, 27271, 27277, 27281, 27283, 27299, 27329, 27337],
[27361, 27367, 27397, 27407, 27409, 27427, 27431, 27437, 27449, 27457],
[27479, 27481, 27487, 27509, 27527, 27529, 27539, 27541, 27551, 27581],
[27583, 27611, 27617, 27631, 27647, 27653, 27673, 27689, 27691, 27697],
[27701, 27733, 27737, 27739, 27743, 27749, 27751, 27763, 27767, 27773],
[27779, 27791, 27793, 27799, 27803, 27809, 27817, 27823, 27827, 27847],
[27851, 27883, 27893, 27901, 27917, 27919, 27941, 27943, 27947, 27953],
[27961, 27967, 27983, 27997, 28001, 28019, 28027, 28031, 28051, 28057],
[28069, 28081, 28087, 28097, 28099, 28109, 28111, 28123, 28151, 28163],
[28181, 28183, 28201, 28211, 28219, 28229, 28277, 28279, 28283, 28289],
[28297, 28307, 28309, 28319, 28349, 28351, 28387, 28393, 28403, 28409],
[28411, 28429, 28433, 28439, 28447, 28463, 28477, 28493, 28499, 28513],
[28517, 28537, 28541, 28547, 28549, 28559, 28571, 28573, 28579, 28591],
[28597, 28603, 28607, 28619, 28621, 28627, 28631, 28643, 28649, 28657],
[28661, 28663, 28669, 28687, 28697, 28703, 28711, 28723, 28729, 28751],
[28753, 28759, 28771, 28789, 28793, 28807, 28813, 28817, 28837, 28843],
[28859, 28867, 28871, 28879, 28901, 28909, 28921, 28927, 28933, 28949],
[28961, 28979, 29009, 29017, 29021, 29023, 29027, 29033, 29059, 29063],
[29077, 29101, 29123, 29129, 29131, 29137, 29147, 29153, 29167, 29173],
[29179, 29191, 29201, 29207, 29209, 29221, 29231, 29243, 29251, 29269],
[29287, 29297, 29303, 29311, 29327, 29333, 29339, 29347, 29363, 29383],
[29387, 29389, 29399, 29401, 29411, 29423, 29429, 29437, 29443, 29453],
[29473, 29483, 29501, 29527, 29531, 29537, 29567, 29569, 29573, 29581],
[29587, 29599, 29611, 29629, 29633, 29641, 29663, 29669, 29671, 29683],
[29717, 29723, 29741, 29753, 29759, 29761, 29789, 29803, 29819, 29833],
[29837, 29851, 29863, 29867, 29873, 29879, 29881, 29917, 29921, 29927],
[29947, 29959, 29983, 29989, 30011, 30013, 30029, 30047, 30059, 30071],
[30089, 30091, 30097, 30103, 30109, 30113, 30119, 30133, 30137, 30139],
[30161, 30169, 30181, 30187, 30197, 30203, 30211, 30223, 30241, 30253],
[30259, 30269, 30271, 30293, 30307, 30313, 30319, 30323, 30341, 30347],
[30367, 30389, 30391, 30403, 30427, 30431, 30449, 30467, 30469, 30491],
[30493, 30497, 30509, 30517, 30529, 30539, 30553, 30557, 30559, 30577],
[30593, 30631, 30637, 30643, 30649, 30661, 30671, 30677, 30689, 30697],
[30703, 30707, 30713, 30727, 30757, 30763, 30773, 30781, 30803, 30809],
[30817, 30829, 30839, 30841, 30851, 30853, 30859, 30869, 30871, 30881],
[30893, 30911, 30931, 30937, 30941, 30949, 30971, 30977, 30983, 31013],
[31019, 31033, 31039, 31051, 31063, 31069, 31079, 31081, 31091, 31121],
[31123, 31139, 31147, 31151, 31153, 31159, 31177, 31181, 31183, 31189],
[31193, 31219, 31223, 31231, 31237, 31247, 31249, 31253, 31259, 31267],
[31271, 31277, 31307, 31319, 31321, 31327, 31333, 31337, 31357, 31379],
[31387, 31391, 31393, 31397, 31469, 31477, 31481, 31489, 31511, 31513],
[31517, 31531, 31541, 31543, 31547, 31567, 31573, 31583, 31601, 31607],
[31627, 31643, 31649, 31657, 31663, 31667, 31687, 31699, 31721, 31723],
[31727, 31729, 31741, 31751, 31769, 31771, 31793, 31799, 31817, 31847],
[31849, 31859, 31873, 31883, 31891, 31907, 31957, 31963, 31973, 31981],
[31991, 32003, 32009, 32027, 32029, 32051, 32057, 32059, 32063, 32069],
[32077, 32083, 32089, 32099, 32117, 32119, 32141, 32143, 32159, 32173],
[32183, 32189, 32191, 32203, 32213, 32233, 32237, 32251, 32257, 32261],
[32297, 32299, 32303, 32309, 32321, 32323, 32327, 32341, 32353, 32359],
[32363, 32369, 32371, 32377, 32381, 32401, 32411, 32413, 32423, 32429],
[32441, 32443, 32467, 32479, 32491, 32497, 32503, 32507, 32531, 32533],
[32537, 32561, 32563, 32569, 32573, 32579, 32587, 32603, 32609, 32611],
[32621, 32633, 32647, 32653, 32687, 32693, 32707, 32713, 32717, 32719],
[32749, 32771, 32779, 32783, 32789, 32797, 32801, 32803, 32831, 32833],
[32839, 32843, 32869, 32887, 32909, 32911, 32917, 32933, 32939, 32941],
[32957, 32969, 32971, 32983, 32987, 32993, 32999, 33013, 33023, 33029],
[33037, 33049, 33053, 33071, 33073, 33083, 33091, 33107, 33113, 33119],
[33149, 33151, 33161, 33179, 33181, 33191, 33199, 33203, 33211, 33223],
[33247, 33287, 33289, 33301, 33311, 33317, 33329, 33331, 33343, 33347],
[33349, 33353, 33359, 33377, 33391, 33403, 33409, 33413, 33427, 33457],
[33461, 33469, 33479, 33487, 33493, 33503, 33521, 33529, 33533, 33547],
[33563, 33569, 33577, 33581, 33587, 33589, 33599, 33601, 33613, 33617],
[33619, 33623, 33629, 33637, 33641, 33647, 33679, 33703, 33713, 33721],
[33739, 33749, 33751, 33757, 33767, 33769, 33773, 33791, 33797, 33809],
[33811, 33827, 33829, 33851, 33857, 33863, 33871, 33889, 33893, 33911],
[33923, 33931, 33937, 33941, 33961, 33967, 33997, 34019, 34031, 34033],
[34039, 34057, 34061, 34123, 34127, 34129, 34141, 34147, 34157, 34159],
[34171, 34183, 34211, 34213, 34217, 34231, 34253, 34259, 34261, 34267],
[34273, 34283, 34297, 34301, 34303, 34313, 34319, 34327, 34337, 34351],
[34361, 34367, 34369, 34381, 34403, 34421, 34429, 34439, 34457, 34469],
[34471, 34483, 34487, 34499, 34501, 34511, 34513, 34519, 34537, 34543],
[34549, 34583, 34589, 34591, 34603, 34607, 34613, 34631, 34649, 34651],
[34667, 34673, 34679, 34687, 34693, 34703, 34721, 34729, 34739, 34747],
[34757, 34759, 34763, 34781, 34807, 34819, 34841, 34843, 34847, 34849],
[34871, 34877, 34883, 34897, 34913, 34919, 34939, 34949, 34961, 34963],
[34981, 35023, 35027, 35051, 35053, 35059, 35069, 35081, 35083, 35089],
[35099, 35107, 35111, 35117, 35129, 35141, 35149, 35153, 35159, 35171],
[35201, 35221, 35227, 35251, 35257, 35267, 35279, 35281, 35291, 35311],
[35317, 35323, 35327, 35339, 35353, 35363, 35381, 35393, 35401, 35407],
[35419, 35423, 35437, 35447, 35449, 35461, 35491, 35507, 35509, 35521],
[35527, 35531, 35533, 35537, 35543, 35569, 35573, 35591, 35593, 35597],
[35603, 35617, 35671, 35677, 35729, 35731, 35747, 35753, 35759, 35771],
[35797, 35801, 35803, 35809, 35831, 35837, 35839, 35851, 35863, 35869],
[35879, 35897, 35899, 35911, 35923, 35933, 35951, 35963, 35969, 35977],
[35983, 35993, 35999, 36007, 36011, 36013, 36017, 36037, 36061, 36067],
[36073, 36083, 36097, 36107, 36109, 36131, 36137, 36151, 36161, 36187],
[36191, 36209, 36217, 36229, 36241, 36251, 36263, 36269, 36277, 36293],
[36299, 36307, 36313, 36319, 36341, 36343, 36353, 36373, 36383, 36389],
[36433, 36451, 36457, 36467, 36469, 36473, 36479, 36493, 36497, 36523],
[36527, 36529, 36541, 36551, 36559, 36563, 36571, 36583, 36587, 36599],
[36607, 36629, 36637, 36643, 36653, 36671, 36677, 36683, 36691, 36697],
[36709, 36713, 36721, 36739, 36749, 36761, 36767, 36779, 36781, 36787],
[36791, 36793, 36809, 36821, 36833, 36847, 36857, 36871, 36877, 36887],
[36899, 36901, 36913, 36919, 36923, 36929, 36931, 36943, 36947, 36973],
[36979, 36997, 37003, 37013, 37019, 37021, 37039, 37049, 37057, 37061],
[37087, 37097, 37117, 37123, 37139, 37159, 37171, 37181, 37189, 37199],
[37201, 37217, 37223, 37243, 37253, 37273, 37277, 37307, 37309, 37313],
[37321, 37337, 37339, 37357, 37361, 37363, 37369, 37379, 37397, 37409],
[37423, 37441, 37447, 37463, 37483, 37489, 37493, 37501, 37507, 37511],
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039]]}}]}
================================================================================ ================================================================================

121
modules/services/serviceAi/merge_2.txt Normal file
View file

@ -0,0 +1,121 @@
================================================================================
JSON MERGE OPERATION #2
================================================================================
Timestamp: 2026-01-05T08:30:55.472639
INPUT:
Accumulated length: 414 chars
New Fragment length: 245 chars
Accumulated: 19 lines (showing first 5 and last 5)
{
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
... (9 lines omitted) ...
"elements": [
{
"type": "heading",
"content": {
"text": "Introduction"}}]}]}]}
New Fragment: 14 lines (showing first 5 and last 5)
t_type": "paragraph",
"elements": [
{
"type": "paragraph",
"content": {
... (4 lines omitted) ...
}
]
}
]
}
Normalized Accumulated (414 chars)
(showing first 5 and last 5 of 19 lines)
{
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
... (9 lines omitted) ...
"elements": [
{
"type": "heading",
"content": {
"text": "Introduction"}}]}]}]}
Normalized New Fragment (245 chars)
(showing first 5 and last 5 of 14 lines)
t_type": "paragraph",
"elements": [
{
"type": "paragraph",
"content": {
... (4 lines omitted) ...
}
]
}
]
}
STEP: PHASE 1
Description: Finding overlap between JSON strings
⏳ In progress...
Overlap Detection (string):
Overlap length: 0
⚠️ No overlap detected - appending all
⚠️ NO OVERLAP FOUND - This indicates iterations should stop
Closing JSON and returning final result
Closed JSON (414 chars):
==============================================================================
{
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
},
"documents": [
{
"id": "doc1",
"title": "Document 1",
"sections": [
{
"id": "sec1",
"content_type": "heading",
"elements": [
{
"type": "heading",
"content": {
"text": "Introduction"}}]}]}]}
==============================================================================
================================================================================
MERGE RESULT: ✅ SUCCESS
================================================================================
Final result length: 414 chars
Final result (COMPLETE):
================================================================================
{
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
},
"documents": [
{
"id": "doc1",
"title": "Document 1",
"sections": [
{
"id": "sec1",
"content_type": "heading",
"elements": [
{
"type": "heading",
"content": {
"text": "Introduction"}}]}]}]}
================================================================================

14
modules/services/serviceAi/subAiCallLooping.py
View file

@ -324,6 +324,8 @@ class AiCallLooper:
# JSON is already closed by mergeJsonStringsWithOverlap when no overlap # JSON is already closed by mergeJsonStringsWithOverlap when no overlap
# Use the merged (closed) JSON string directly # Use the merged (closed) JSON string directly
result = mergedJsonString result = mergedJsonString
# CRITICAL: Update lastRawResponse with merged result for next iteration
lastRawResponse = mergedJsonString
# Try to parse it to get parsedJsonForUseCase # Try to parse it to get parsedJsonForUseCase
try: try:
extracted = extractJsonString(mergedJsonString) extracted = extractJsonString(mergedJsonString)
@ -333,6 +335,8 @@ class AiCallLooper:
normalized = self._normalizeJsonStructure(parsed, useCase) normalized = self._normalizeJsonStructure(parsed, useCase)
parsedJsonForUseCase = normalized parsedJsonForUseCase = normalized
result = json.dumps(normalized, indent=2, ensure_ascii=False) result = json.dumps(normalized, indent=2, ensure_ascii=False)
# CRITICAL: Update lastRawResponse with final result
lastRawResponse = result
else: else:
# Parsing failed - try to repair JSON # Parsing failed - try to repair JSON
from modules.shared.jsonUtils import repairBrokenJson from modules.shared.jsonUtils import repairBrokenJson
@ -346,6 +350,8 @@ class AiCallLooper:
normalized = self._normalizeJsonStructure(repairedJson, useCase) normalized = self._normalizeJsonStructure(repairedJson, useCase)
parsedJsonForUseCase = normalized parsedJsonForUseCase = normalized
result = json.dumps(normalized, indent=2, ensure_ascii=False) result = json.dumps(normalized, indent=2, ensure_ascii=False)
# CRITICAL: Update lastRawResponse with final result
lastRawResponse = result
logger.info(f"Iteration {iteration}: Successfully repaired JSON after no-overlap merge") logger.info(f"Iteration {iteration}: Successfully repaired JSON after no-overlap merge")
except Exception as e: except Exception as e:
# Last resort: try repair on the original merged string # Last resort: try repair on the original merged string
@ -379,6 +385,8 @@ class AiCallLooper:
normalized = self._normalizeJsonStructure(parsed, useCase) normalized = self._normalizeJsonStructure(parsed, useCase)
parsedJsonForUseCase = normalized parsedJsonForUseCase = normalized
result = json.dumps(normalized, indent=2, ensure_ascii=False) result = json.dumps(normalized, indent=2, ensure_ascii=False)
# CRITICAL: Update lastRawResponse with merged result
lastRawResponse = result
else: else:
# Parsing failed - try to extract partial data using Deep-Structure-Merging # Parsing failed - try to extract partial data using Deep-Structure-Merging
# This fallback works for all use cases: parse what we can from each part # This fallback works for all use cases: parse what we can from each part
@ -404,9 +412,13 @@ class AiCallLooper:
parsedJsonForUseCase = mergedJsonObj parsedJsonForUseCase = mergedJsonObj
result = json.dumps(mergedJsonObj, indent=2, ensure_ascii=False) result = json.dumps(mergedJsonObj, indent=2, ensure_ascii=False)
# CRITICAL: Update lastRawResponse with merged result
lastRawResponse = result
else: else:
# All parsing failed - use string merge result # All parsing failed - use string merge result
result = mergedJsonString result = mergedJsonString
# CRITICAL: Update lastRawResponse with merged result
lastRawResponse = mergedJsonString
except Exception as e: except Exception as e:
logger.warning(f"Failed data-based merge, falling back to string merging: {e}") logger.warning(f"Failed data-based merge, falling back to string merging: {e}")
# Fallback to string merging # Fallback to string merging
@ -424,6 +436,8 @@ class AiCallLooper:
hasOverlap = False hasOverlap = False
logger.info(f"Iteration {iteration}: No overlap found in final fallback merge - stopping iterations") logger.info(f"Iteration {iteration}: No overlap found in final fallback merge - stopping iterations")
result = mergedJsonString result = mergedJsonString
# CRITICAL: Update lastRawResponse with merged result
lastRawResponse = mergedJsonString
# If no overlap was found, mark as complete and use closed JSON # If no overlap was found, mark as complete and use closed JSON
if not hasOverlap: if not hasOverlap:

80
modules/services/serviceAi/subStructureFilling.py
View file

@ -2198,75 +2198,15 @@ Output requirements:
incompletePart = continuationContext.incomplete_part incompletePart = continuationContext.incomplete_part
lastRawJson = continuationContext.last_raw_json lastRawJson = continuationContext.last_raw_json
# Build overlap context: extract cut part and full part before (same level) for overlap # Generate both overlap context and hierarchy context using jsonContinuation
overlapContext = "" overlapContext = ""
if lastRawJson:
# Find break position in raw JSON
lastCompletePart = continuationContext.last_complete_part
breakPos = len(lastRawJson.rstrip())
if lastCompletePart:
from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
normalizedRaw = stripCodeFences(normalizeJsonText(lastRawJson)).strip()
normalizedComplete = stripCodeFences(normalizeJsonText(lastCompletePart)).strip()
# Find where normalizedComplete ends in normalizedRaw
pos = normalizedRaw.find(normalizedComplete)
if pos >= 0:
breakPos = pos + len(normalizedComplete)
else:
pos = lastRawJson.find(lastCompletePart)
if pos >= 0:
breakPos = pos + len(lastCompletePart)
elif incompletePart:
pos = lastRawJson.find(incompletePart)
if pos >= 0:
breakPos = pos
# Extract cut part and full part before (same level)
overlapContext = self._extractOverlapContext(lastRawJson, breakPos)
# Build unified context showing structure hierarchy with cut point
unifiedContext = "" unifiedContext = ""
if lastRawJson: if lastRawJson:
# Find break position in raw JSON # Get contexts directly from jsonContinuation
# Use last_complete_part length to find where complete part ends from modules.shared.jsonContinuation import getContexts
lastCompletePart = continuationContext.last_complete_part contexts = getContexts(lastRawJson)
if lastCompletePart: overlapContext = contexts.overlapContext
# Break position is where the complete part ends unifiedContext = contexts.hierarchyContext
# Normalize lastRawJson to match the normalized lastCompletePart
from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
normalizedRaw = stripCodeFences(normalizeJsonText(lastRawJson)).strip()
normalizedComplete = stripCodeFences(normalizeJsonText(lastCompletePart)).strip()
# Find where normalizedComplete ends in normalizedRaw
breakPos = normalizedRaw.find(normalizedComplete)
if breakPos >= 0:
breakPos = breakPos + len(normalizedComplete)
else:
# Fallback: use length of lastCompletePart in original string
breakPos = lastRawJson.find(lastCompletePart)
if breakPos >= 0:
breakPos = breakPos + len(lastCompletePart)
else:
# Last resort: use incompletePart position
if incompletePart:
breakPos = lastRawJson.find(incompletePart)
if breakPos == -1:
breakPos = len(lastRawJson.rstrip())
else:
breakPos = len(lastRawJson.rstrip())
elif incompletePart:
# If no complete part, find where incomplete part starts
breakPos = lastRawJson.find(incompletePart)
if breakPos == -1:
breakPos = len(lastRawJson.rstrip())
else:
breakPos = len(lastRawJson.rstrip())
# Build intelligent context showing hierarchy
from modules.shared.jsonUtils import buildIncompleteContext
unifiedContext = buildIncompleteContext(lastRawJson, breakPos)
elif incompletePart: elif incompletePart:
unifiedContext = incompletePart unifiedContext = incompletePart
else: else:
@ -2308,14 +2248,6 @@ CRITICAL:
- Complete the incomplete element and continue with remaining elements""" - Complete the incomplete element and continue with remaining elements"""
return continuationPrompt return continuationPrompt
def _extractOverlapContext(self, jsonContent: str, breakPosition: int) -> str:
"""
Extract overlap context: cut part and full part before (same level).
Delegates to shared function in jsonUtils for consistency.
"""
from modules.shared.jsonUtils import extractOverlapContext
return extractOverlapContext(jsonContent, breakPosition)
def _extractAndMergeMultipleJsonBlocks(self, responseText: str, contentType: str, sectionId: str) -> List[Dict[str, Any]]: def _extractAndMergeMultipleJsonBlocks(self, responseText: str, contentType: str, sectionId: str) -> List[Dict[str, Any]]:
""" """
Extract multiple JSON blocks from response and merge them appropriately. Extract multiple JSON blocks from response and merge them appropriately.

176
modules/services/serviceAi/subStructureGeneration.py
View file

@ -128,33 +128,15 @@ class StructureGenerator:
incompletePart = continuationContext.incomplete_part incompletePart = continuationContext.incomplete_part
lastRawJson = continuationContext.last_raw_json lastRawJson = continuationContext.last_raw_json
# Build overlap context: extract cut part and full part before (same level) for overlap # Generate both overlap context and hierarchy context using jsonContinuation
overlapContext = "" overlapContext = ""
if lastRawJson:
# Find break position
breakPos = len(lastRawJson.rstrip())
if incompletePart:
pos = lastRawJson.find(incompletePart)
if pos >= 0:
breakPos = pos
# Extract cut part and full part before (same level)
overlapContext = StructureGenerator._extractOverlapContext(lastRawJson, breakPos)
# Build unified context showing structure hierarchy with cut point
unifiedContext = "" unifiedContext = ""
if lastRawJson: if lastRawJson:
# Find break position in raw JSON # Get contexts directly from jsonContinuation
if incompletePart: from modules.shared.jsonContinuation import getContexts
breakPos = lastRawJson.find(incompletePart) contexts = getContexts(lastRawJson)
if breakPos == -1: overlapContext = contexts.overlapContext
breakPos = len(lastRawJson.rstrip()) unifiedContext = contexts.hierarchyContext
else:
breakPos = len(lastRawJson.rstrip())
# Build intelligent context showing hierarchy
from modules.shared.jsonUtils import buildIncompleteContext
unifiedContext = buildIncompleteContext(lastRawJson, breakPos)
elif incompletePart: elif incompletePart:
unifiedContext = incompletePart unifiedContext = incompletePart
else: else:
@ -195,144 +177,6 @@ CRITICAL:
- Start with overlap context (cut part and full part before at same level) then continue seamlessly - Start with overlap context (cut part and full part before at same level) then continue seamlessly
- Complete the incomplete element and continue with remaining elements""" - Complete the incomplete element and continue with remaining elements"""
return continuationPrompt return continuationPrompt
"""
Extract overlap context: cut part and full part before (same level).
Returns a string showing:
1. The last complete element at the same level before the cut point
2. The cut part (incomplete element at the cut point)
"""
if not jsonContent or breakPosition <= 0:
return jsonContent[-200:].strip() if jsonContent else ""
from modules.shared.jsonUtils import findStructureHierarchy, extractCutPiece
# Find structure hierarchy
hierarchy = findStructureHierarchy(jsonContent, breakPosition)
if not hierarchy:
# Fallback: show last 200 chars before break
start = max(0, breakPosition - 200)
return jsonContent[start:breakPosition + 100].strip()
# Get cut level (the array/object containing the cut piece)
cutLevel = hierarchy[-1]
cutLevelStart = cutLevel['start_pos']
cutLevelType = cutLevel['type']
# Extract cut piece (incomplete element)
cutPiece = extractCutPiece(jsonContent, breakPosition)
# Find the last complete element at the same level before the cut point
overlapParts = []
if cutLevelType == 'array':
# Find the last complete array element before breakPosition
i = breakPosition - 1
depth = 0
inString = False
escapeNext = False
elementStart = breakPosition
# Find the start of the incomplete element (or last complete element)
while i >= cutLevelStart:
char = jsonContent[i]
if escapeNext:
escapeNext = False
i -= 1
continue
if char == '\\':
escapeNext = True
i -= 1
continue
if char == '"':
inString = not inString
i -= 1
continue
if not inString:
if char == ']':
depth += 1
elif char == '[':
depth -= 1
if depth < 0:
elementStart = i + 1
break
elif char == ',' and depth == 0:
elementStart = i + 1
break
i -= 1
# Extract the last complete element (if exists) and the cut part
if elementStart < breakPosition:
contentBeforeBreak = jsonContent[max(cutLevelStart, elementStart - 500):breakPosition].strip()
# Find the last complete element by looking for balanced brackets/braces
lastCompleteEnd = breakPosition
braceCount = 0
bracketCount = 0
inString = False
escapeNext = False
# Go backwards from breakPosition to find where last complete element ends
for j in range(breakPosition - 1, max(cutLevelStart, breakPosition - 1000), -1):
char = jsonContent[j]
if escapeNext:
escapeNext = False
continue
if char == '\\':
escapeNext = True
continue
if char == '"':
inString = not inString
continue
if not inString:
if char == '}':
braceCount += 1
elif char == '{':
braceCount -= 1
if braceCount == 0 and bracketCount == 0:
lastCompleteEnd = j
break
elif char == ']':
bracketCount += 1
elif char == '[':
bracketCount -= 1
if bracketCount == 0 and braceCount == 0:
lastCompleteEnd = j + 1
break
elif char == ',' and braceCount == 0 and bracketCount == 0:
lastCompleteEnd = j + 1
break
# Extract last complete element and cut part
if lastCompleteEnd < breakPosition:
lastCompleteElement = jsonContent[max(cutLevelStart, lastCompleteEnd - 300):lastCompleteEnd].strip()
cutPart = jsonContent[lastCompleteEnd:breakPosition + len(cutPiece)].strip()
if lastCompleteElement:
overlapParts.append(f"Last complete element at same level:\n{lastCompleteElement}")
if cutPart:
overlapParts.append(f"Cut part (incomplete):\n{cutPart}")
else:
contextStart = max(cutLevelStart, breakPosition - 300)
overlapParts.append(jsonContent[contextStart:breakPosition + len(cutPiece)].strip())
else:
contextStart = max(cutLevelStart, breakPosition - 300)
overlapParts.append(jsonContent[contextStart:breakPosition + len(cutPiece)].strip())
else:
# For objects or other types, show context around break point
contextStart = max(cutLevelStart, breakPosition - 300)
overlapParts.append(jsonContent[contextStart:breakPosition + len(cutPiece)].strip())
return "\n\n".join(overlapParts) if overlapParts else jsonContent[max(0, breakPosition - 200):breakPosition + 100].strip()
# Call AI with looping support # Call AI with looping support
# NOTE: Do NOT pass contentParts here - we only need metadata for structure generation # NOTE: Do NOT pass contentParts here - we only need metadata for structure generation
@ -457,14 +301,6 @@ CRITICAL:
raise raise
@staticmethod @staticmethod
def _extractOverlapContext(jsonContent: str, breakPosition: int) -> str:
"""
Extract overlap context: cut part and full part before (same level).
Delegates to shared function in jsonUtils for consistency.
"""
from modules.shared.jsonUtils import extractOverlapContext
return extractOverlapContext(jsonContent, breakPosition)
def _buildChapterStructurePrompt( def _buildChapterStructurePrompt(
self, self,
userPrompt: str, userPrompt: str,

594
modules/services/serviceAi/test_json_merger.py
View file

@ -1,594 +0,0 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Test cases for JSON merger with different use cases and random cuts.
Tests the robustness of the JSON merger by:
1. Creating test JSON for different use cases
2. Cutting it randomly at various points
3. Running the merger for each piece
4. Checking completeness against original
"""
import json
import random
import logging
import sys
import os
from typing import Dict, Any, List, Tuple
# Add project root to Python path
# Find project root by looking for gateway/modules structure
currentFile = os.path.abspath(__file__)
currentDir = os.path.dirname(currentFile)
# Navigate up from: gateway/modules/services/serviceAi/test_json_merger.py
# To project root: D:\Athi\Local\Web\poweron
# Try different levels up
candidates = [
os.path.abspath(os.path.join(currentDir, '../../../../')), # From gateway/modules/services/serviceAi
os.path.abspath(os.path.join(currentDir, '../../..')), # Alternative
os.path.abspath(os.path.join(currentDir, '../..')), # Another alternative
]
projectRoot = None
for candidate in candidates:
gatewayModulesPath = os.path.join(candidate, 'gateway', 'modules')
if os.path.exists(gatewayModulesPath):
projectRoot = candidate
break
# If still not found, try to find by looking for gateway directory
if projectRoot is None:
searchDir = currentDir
for _ in range(10): # Max 10 levels up
gatewayPath = os.path.join(searchDir, 'gateway')
if os.path.exists(gatewayPath) and os.path.exists(os.path.join(gatewayPath, 'modules')):
projectRoot = searchDir
break
parent = os.path.dirname(searchDir)
if parent == searchDir: # Reached root
break
searchDir = parent
if projectRoot is None:
raise RuntimeError(f"Could not find project root. Current file: {currentFile}")
# Add gateway directory to Python path (not project root)
gatewayPath = os.path.join(projectRoot, 'gateway')
if gatewayPath not in sys.path:
sys.path.insert(0, gatewayPath)
# Verify the path is correct
modulesPath = os.path.join(projectRoot, 'gateway', 'modules')
if not os.path.exists(modulesPath):
raise RuntimeError(f"Project root verification failed. Expected gateway/modules at: {modulesPath}")
try:
from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler
from modules.services.serviceAi.subJsonMerger import JsonMergeLogger
from modules.shared.jsonUtils import (
normalizeJsonText, stripCodeFences, closeJsonStructures, tryParseJson,
extractJsonStructureContext
)
except ImportError as e:
# Try to help debug
print(f"Import error: {e}")
print(f"Project root: {projectRoot}")
print(f"Gateway path: {gatewayPath}")
print(f"Python path (first 3): {sys.path[:3]}")
print(f"Looking for modules at: {modulesPath}")
print(f"Exists: {os.path.exists(modulesPath)}")
if os.path.exists(modulesPath):
print(f"Contents: {os.listdir(modulesPath)[:5]}")
raise
logger = logging.getLogger(__name__)
def createTestJsonForUseCase(useCaseId: str, size: int = 100) -> Dict[str, Any]:
"""
Create test JSON for a specific use case.
Args:
useCaseId: Use case ID (section_content, chapter_structure, etc.)
size: Size of test data (number of elements/rows/items)
Returns:
Test JSON dictionary
"""
if useCaseId == "section_content":
# Create table with rows
elements = [{
"type": "table",
"content": {
"headers": ["Year", "Value"],
"rows": [[str(1947 + i), str(10000 + i * 100)] for i in range(size)]
}
}]
return {"elements": elements}
elif useCaseId == "chapter_structure":
chapters = [{
"id": f"chapter_{i}",
"title": f"Chapter {i}",
"level": 1
} for i in range(size)]
return {"documents": [{"chapters": chapters}]}
elif useCaseId == "code_structure":
files = [{
"id": f"file_{i}",
"filename": f"file_{i}.py",
"fileType": "python",
"functions": [f"function_{i}_{j}" for j in range(5)]
} for i in range(size)]
return {"files": files}
elif useCaseId == "code_content":
files = [{
"id": f"file_{i}",
"content": f"# File {i}\ndef function_{i}():\n pass\n" * 10,
"functions": [{"name": f"function_{i}_{j}", "line": j * 3} for j in range(5)]
} for i in range(size)]
return {"files": files}
else:
raise ValueError(f"Unknown use case: {useCaseId}")
def cutJsonRandomly(jsonString: str, numCuts: int = 5, overlapSize: int = 100) -> List[str]:
"""
Cut JSON string RANDOMLY at different points WITH OVERLAP between fragments.
Each fragment overlaps with the previous one to help merging.
Args:
jsonString: JSON string to cut
numCuts: Number of cuts to make
overlapSize: Size of overlap between fragments (in characters)
Returns:
List of JSON fragments with overlap
"""
fragments = []
currentPos = 0
totalLength = len(jsonString)
if totalLength == 0:
return []
# First fragment: from start to first cut point
if numCuts > 0:
# First cut point (between 20% and 40% of total)
firstCutPoint = random.randint(int(totalLength * 0.2), int(totalLength * 0.4))
fragment = jsonString[:firstCutPoint]
fragments.append(fragment)
currentPos = firstCutPoint
else:
# No cuts - return whole string
return [jsonString]
# Subsequent fragments: each starts with overlap from previous, then continues
for i in range(numCuts - 1):
if currentPos >= totalLength:
break
# Calculate overlap start (go back overlapSize from current position)
overlapStart = max(0, currentPos - overlapSize)
# Calculate next cut point (between 20% and 40% of remaining)
remaining = totalLength - currentPos
if remaining < overlapSize * 2:
# Not enough remaining - add rest as last fragment
fragment = jsonString[overlapStart:]
fragments.append(fragment)
break
# Next cut point from current position
nextCutPoint = currentPos + random.randint(int(remaining * 0.2), int(remaining * 0.4))
nextCutPoint = min(nextCutPoint, totalLength)
# Fragment: from overlap start to next cut point
fragment = jsonString[overlapStart:nextCutPoint]
fragments.append(fragment)
currentPos = nextCutPoint
# Add remaining as last fragment (with overlap)
if currentPos < totalLength:
overlapStart = max(0, currentPos - overlapSize)
fragment = jsonString[overlapStart:]
fragments.append(fragment)
return fragments
def testMergerWithFragments(
originalJson: Dict[str, Any],
fragments: List[str],
useCaseId: str
) -> Tuple[bool, Dict[str, Any], str]:
"""
Test merger by merging fragments sequentially.
Args:
originalJson: Original complete JSON
fragments: List of JSON fragments to merge
useCaseId: Use case ID
Returns:
Tuple of (success, merged_json, error_message)
"""
if not fragments:
return False, {}, "No fragments provided"
# Log structure context for each fragment (especially incomplete ones)
print(f"\n{'='*60}")
print(f"FRAGMENT ANALYSIS (use case: {useCaseId})")
print(f"{'='*60}")
for fragIdx, fragment in enumerate(fragments):
print(f"\nFragment {fragIdx + 1}/{len(fragments)}:")
print(f" Length: {len(fragment)} chars")
# Extract structure context for this fragment
try:
structureContext = extractJsonStructureContext(fragment, useCaseId)
templateStructure = structureContext.get("template_structure", "")
lastCompletePart = structureContext.get("last_complete_part", "")
incompletePart = structureContext.get("incomplete_part", "")
structureContextJson = structureContext.get("structure_context", "")
# Check if fragment is incomplete
normalized = stripCodeFences(normalizeJsonText(fragment)).strip()
parsed, parseErr, _ = tryParseJson(normalized)
isIncomplete = parseErr is not None or (parsed is None)
if isIncomplete:
print(f" Status: INCOMPLETE (cut off)")
print(f" Template Structure:")
if templateStructure:
# Show first few lines of template
templateLines = templateStructure.split('\n')
templateLinesToShow = templateLines[:5]
for line in templateLinesToShow:
print(f" {line}")
if len(templateLines) > 5:
remainingLines = len(templateLines) - 5
print(f" ... ({remainingLines} more lines)")
else:
print(f" (not available)")
print(f" Structure Context:")
if structureContextJson:
# Show structure context
contextLines = structureContextJson.split('\n')
contextLinesToShow = contextLines[:5]
for line in contextLinesToShow:
print(f" {line}")
if len(contextLines) > 5:
remainingContextLines = len(contextLines) - 5
print(f" ... ({remainingContextLines} more lines)")
else:
print(f" (not available)")
print(f" Last Complete Part:")
if lastCompletePart:
# Show last complete part (truncated if too long)
if len(lastCompletePart) > 200:
print(f" {lastCompletePart[:200]}... ({len(lastCompletePart)} chars total)")
else:
print(f" {lastCompletePart}")
else:
print(f" (not available)")
print(f" Incomplete Part:")
if incompletePart:
# Show incomplete part (truncated if too long)
if len(incompletePart) > 200:
print(f" {incompletePart[:200]}... ({len(incompletePart)} chars total)")
else:
print(f" {incompletePart}")
else:
print(f" (not available)")
else:
print(f" Status: COMPLETE")
if structureContextJson:
print(f" Structure Context:")
contextLines = structureContextJson.split('\n')
contextLinesToShow = contextLines[:3]
for line in contextLinesToShow:
print(f" {line}")
if len(contextLines) > 3:
remainingContextLines = len(contextLines) - 3
print(f" ... ({remainingContextLines} more lines)")
except Exception as e:
print(f" Error extracting structure context: {e}")
print(f"\n{'='*60}\n")
# Start with first fragment
accumulated = fragments[0]
# Merge each subsequent fragment
for i, fragment in enumerate(fragments[1:], 1):
try:
accumulated, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap(
accumulated, fragment
)
# Log if no overlap was found (iterations would stop in real scenario)
if not hasOverlap:
print(f" ⚠️ Fragment {i}: No overlap found - iterations would stop here")
# Check if result is empty (should never happen)
if not accumulated or accumulated.strip() in ['{"elements": []}', '{}', '']:
return False, {}, f"Merge {i} returned empty JSON"
except Exception as e:
return False, {}, f"Merge {i} failed with error: {str(e)}"
# Parse merged result
try:
# Normalize and try to parse
normalized = stripCodeFences(normalizeJsonText(accumulated)).strip()
# Try to parse directly
parsed, parseErr, _ = tryParseJson(normalized)
if parseErr is not None:
# Try closing structures if incomplete
try:
closed = closeJsonStructures(normalized)
parsed, parseErr2, _ = tryParseJson(closed)
if parseErr2 is not None:
# Try to extract valid JSON prefix
# JsonResponseHandler is already imported at module level
validPrefix = JsonResponseHandler._extractValidJsonPrefix(normalized)
if validPrefix:
parsed, parseErr3, _ = tryParseJson(validPrefix)
if parseErr3 is not None:
return False, {}, f"Final parse error: {str(parseErr3)}"
else:
return False, {}, f"Final parse error: {str(parseErr2)}"
except Exception as parseErr:
return False, {}, f"Final parse error: {str(parseErr)}"
if not parsed:
return False, {}, "Final parse returned None"
# CRITICAL: Ensure parsed is a dict, not a list
# If it's a list, wrap it in the expected structure based on use case
if isinstance(parsed, list):
# Try to normalize list to expected structure
if useCaseId == "section_content":
# List of elements - wrap in elements structure
parsed = {"elements": parsed}
elif useCaseId == "chapter_structure":
# List of chapters - wrap in documents structure
parsed = {"documents": [{"chapters": parsed}]}
elif useCaseId == "code_structure":
# List of files - wrap in files structure
parsed = {"files": parsed}
elif useCaseId == "code_content":
# List of files - wrap in files structure
parsed = {"files": parsed}
else:
# Unknown use case - try to wrap as elements
parsed = {"elements": parsed}
# Ensure it's a dict now
if not isinstance(parsed, dict):
return False, {}, f"Final parse returned unexpected type: {type(parsed).__name__}"
return True, parsed, ""
except Exception as e:
return False, {}, f"Final parse failed: {str(e)}"
def compareJsonCompleteness(
original: Dict[str, Any],
merged: Dict[str, Any],
useCaseId: str
) -> Tuple[bool, str]:
"""
Compare merged JSON with original to check completeness.
Args:
original: Original JSON
merged: Merged JSON (must be a dict)
useCaseId: Use case ID
Returns:
Tuple of (is_complete, message)
"""
# CRITICAL: Ensure merged is a dict
if not isinstance(merged, dict):
return False, f"Merged JSON is not a dict, got {type(merged).__name__}"
if useCaseId == "section_content":
origElements = original.get("elements", [])
mergedElements = merged.get("elements", [])
if not isinstance(origElements, list):
return False, f"Original elements is not a list: {type(origElements).__name__}"
if not isinstance(mergedElements, list):
return False, f"Merged elements is not a list: {type(mergedElements).__name__}"
if len(mergedElements) < len(origElements):
return False, f"Missing elements: {len(origElements)} expected, {len(mergedElements)} found"
# Check table rows
if origElements and mergedElements:
origTable = origElements[0] if isinstance(origElements[0], dict) else {}
mergedTable = mergedElements[0] if isinstance(mergedElements[0], dict) else {}
if not origTable or not mergedTable:
return False, f"Table structure missing: origTable={bool(origTable)}, mergedTable={bool(mergedTable)}"
origRows = origTable.get("content", {}).get("rows", []) if isinstance(origTable.get("content"), dict) else origTable.get("rows", [])
mergedRows = mergedTable.get("content", {}).get("rows", []) if isinstance(mergedTable.get("content"), dict) else mergedTable.get("rows", [])
if not isinstance(origRows, list):
return False, f"Original rows is not a list: {type(origRows).__name__}"
if not isinstance(mergedRows, list):
return False, f"Merged rows is not a list: {type(mergedRows).__name__}"
if len(mergedRows) < len(origRows):
return False, f"Missing rows: {len(origRows)} expected, {len(mergedRows)} found"
return True, "Complete"
elif useCaseId == "chapter_structure":
origChapters = original.get("documents", [{}])[0].get("chapters", [])
mergedChapters = merged.get("documents", [{}])[0].get("chapters", [])
if len(mergedChapters) < len(origChapters):
return False, f"Missing chapters: {len(origChapters)} expected, {len(mergedChapters)} found"
return True, "Complete"
elif useCaseId == "code_structure":
origFiles = original.get("files", [])
mergedFiles = merged.get("files", [])
if len(mergedFiles) < len(origFiles):
return False, f"Missing files: {len(origFiles)} expected, {len(mergedFiles)} found"
return True, "Complete"
elif useCaseId == "code_content":
origFiles = original.get("files", [])
mergedFiles = merged.get("files", [])
if len(mergedFiles) < len(origFiles):
return False, f"Missing files: {len(origFiles)} expected, {len(mergedFiles)} found"
return True, "Complete"
else:
return False, f"Unknown use case: {useCaseId}"
def runTestForUseCase(useCaseId: str, size: int = 50, numTests: int = 10) -> Dict[str, Any]:
"""
Run multiple tests for a use case with random cuts.
Args:
useCaseId: Use case ID
size: Size of test data
numTests: Number of test runs
Returns:
Test results dictionary
"""
results = {
"useCaseId": useCaseId,
"size": size,
"numTests": numTests,
"passed": 0,
"failed": 0,
"errors": []
}
for testNum in range(numTests):
try:
# Create test JSON
originalJson = createTestJsonForUseCase(useCaseId, size)
originalString = json.dumps(originalJson, indent=2, ensure_ascii=False)
# Cut randomly
fragments = cutJsonRandomly(originalString, numCuts=random.randint(3, 7))
# Test merger
success, mergedJson, errorMsg = testMergerWithFragments(
originalJson, fragments, useCaseId
)
if not success:
results["failed"] += 1
results["errors"].append(f"Test {testNum + 1}: {errorMsg}")
continue
# Check completeness
isComplete, completenessMsg = compareJsonCompleteness(
originalJson, mergedJson, useCaseId
)
if isComplete:
results["passed"] += 1
else:
results["failed"] += 1
results["errors"].append(f"Test {testNum + 1}: {completenessMsg}")
except Exception as e:
results["failed"] += 1
results["errors"].append(f"Test {testNum + 1}: Exception - {str(e)}")
return results
def runAllTests():
"""Run tests for all use cases."""
useCases = [
"section_content",
"chapter_structure",
"code_structure",
"code_content"
]
allResults = []
for useCaseId in useCases:
print(f"\n{'='*60}")
print(f"Testing use case: {useCaseId}")
print(f"{'='*60}")
# Initialize log file for this use case
# Initialize log file (overwrite on each test run)
logFileName = f"json_merger_{useCaseId}.txt"
JsonMergeLogger.initializeLogFile(logFileName)
print(f"Log file: {logFileName}")
results = runTestForUseCase(useCaseId, size=50, numTests=10)
allResults.append(results)
print(f"Passed: {results['passed']}/{results['numTests']}")
print(f"Failed: {results['failed']}/{results['numTests']}")
if results["errors"]:
print("\nErrors:")
for error in results["errors"][:5]: # Show first 5 errors
print(f" - {error}")
# Summary
print(f"\n{'='*60}")
print("SUMMARY")
print(f"{'='*60}")
totalPassed = sum(r["passed"] for r in allResults)
totalFailed = sum(r["failed"] for r in allResults)
totalTests = sum(r["numTests"] for r in allResults)
print(f"Total tests: {totalTests}")
print(f"Passed: {totalPassed}")
print(f"Failed: {totalFailed}")
print(f"Success rate: {totalPassed / totalTests * 100:.1f}%")
return allResults
if __name__ == "__main__":
# Set up logging - use WARNING level to reduce noise from jsonUtils
logging.basicConfig(level=logging.WARNING)
# Run tests
results = runAllTests()
# Save results to file (in project root)
resultsFile = os.path.join(projectRoot, "test_json_merger_results.json")
with open(resultsFile, "w", encoding="utf-8") as f:
json.dump(results, f, indent=2, ensure_ascii=False)
print(f"\nResults saved to {resultsFile}")

60
modules/services/serviceGeneration/paths/codePath.py
View file

@ -26,14 +26,6 @@ class CodeGenerationPath:
def __init__(self, services): def __init__(self, services):
self.services = services self.services = services
@staticmethod
def _extractOverlapContext(jsonContent: str, breakPosition: int) -> str:
"""
Extract overlap context: cut part and full part before (same level).
Delegates to shared function in jsonUtils for consistency.
"""
from modules.shared.jsonUtils import extractOverlapContext
return extractOverlapContext(jsonContent, breakPosition)
async def generateCode( async def generateCode(
self, self,
@ -346,25 +338,15 @@ Return ONLY valid JSON matching the request above.
incompletePart = continuationContext.incomplete_part incompletePart = continuationContext.incomplete_part
lastRawJson = continuationContext.last_raw_json lastRawJson = continuationContext.last_raw_json
# Build overlap context: extract last ~100 characters from the response for overlap # Generate both overlap context and hierarchy context using jsonContinuation
overlapContext = "" overlapContext = ""
if lastRawJson:
overlapContext = lastRawJson[-100:].strip()
# Build unified context showing structure hierarchy with cut point
unifiedContext = "" unifiedContext = ""
if lastRawJson: if lastRawJson:
# Find break position in raw JSON # Get contexts directly from jsonContinuation
if incompletePart: from modules.shared.jsonContinuation import getContexts
breakPos = lastRawJson.find(incompletePart) contexts = getContexts(lastRawJson)
if breakPos == -1: overlapContext = contexts.overlapContext
breakPos = len(lastRawJson.rstrip()) unifiedContext = contexts.hierarchyContext
else:
breakPos = len(lastRawJson.rstrip())
# Build intelligent context showing hierarchy
from modules.shared.jsonUtils import buildIncompleteContext
unifiedContext = buildIncompleteContext(lastRawJson, breakPos)
elif incompletePart: elif incompletePart:
unifiedContext = incompletePart unifiedContext = incompletePart
else: else:
@ -808,33 +790,15 @@ Return ONLY valid JSON in this format:
incompletePart = continuationContext.incomplete_part incompletePart = continuationContext.incomplete_part
lastRawJson = continuationContext.last_raw_json lastRawJson = continuationContext.last_raw_json
# Build overlap context: extract cut part and full part before (same level) for overlap # Generate both overlap context and hierarchy context using jsonContinuation
overlapContext = "" overlapContext = ""
if lastRawJson:
# Find break position
breakPos = len(lastRawJson.rstrip())
if incompletePart:
pos = lastRawJson.find(incompletePart)
if pos >= 0:
breakPos = pos
# Extract cut part and full part before (same level)
overlapContext = CodeGenerationPath._extractOverlapContext(lastRawJson, breakPos)
# Build unified context showing structure hierarchy with cut point
unifiedContext = "" unifiedContext = ""
if lastRawJson: if lastRawJson:
# Find break position in raw JSON # Get contexts directly from jsonContinuation
if incompletePart: from modules.shared.jsonContinuation import getContexts
breakPos = lastRawJson.find(incompletePart) contexts = getContexts(lastRawJson)
if breakPos == -1: overlapContext = contexts.overlapContext
breakPos = len(lastRawJson.rstrip()) unifiedContext = contexts.hierarchyContext
else:
breakPos = len(lastRawJson.rstrip())
# Build intelligent context showing hierarchy
from modules.shared.jsonUtils import buildIncompleteContext
unifiedContext = buildIncompleteContext(lastRawJson, breakPos)
elif incompletePart: elif incompletePart:
unifiedContext = incompletePart unifiedContext = incompletePart
else: else:

22
modules/services/serviceGeneration/subPromptBuilderGeneration.py
View file

@ -64,25 +64,27 @@ async def buildGenerationPrompt(
) )
if hasContinuation: if hasContinuation:
# CONTINUATION PROMPT - use new summary format from buildContinuationContext # CONTINUATION PROMPT - use centralized jsonContinuation system
delivered_summary = continuationContext.get("delivered_summary", "") delivered_summary = continuationContext.get("delivered_summary", "")
element_before_cutoff = continuationContext.get("element_before_cutoff")
cut_off_element = continuationContext.get("cut_off_element") # Use centralized system: overlap_context and hierarchy_context from jsonContinuation.getContexts()
overlap_context = continuationContext.get("overlap_context")
hierarchy_context = continuationContext.get("hierarchy_context")
# Build continuation text with delivered summary and cut-off information # Build continuation text with delivered summary and cut-off information
# CRITICAL: Always include cut-off information if available (per loop_plan.md) # CRITICAL: Always include cut-off information if available (per loop_plan.md)
continuationText = f"{delivered_summary}\n\n" continuationText = f"{delivered_summary}\n\n"
continuationText += "⚠️ CONTINUATION: Response was cut off. Generate ONLY the remaining content that comes AFTER the reference elements below.\n\n" continuationText += "⚠️ CONTINUATION: Response was cut off. Generate ONLY the remaining content that comes AFTER the reference elements below.\n\n"
# Add cut-off point information (per loop_plan.md: always add if available) # Add cut-off point information using centralized jsonContinuation contexts
# These are shown ONLY as REFERENCE to know where generation stopped # These are shown ONLY as REFERENCE to know where generation stopped
if element_before_cutoff: if hierarchy_context:
continuationText += "# REFERENCE: Last complete element (already delivered - DO NOT repeat):\n" continuationText += "# REFERENCE: Structure context (already delivered - DO NOT repeat):\n"
continuationText += f"{element_before_cutoff}\n\n" continuationText += f"{hierarchy_context}\n\n"
if cut_off_element: if overlap_context:
continuationText += "# REFERENCE: Incomplete element (cut off here - DO NOT repeat):\n" continuationText += "# REFERENCE: Overlap context - incomplete element at cut point (DO NOT repeat):\n"
continuationText += f"{cut_off_element}\n\n" continuationText += f"{overlap_context}\n\n"
continuationText += "⚠️ CRITICAL: The elements above are REFERENCE ONLY. They are already delivered.\n" continuationText += "⚠️ CRITICAL: The elements above are REFERENCE ONLY. They are already delivered.\n"
continuationText += "Generate ONLY what comes AFTER these elements. DO NOT regenerate the entire JSON structure.\n" continuationText += "Generate ONLY what comes AFTER these elements. DO NOT regenerate the entire JSON structure.\n"

164
modules/shared/jsonContinuation.md Normal file
View file

@ -0,0 +1,164 @@
# JSON Continuation Context Module
Ein Python-Modul zur Generierung von Kontextinformationen für abgeschnittene JSON-Strings, um AI-Modellen die Fortsetzung zu ermöglichen.
## Problem
Wenn eine AI-Antwort als JSON abgeschnitten wird (z.B. Token-Limit erreicht), muss die nächste Iteration wissen:
- **Wo** der JSON abgeschnitten wurde
- **Was** bereits generiert wurde
- **Was** als nächstes geliefert werden soll
## Lösung: Drei Kontexte
### 1. Overlap Context
- Zeigt das **innerste Objekt/Array-Element**, das den Cut-Punkt enthält
- Wird verwendet, um den abgeschnittenen Teil mit dem neuen Teil zu **mergen**
- Exakt so wie im Original-String (für String-Matching beim Merge)
### 2. Hierarchy Context
- Zeigt die **hierarchische Struktur** vom Root bis zum Cut-Punkt
- Mit **Budget-Logik**: Näher am Cut = vollständige Werte, weiter weg = `"..."` Platzhalter
- Gibt der AI den Kontext der gesamten JSON-Struktur
### 3. Complete Part (NEU)
- Der **vollständige, valide JSON** bis zum Cut-Punkt
- Alle offenen Strukturen werden geschlossen (`}`, `]`, `"`)
- Unvollständige Keys werden entfernt
- Kann direkt als valides JSON geparst werden
## Installation
```bash
# Keine externen Abhängigkeiten erforderlich
cp json_continuation.py /your/project/
```
## Modulkonstanten
```python
# Diese Konstanten können vor dem Import angepasst werden
BUDGET_LIMIT: int = 500 # Zeichen-Budget für Datenwerte
OVERLAP_MAX_CHARS: int = 1000 # Max Zeichen für Overlap Context
```
## Verwendung
### Grundlegende Verwendung
```python
from json_continuation import extract_continuation_contexts
truncated_json = '''{"customers": [
{"id": 1, "name": "John"},
{"id": 2, "name": "Jane", "email": "jane@exa'''
overlap, hierarchy, complete = extract_continuation_contexts(truncated_json)
print("Overlap Context:")
print(overlap)
# {"id": 2, "name": "Jane", "email": "jane@exa
print("Hierarchy Context:")
print(hierarchy)
# {"customers": [...structure with budget logic...]
print("Complete Part (valid JSON):")
print(complete)
# {"customers": [{"id": 1, "name": "John"}, {"id": 2, "name": "Jane", "email": "jane@exa"}]}
import json
parsed = json.loads(complete) # ✓ Funktioniert!
```
### Mit Dictionary-Interface
```python
from json_continuation import get_contexts
contexts = get_contexts(truncated_json)
print(contexts['overlap'])
print(contexts['hierarchy'])
print(contexts['complete_part'])
```
### Konstanten anpassen
```python
import json_continuation
# Budget anpassen bevor Funktionen aufgerufen werden
json_continuation.BUDGET_LIMIT = 200
json_continuation.OVERLAP_MAX_CHARS = 500
overlap, hierarchy, complete = json_continuation.extract_continuation_contexts(truncated_json)
```
## Rückgabewerte
| Rückgabe | Typ | Beschreibung |
|----------|-----|--------------|
| `overlap` | str | Innerstes Element mit Cut-Punkt (für Merge) |
| `hierarchy` | str | Volle Struktur mit Budget-Logik |
| `complete_part` | str | Valides JSON mit geschlossenen Strukturen |
## Beispiele
### Verschachtelte Objekte
```python
json_str = '{"user": {"profile": {"bio": "Hello Wor'
overlap, hierarchy, complete = extract_continuation_contexts(json_str)
# Overlap: {"bio": "Hello Wor
# Hierarchy: {"user": {"profile": {"bio": "Hello Wor
# Complete: {"user": {"profile": {"bio": "Hello Wor"}}} ← Valides JSON!
```
### Array von Objekten mit unvollständigem Key
```python
json_str = '''{
"items": [
{"id": 1, "name": "First"},
{"id": 2, "name": "Second"},
{"id": 3, "name": "Third", "add'''
overlap, hierarchy, complete = extract_continuation_contexts(json_str)
# Complete entfernt den unvollständigen Key "add":
# {"items": [{"id": 1, ...}, {"id": 2, ...}, {"id": 3, "name": "Third"}]}
```
## Budget-Logik
Die Budget-Logik funktioniert wie folgt:
1. **Sammeln**: Alle String-Werte werden mit ihrer Position gesammelt
2. **Sortieren**: Nach Entfernung zum Cut-Punkt (näher = höhere Priorität)
3. **Zuweisen**: Budget wird von hinten nach vorne aufgebraucht
4. **Ersetzen**: Werte außerhalb des Budgets werden durch `"..."` ersetzt
## Tests ausführen
```bash
python -m unittest test_json_continuation -v
```
## API Referenz
### `extract_continuation_contexts(truncated_json: str) -> Tuple[str, str, str]`
Hauptfunktion. Gibt `(overlap, hierarchy, complete_part)` zurück.
### `get_contexts(truncated_json: str) -> dict`
Convenience-Funktion. Gibt Dictionary mit Keys `'overlap'`, `'hierarchy'`, `'complete_part'` zurück.
### Modulkonstanten
- `BUDGET_LIMIT`: int (default: 500) - Zeichen-Budget für Hierarchy-Context
- `OVERLAP_MAX_CHARS`: int (default: 1000) - Max Zeichen für Overlap-Context

1232
modules/shared/jsonContinuation.py Normal file
View file

File diff suppressed because it is too large Load diff

2127
modules/shared/jsonUtils.py
View file

File diff suppressed because it is too large Load diff

694
tests/functional/test12_json_split_merge.py Normal file
View file

@ -0,0 +1,694 @@
#!/usr/bin/env python3
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
JSON Split and Merge Test 12 - Tests JSON splitting and merging using workflow tools
Tests random splitting of JSON files into 3 parts and merging them back using ModularJsonMerger.
"""
import asyncio
import json
import sys
import os
import time
import random
from typing import Dict, Any, List, Optional, Tuple
# Add the gateway to path (go up 2 levels from tests/functional/)
_gateway_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
if _gateway_path not in sys.path:
sys.path.insert(0, _gateway_path)
# Import JSON merger from workflow tools
from modules.services.serviceAi.subJsonMerger import ModularJsonMerger, JsonMergeLogger
from modules.shared.jsonContinuation import getContexts
class JsonSplitMergeTester12:
def __init__(self):
self.testResults = {}
self.testJsonFiles = []
self.logBuffer = []
self.logFile = None
def createTestJsonFiles(self) -> List[Dict[str, Any]]:
"""Create various test JSON files with different structures."""
testFiles = [
{
"name": "config.json",
"data": {
"application": "Customer Manager",
"version": "1.0.0",
"database": {
"host": "localhost",
"port": 5432,
"name": "customers_db"
},
"api": {
"baseUrl": "https://api.example.com",
"timeout": 30
}
}
},
{
"name": "customers.json",
"data": {
"customers": [
{"id": 1, "name": "John Doe", "email": "john@example.com", "phone": "+1234567890", "address": "123 Main St"},
{"id": 2, "name": "Jane Smith", "email": "jane@example.com", "phone": "+0987654321", "address": "456 Oak Ave"},
{"id": 3, "name": "Bob Johnson", "email": "bob@example.com", "phone": "+1122334455", "address": "789 Pine Rd"},
{"id": 4, "name": "Alice Williams", "email": "alice@example.com", "phone": "+5566778899", "address": "321 Elm St"},
{"id": 5, "name": "Charlie Brown", "email": "charlie@example.com", "phone": "+9988776655", "address": "654 Maple Dr"}
]
}
},
{
"name": "settings.json",
"data": {
"theme": {
"darkMode": True,
"fontSize": 14,
"language": "en"
},
"notifications": {
"email": True,
"sms": False,
"push": True
},
"features": {
"enableAnalytics": True,
"enableReports": False
}
}
},
{
"name": "products.json",
"data": {
"products": [
{"id": "P001", "name": "Product A", "price": 29.99, "category": "Electronics", "inStock": True},
{"id": "P002", "name": "Product B", "price": 49.99, "category": "Clothing", "inStock": True},
{"id": "P003", "name": "Product C", "price": 19.99, "category": "Books", "inStock": False},
{"id": "P004", "name": "Product D", "price": 99.99, "category": "Electronics", "inStock": True},
{"id": "P005", "name": "Product E", "price": 14.99, "category": "Books", "inStock": True},
{"id": "P006", "name": "Product F", "price": 79.99, "category": "Clothing", "inStock": True}
]
}
},
{
"name": "document_structure.json",
"data": {
"metadata": {
"title": "Test Document",
"author": "Test Author",
"date": "2025-01-05"
},
"documents": [
{
"id": "doc1",
"title": "Document 1",
"sections": [
{
"id": "sec1",
"content_type": "heading",
"elements": [
{"type": "heading", "content": {"text": "Introduction", "level": 1}}
]
},
{
"id": "sec2",
"content_type": "paragraph",
"elements": [
{"type": "paragraph", "content": {"text": "This is a test paragraph."}}
]
}
]
}
]
}
},
{
"name": "large_customers.json",
"data": self._createLargeCustomersData()
},
{
"name": "large_products.json",
"data": self._createLargeProductsData()
},
{
"name": "large_documents.json",
"data": self._createLargeDocumentsData()
}
]
return testFiles
def _createLargeCustomersData(self) -> Dict[str, Any]:
"""Create a large customers dataset for budget testing."""
customers = []
# Create 100 customers with long descriptions
for i in range(100):
customers.append({
"id": i + 1,
"name": f"Customer {i + 1}",
"email": f"customer{i+1}@example.com",
"phone": f"+1{5550000000 + i}",
"address": f"{100 + i} Main Street, City {i % 10}, State {i % 5}, ZIP {10000 + i}",
"description": f"This is a detailed description for customer {i + 1}. " * 10 +
f"They have been a loyal customer since {2000 + (i % 25)}. " +
f"Their preferences include various products and services. " * 5,
"orders": [
{
"orderId": f"ORD-{i+1}-{j+1}",
"date": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
"total": round(100.0 + (i * 10) + (j * 5), 2),
"items": [
{
"productId": f"PROD-{k+1}",
"quantity": (k % 5) + 1,
"price": round(10.0 + k * 2, 2)
}
for k in range(3)
]
}
for j in range(5)
],
"metadata": {
"created": f"2020-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
"lastLogin": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
"tags": [f"tag-{i % 10}", f"category-{i % 5}", f"segment-{i % 3}"]
}
})
return {"customers": customers}
def _createLargeProductsData(self) -> Dict[str, Any]:
"""Create a large products dataset for budget testing."""
products = []
# Create 200 products with detailed information
categories = ["Electronics", "Clothing", "Books", "Home & Garden", "Sports", "Toys", "Automotive", "Health"]
for i in range(200):
category = categories[i % len(categories)]
products.append({
"id": f"PROD-{i+1:04d}",
"name": f"Product {i+1} - {category}",
"category": category,
"price": round(10.0 + (i * 2.5), 2),
"cost": round(5.0 + (i * 1.5), 2),
"inStock": i % 3 != 0,
"stockQuantity": (i % 100) * 10,
"description": f"This is a comprehensive product description for Product {i+1}. " * 15 +
f"It belongs to the {category} category and offers excellent value. " * 10 +
f"Features include: feature-1, feature-2, feature-3, and many more. " * 5,
"specifications": {
"weight": f"{1.0 + (i % 10)} kg",
"dimensions": f"{10 + (i % 20)}x{5 + (i % 15)}x{3 + (i % 10)} cm",
"color": ["red", "blue", "green", "black", "white"][i % 5],
"material": ["plastic", "metal", "wood", "fabric"][i % 4],
"warranty": f"{1 + (i % 5)} years"
},
"reviews": [
{
"userId": f"USER-{j+1}",
"rating": (j % 5) + 1,
"comment": f"Review {j+1} for product {i+1}: " + "This is a detailed review comment. " * 10,
"date": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}"
}
for j in range(3)
],
"relatedProducts": [f"PROD-{k+1:04d}" for k in range(max(0, i-2), min(200, i+3)) if k != i]
})
return {"products": products}
def _createLargeDocumentsData(self) -> Dict[str, Any]:
"""Create a large documents dataset for budget testing."""
documents = []
# Create 50 documents with nested structures
for i in range(50):
sections = []
for j in range(10):
elements = []
for k in range(5):
if k % 2 == 0:
elements.append({
"type": "heading",
"level": (k % 3) + 1,
"content": {
"text": f"Section {j+1} Heading {k+1} for Document {i+1}",
"style": "bold"
}
})
else:
elements.append({
"type": "paragraph",
"content": {
"text": f"This is paragraph {k+1} in section {j+1} of document {i+1}. " * 20 +
f"It contains detailed information about various topics. " * 15 +
f"The content is structured and well-organized. " * 10
}
})
sections.append({
"id": f"sec-{i+1}-{j+1}",
"title": f"Section {j+1}",
"content_type": "mixed",
"elements": elements,
"metadata": {
"created": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
"modified": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
"author": f"Author-{(i % 10) + 1}",
"tags": [f"tag-{j % 10}", f"category-{i % 5}"]
}
})
documents.append({
"id": f"doc-{i+1:03d}",
"title": f"Document {i+1} - Comprehensive Report",
"description": f"This is a comprehensive document with detailed information. " * 30 +
f"It covers multiple topics and sections. " * 20 +
f"The content is extensive and well-structured. " * 15,
"sections": sections,
"metadata": {
"created": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
"modified": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
"author": f"Author-{(i % 10) + 1}",
"version": f"1.{(i % 10)}",
"status": ["draft", "review", "published"][i % 3],
"tags": [f"tag-{i % 20}" for _ in range(5)]
}
})
return {
"metadata": {
"title": "Large Document Collection",
"description": "A collection of 50 comprehensive documents for testing budget limits",
"totalDocuments": 50,
"created": "2024-01-01"
},
"documents": documents
}
def splitJsonRandomly(self, jsonString: str, numParts: int = 3) -> List[str]:
"""
Split JSON string randomly into specified number of parts.
Simulates real AI response cuts - can split anywhere, even in the middle of strings/numbers/structures.
This is the REAL scenario: AI response gets cut off randomly, not at convenient points.
"""
if numParts < 2:
return [jsonString]
jsonLength = len(jsonString)
# Generate truly random split points - can be anywhere!
# Only ensure minimum part size to avoid empty parts
minPartSize = max(10, jsonLength // (numParts * 3)) # Smaller minimum to allow more randomness
splitPoints = []
for _ in range(numParts - 1):
# Generate random point - can be anywhere in the string
# Only ensure we don't create parts smaller than minimum
minPoint = len(splitPoints) * minPartSize if splitPoints else minPartSize
maxPoint = jsonLength - (numParts - len(splitPoints) - 1) * minPartSize
if maxPoint <= minPoint:
# If we can't avoid minimum size, just use the boundary
splitPoint = minPoint
else:
# Truly random point - can be in the middle of anything!
splitPoint = random.randint(minPoint, maxPoint)
splitPoints.append(splitPoint)
splitPoints.sort()
# Create parts - these can be cut anywhere, even mid-string, mid-number, etc.
parts = []
start = 0
for splitPoint in splitPoints:
parts.append(jsonString[start:splitPoint])
start = splitPoint
parts.append(jsonString[start:]) # Last part
return parts
def _log(self, message: str):
"""Add message to log buffer."""
self.logBuffer.append(message)
print(message)
def normalizeJson(self, jsonString: str) -> Optional[Dict[str, Any]]:
"""Normalize JSON string by parsing and re-serializing. Returns None if parsing fails."""
try:
parsed = json.loads(jsonString)
return parsed
except json.JSONDecodeError:
# Try to close incomplete JSON structures
try:
from modules.shared.jsonUtils import closeJsonStructures, tryParseJson
closed = closeJsonStructures(jsonString)
parsed, error, _ = tryParseJson(closed)
if error is None and parsed is not None:
return parsed
except Exception:
pass
# Return None if all parsing attempts fail
return None
def compareJson(self, original: Dict[str, Any], merged: Dict[str, Any]) -> Dict[str, Any]:
"""Compare original and merged JSON structures."""
originalStr = json.dumps(original, sort_keys=True, indent=2)
mergedStr = json.dumps(merged, sort_keys=True, indent=2)
exactMatch = originalStr == mergedStr
# Deep comparison
differences = []
self._findDifferences(original, merged, "", differences)
return {
"exactMatch": exactMatch,
"differences": differences,
"originalSize": len(originalStr),
"mergedSize": len(mergedStr),
"sizeMatch": len(originalStr) == len(mergedStr)
}
def _findDifferences(self, obj1: Any, obj2: Any, path: str, differences: List[str]):
"""Recursively find differences between two JSON objects."""
if type(obj1) != type(obj2):
differences.append(f"{path}: Type mismatch - {type(obj1).__name__} vs {type(obj2).__name__}")
return
if isinstance(obj1, dict):
allKeys = set(obj1.keys()) | set(obj2.keys())
for key in allKeys:
newPath = f"{path}.{key}" if path else key
if key not in obj1:
differences.append(f"{newPath}: Missing in original")
elif key not in obj2:
differences.append(f"{newPath}: Missing in merged")
else:
self._findDifferences(obj1[key], obj2[key], newPath, differences)
elif isinstance(obj1, list):
if len(obj1) != len(obj2):
differences.append(f"{path}: Length mismatch - {len(obj1)} vs {len(obj2)}")
else:
for i, (item1, item2) in enumerate(zip(obj1, obj2)):
newPath = f"{path}[{i}]"
self._findDifferences(item1, item2, newPath, differences)
else:
if obj1 != obj2:
differences.append(f"{path}: Value mismatch - {obj1} vs {obj2}")
async def testJsonSplitMerge(self, jsonFile: Dict[str, Any]) -> Dict[str, Any]:
"""Test splitting and merging a single JSON file."""
fileName = jsonFile["name"]
originalData = jsonFile["data"]
self._log("")
self._log("="*80)
self._log(f"TESTING JSON SPLIT AND MERGE: {fileName}")
self._log("="*80)
# Convert to JSON string
originalJsonString = json.dumps(originalData, indent=2, ensure_ascii=False)
originalSize = len(originalJsonString)
# Log original JSON
self._log("")
self._log("="*80)
self._log("ORIGINAL JSON")
self._log("="*80)
self._log(f"JSON length: {originalSize} characters")
self._log("")
self._log("Full JSON content:")
self._log("-"*80)
jsonLines = originalJsonString.split('\n')
if len(jsonLines) > 50:
for line in jsonLines[:25]:
self._log(line)
self._log(f"... ({len(jsonLines) - 50} lines omitted) ...")
for line in jsonLines[-25:]:
self._log(line)
else:
for line in jsonLines:
self._log(line)
# Split JSON at random position (simulating AI response cut)
self._log("")
self._log("="*80)
self._log("SPLITTING JSON AT RANDOM POSITION (SIMULATING AI RESPONSE CUT)")
self._log("="*80)
# Find random cut position (not at start or end)
import random
minCutPos = max(100, originalSize // 10) # At least 10% from start
maxCutPos = min(originalSize - 100, originalSize * 9 // 10) # At least 10% from end
cutPosition = random.randint(minCutPos, maxCutPos)
# Get part from start to cut
partContent = originalJsonString[:cutPosition]
self._log("")
self._log("="*80)
self._log("PART (from start to cut):")
self._log("="*80)
self._log(f"Cut position: {cutPosition} characters")
self._log(f"Part length: {len(partContent)} characters")
self._log("")
self._log("Part content:")
partLines = partContent.split('\n')
if len(partLines) > 30:
for line in partLines[:15]:
self._log(f" {line}")
self._log(f" ... ({len(partLines) - 30} lines omitted) ...")
for line in partLines[-15:]:
self._log(f" {line}")
else:
for line in partLines:
self._log(f" {line}")
# Generate contexts using getContexts()
self._log("")
self._log("="*80)
self._log("GENERATING CONTINUATION CONTEXTS")
self._log("="*80)
contexts = getContexts(partContent)
# Log overlap context
self._log("")
self._log("="*80)
self._log("OVERLAP CONTEXT (for merging):")
self._log("="*80)
overlapLines = contexts.overlapContext.split('\n')
if len(overlapLines) > 30:
for line in overlapLines[:15]:
self._log(f" {line}")
self._log(f" ... ({len(overlapLines) - 30} lines omitted) ...")
for line in overlapLines[-15:]:
self._log(f" {line}")
else:
for line in overlapLines:
self._log(f" {line}")
# Log hierarchy context
self._log("")
self._log("="*80)
self._log("HIERARCHY CONTEXT (with budget logic):")
self._log("="*80)
hierarchyLines = contexts.hierarchyContext.split('\n')
if len(hierarchyLines) > 30:
for line in hierarchyLines[:15]:
self._log(f" {line}")
self._log(f" ... ({len(hierarchyLines) - 30} lines omitted) ...")
for line in hierarchyLines[-15:]:
self._log(f" {line}")
else:
for line in hierarchyLines:
self._log(f" {line}")
# Test completePart as valid JSON
self._log("")
self._log("="*80)
self._log("COMPLETE PART (should be valid JSON):")
self._log("="*80)
completeLines = contexts.completePart.split('\n')
if len(completeLines) > 30:
for line in completeLines[:15]:
self._log(f" {line}")
self._log(f" ... ({len(completeLines) - 30} lines omitted) ...")
for line in completeLines[-15:]:
self._log(f" {line}")
else:
for line in completeLines:
self._log(f" {line}")
# Validate completePart as JSON
self._log("")
self._log("="*80)
self._log("VALIDATING COMPLETE PART AS JSON:")
self._log("="*80)
isValidJson = False
parsedCompletePart = None
jsonError = None
try:
parsedCompletePart = json.loads(contexts.completePart)
isValidJson = True
self._log(" ✅ completePart is valid JSON")
self._log(f" Parsed type: {type(parsedCompletePart).__name__}")
# Compare with original if possible
if isinstance(parsedCompletePart, dict) and isinstance(originalData, dict):
comparison = self.compareJson(originalData, parsedCompletePart)
self._log(f" Comparison with original:")
self._log(f" Exact match: {comparison['exactMatch']}")
self._log(f" Size match: {comparison['sizeMatch']}")
if comparison['differences']:
self._log(f" Differences found: {len(comparison['differences'])}")
for diff in comparison['differences'][:10]: # Show first 10 differences
self._log(f" - {diff}")
if len(comparison['differences']) > 10:
self._log(f" ... ({len(comparison['differences']) - 10} more differences)")
else:
self._log(" No differences found")
elif isinstance(parsedCompletePart, list) and isinstance(originalData, list):
self._log(f" Both are lists: original={len(originalData)} items, completePart={len(parsedCompletePart)} items")
else:
self._log(f" Different types: original={type(originalData).__name__}, completePart={type(parsedCompletePart).__name__}")
except json.JSONDecodeError as e:
isValidJson = False
jsonError = str(e)
self._log(f" ❌ completePart is NOT valid JSON")
self._log(f" Error: {jsonError}")
self._log(f" Error position: line {e.lineno}, column {e.colno}")
# Return test results
return {
"success": isValidJson,
"fileName": fileName,
"originalSize": originalSize,
"cutPosition": cutPosition,
"partSize": len(partContent),
"overlapContextSize": len(contexts.overlapContext),
"hierarchyContextSize": len(contexts.hierarchyContext),
"completePartSize": len(contexts.completePart),
"isValidJson": isValidJson,
"jsonError": jsonError,
"parsedCompletePart": parsedCompletePart is not None
}
async def testAllJsonFiles(self) -> Dict[str, Any]:
"""Test splitting and merging all test JSON files."""
print("\n" + "="*80)
print("TESTING JSON SPLIT AND MERGE")
print("="*80)
testFiles = self.createTestJsonFiles()
results = {}
for jsonFile in testFiles:
try:
result = await self.testJsonSplitMerge(jsonFile)
results[jsonFile["name"]] = result
# Small delay between tests
await asyncio.sleep(0.5)
except Exception as e:
import traceback
print(f"\n❌ Error testing {jsonFile['name']}: {str(e)}")
print(traceback.format_exc())
results[jsonFile["name"]] = {
"success": False,
"error": str(e),
"traceback": traceback.format_exc()
}
return results
def _writeLogFile(self):
"""Write log buffer to file."""
logDir = os.path.join(os.path.dirname(__file__), "..", "..", "..", "local", "debug")
os.makedirs(logDir, exist_ok=True)
logFilePath = os.path.join(logDir, "test12_json_split_merge_results.txt")
with open(logFilePath, 'w', encoding='utf-8') as f:
f.write('\n'.join(self.logBuffer))
self.logFile = logFilePath
print(f"\n📝 Detailed log written to: {logFilePath}")
async def runTest(self):
"""Run the complete test."""
self._log("="*80)
self._log("JSON SPLIT AND MERGE TEST 12")
self._log("="*80)
try:
# Test all JSON files
results = await self.testAllJsonFiles()
# Write log file
self._writeLogFile()
# Summary
print("\n" + "="*80)
print("TEST SUMMARY")
print("="*80)
successCount = 0
for fileName, result in results.items():
if result.get("success"):
successCount += 1
isValidJson = result.get("isValidJson", False)
if isValidJson:
print(f"✅ {fileName:30s}: Valid JSON - completePart parsed successfully")
else:
jsonError = result.get("jsonError", "Unknown error")
print(f"⚠️ {fileName:30s}: Contexts generated but completePart is not valid JSON - {jsonError}")
else:
error = result.get("error", "Unknown error")
print(f"❌ {fileName:30s}: FAILED - {error}")
print(f"\nResults: {successCount}/{len(results)} successful")
self.testResults = {
"success": successCount == len(results),
"totalFiles": len(results),
"successCount": successCount,
"results": results
}
return self.testResults
except Exception as e:
import traceback
print(f"\n❌ Test failed with error: {type(e).__name__}: {str(e)}")
print(f"Traceback:\n{traceback.format_exc()}")
self.testResults = {
"success": False,
"error": str(e),
"traceback": traceback.format_exc()
}
return self.testResults
async def main():
"""Run JSON split and merge test 12."""
tester = JsonSplitMergeTester12()
results = await tester.runTest()
# Print final results as JSON for easy parsing
print("\n" + "="*80)
print("FINAL RESULTS (JSON)")
print("="*80)
print(json.dumps(results, indent=2, default=str))
if __name__ == "__main__":
asyncio.run(main())

216
tests/test_overlap_context.py
View file

@ -1,216 +0,0 @@
# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Test function to verify structure hierarchy and overlap context generation.
Tests the functions used to generate continuation prompts for incomplete JSON.
"""
import json
import os
from pathlib import Path
def testOverlapContext():
"""
Test function that loads two JSON parts and returns:
1. Structure hierarchy result
2. Overlap requirement context result
"""
# Load the JSON file (incomplete/cut JSON)
basePath = Path(__file__).parent.parent.parent / "local" / "debug" / "prompts"
file1Path = basePath / "20260104-220716-032-chapter_2_section_section_2_response.txt"
# Read JSON (incomplete)
with open(file1Path, 'r', encoding='utf-8') as f:
json1Content = f.read().strip()
# Find the break position in json1 (where it was cut)
# The last line in json1 is incomplete: [37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039
# We need to find where this incomplete array element ends (right after the last number)
# Find the last number in the file - that's where the content actually ends
import re
# Find all numbers at the end and get the position of the last one
# Look for the pattern: number followed by whitespace/newline or end of string
matches = list(re.finditer(r'\d+', json1Content))
if matches:
lastMatch = matches[-1]
# Break position is right after the last number (where the closing ] should be)
breakPosition = lastMatch.end()
else:
# Fallback: use end of file
breakPosition = len(json1Content.rstrip())
print(f"Break position determined: {breakPosition}")
print(f"Content at break position: '{json1Content[max(0, breakPosition-50):breakPosition+10]}'")
# Import the functions we need to test
import sys
sys.path.insert(0, str(Path(__file__).parent.parent))
from modules.shared.jsonUtils import findStructureHierarchy, extractCutPiece, buildIncompleteContext
from modules.services.serviceGeneration.paths.codePath import CodeGenerationPath
# Test 1: Find structure hierarchy
print("=" * 80)
print("TEST 1: Structure Hierarchy")
print("=" * 80)
print(f"Break position: {breakPosition}")
print(f"JSON length: {len(json1Content)}")
print(f"Content around break: '{json1Content[max(0, breakPosition-100):breakPosition+20]}'")
hierarchy = findStructureHierarchy(json1Content, breakPosition)
print(f"\nHierarchy levels found: {len(hierarchy) if hierarchy else 0}")
if not hierarchy:
print("WARNING: No hierarchy found! This suggests the function isn't working correctly.")
else:
print("\nHierarchy details (from root to cut level):")
for i, level in enumerate(hierarchy):
levelType = level['type']
levelKey = level.get('key', 'N/A')
levelDepth = level['depth']
levelStart = level['start_pos']
levelEnd = level['end_pos']
print(f" Level {i}: {levelType:6s} depth={levelDepth} key='{levelKey}' start={levelStart} end={levelEnd}")
# Show a snippet of content at this level
if levelStart < len(json1Content):
snippet = json1Content[levelStart:min(levelStart + 50, levelEnd, len(json1Content))]
print(f" Content: {repr(snippet)}")
# Test 2: Extract cut piece
print("\n" + "=" * 80)
print("TEST 2: Extract Cut Piece")
print("=" * 80)
cutPiece = extractCutPiece(json1Content, breakPosition)
print(f"\nCut piece extracted (length: {len(cutPiece)}):")
if cutPiece:
print(cutPiece[:500] if len(cutPiece) > 500 else cutPiece)
else:
print("WARNING: Cut piece is empty! This suggests the function isn't working correctly.")
# Try to manually find the cut piece
# Look backwards from break position for the start of the incomplete array
i = breakPosition - 1
while i >= 0 and json1Content[i] not in ['[', ',', '\n']:
i -= 1
if i >= 0 and json1Content[i] == '[':
manualCutPiece = json1Content[i:breakPosition]
print(f"\nManually found cut piece: {manualCutPiece[:200]}")
# Test 3: Build incomplete context (structure hierarchy with cut point)
print("\n" + "=" * 80)
print("TEST 3: Build Incomplete Context (Structure Hierarchy with Cut Point)")
print("=" * 80)
print("Expected: Should show complete hierarchy from root to cut point")
print(" with complete elements before cut and cut piece marked")
incompleteContext = buildIncompleteContext(json1Content, breakPosition)
print(f"\nIncomplete context (length: {len(incompleteContext)} chars):")
print("-" * 80)
print(incompleteContext)
print("-" * 80)
# Validate the output
if incompleteContext:
# Check if it shows hierarchy (should have multiple levels of indentation)
lines = incompleteContext.split('\n')
indentLevels = set()
for line in lines:
if line.strip():
indent = len(line) - len(line.lstrip())
indentLevels.add(indent)
print(f"\nValidation: Found {len(indentLevels)} different indent levels (should be > 1 for hierarchy)")
# Check if cut point is marked
if "<-- CUT POINT" in incompleteContext:
print("Validation: Cut point marker found ✓")
else:
print("Validation: WARNING - Cut point marker NOT found!")
# Check if root structure is shown
if incompleteContext.strip().startswith('{') or incompleteContext.strip().startswith('['):
print("Validation: Root structure opening found ✓")
else:
print("Validation: WARNING - Root structure opening NOT found!")
else:
print("WARNING: Incomplete context is empty!")
# Test 4: Extract overlap context (cut part and full part before same level)
print("\n" + "=" * 80)
print("TEST 4: Extract Overlap Context (Cut Part + Full Part Before Same Level)")
print("=" * 80)
overlapContext = CodeGenerationPath._extractOverlapContext(json1Content, breakPosition)
print(f"\nOverlap context:")
print(overlapContext)
# Return results as dictionary
results = {
"hierarchy": hierarchy,
"cutPiece": cutPiece,
"incompleteContext": incompleteContext,
"overlapContext": overlapContext,
"breakPosition": breakPosition,
"json1Length": len(json1Content),
"json1Content": json1Content
}
return results
if __name__ == "__main__":
print("Testing Overlap Context Generation")
print("=" * 80)
results = testOverlapContext()
print("\n" + "=" * 80)
print("SUMMARY")
print("=" * 80)
print(f"\nBreak position: {results['breakPosition']}")
print(f"JSON1 length: {results['json1Length']}")
print(f"Hierarchy levels: {len(results['hierarchy']) if results['hierarchy'] else 0}")
print(f"Cut piece length: {len(results['cutPiece'])}")
print(f"Incomplete context length: {len(results['incompleteContext'])}")
print(f"Overlap context length: {len(results['overlapContext'])}")
# Save results to file for inspection
outputPath = Path(__file__).parent.parent.parent / "local" / "debug" / "test_overlap_results.txt"
outputPath.parent.mkdir(parents=True, exist_ok=True)
with open(outputPath, 'w', encoding='utf-8') as f:
f.write("=" * 80 + "\n")
f.write("OVERLAP CONTEXT TEST RESULTS\n")
f.write("=" * 80 + "\n\n")
f.write("FIRST JSON (CUT/INCOMPLETE):\n")
f.write("-" * 80 + "\n")
f.write(f"Break position: {results['breakPosition']}\n")
f.write(f"JSON length: {results['json1Length']}\n")
json1Content = results['json1Content']
f.write(f"Content around break: '{json1Content[max(0, results['breakPosition']-100):results['breakPosition']+20]}'\n\n")
f.write("Full JSON1 content:\n")
f.write(json1Content)
f.write("\n\n" + "=" * 80 + "\n")
f.write("STRUCTURE HIERARCHY:\n")
f.write("-" * 80 + "\n")
if results['hierarchy']:
f.write(f"Hierarchy levels found: {len(results['hierarchy'])}\n\n")
f.write("Hierarchy details (from root to cut level):\n")
for i, level in enumerate(results['hierarchy']):
levelType = level['type']
levelKey = level.get('key', 'N/A')
levelDepth = level['depth']
levelStart = level['start_pos']
levelEnd = level['end_pos']
f.write(f" Level {i}: {levelType:6s} depth={levelDepth} key='{levelKey}' start={levelStart} end={levelEnd}\n")
else:
f.write("No hierarchy found\n")
f.write("\n\n" + "=" * 80 + "\n")
f.write("INCOMPLETE CONTEXT (Structure Hierarchy with Cut Point):\n")
f.write("-" * 80 + "\n")
f.write(results['incompleteContext'])
f.write("\n\n" + "=" * 80 + "\n")
f.write("OVERLAP CONTEXT (Object containing the cut element):\n")
f.write("-" * 80 + "\n")
f.write(results['overlapContext'])
print(f"\n\nFull results saved to: {outputPath}")