performance tests with json repair and cut handling
This commit is contained in:
ValueOn AG
parent
c40d5e5133
commit
e331c7fbce
11 changed files with 1129 additions and 7294 deletions
|
|
@ -278,11 +278,13 @@ class JsonContinuationContexts(BaseModel):
|
|||
- hierarchyContext: Full structure from root to cut WITHOUT budget limitations (for internal use)
|
||||
- hierarchyContextForPrompt: Full structure from root to cut WITH budget limitations (for prompts)
|
||||
- completePart: Valid JSON with all structures properly closed
|
||||
- jsonParsingSuccess: True if completePart is valid parseable JSON
|
||||
"""
|
||||
overlapContext: str = Field(description="The innermost object/array element containing the cut point (for merging)")
|
||||
hierarchyContext: str = Field(description="Full structure from root to cut WITHOUT budget limitations (for internal use)")
|
||||
hierarchyContextForPrompt: str = Field(description="Full structure from root to cut WITH budget limitations (for prompts)")
|
||||
completePart: str = Field(description="Valid JSON with all structures properly closed")
|
||||
jsonParsingSuccess: bool = Field(default=False, description="True if completePart is valid parseable JSON")
|
||||
|
||||
|
||||
class SectionPromptArgs(BaseModel):
|
||||
|
|
|
|||
|
|
@ -1,901 +0,0 @@
|
|||
================================================================================
|
||||
JSON MERGE OPERATION #1
|
||||
================================================================================
|
||||
Timestamp: 2026-01-05T22:51:14.407688
|
||||
|
||||
INPUT:
|
||||
Accumulated length: 31737 chars
|
||||
New Fragment length: 32830 chars
|
||||
Accumulated: 409 lines (showing first 5 and last 5)
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
"content": {
|
||||
... (399 lines omitted) ...
|
||||
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
|
||||
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
|
||||
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
|
||||
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
|
||||
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039
|
||||
New Fragment: 406 lines (showing first 5 and last 5)
|
||||
```json
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
... (396 lines omitted) ...
|
||||
[79873, 79889, 79901, 79903, 79907, 79939, 79943, 79967, 79973, 79979],
|
||||
[79987, 79997, 79999, 80021, 80039, 80051, 80071, 80077, 80107, 80111],
|
||||
[80141, 80147, 80149, 80153, 80167, 80173, 80177, 80191, 80207, 80209],
|
||||
[80221, 80231, 80233, 80239, 80251, 80263, 80273, 80279, 80287, 80309],
|
||||
[80317, 80329, 80341, 80347, 80363, 80369,
|
||||
|
||||
|
||||
Normalized Accumulated (31737 chars)
|
||||
(showing first 5 and last 5 of 409 lines)
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
"content": {
|
||||
... (399 lines omitted) ...
|
||||
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
|
||||
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
|
||||
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
|
||||
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
|
||||
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039
|
||||
|
||||
Normalized New Fragment (32822 chars)
|
||||
(showing first 5 and last 5 of 405 lines)
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
"content": {
|
||||
... (395 lines omitted) ...
|
||||
[79873, 79889, 79901, 79903, 79907, 79939, 79943, 79967, 79973, 79979],
|
||||
[79987, 79997, 79999, 80021, 80039, 80051, 80071, 80077, 80107, 80111],
|
||||
[80141, 80147, 80149, 80153, 80167, 80173, 80177, 80191, 80207, 80209],
|
||||
[80221, 80231, 80233, 80239, 80251, 80263, 80273, 80279, 80287, 80309],
|
||||
[80317, 80329, 80341, 80347, 80363, 80369,
|
||||
STEP: PHASE 1
|
||||
Description: Finding overlap between JSON strings
|
||||
⏳ In progress...
|
||||
|
||||
Overlap Detection (string):
|
||||
Overlap length: 0
|
||||
⚠️ No overlap detected - appending all
|
||||
|
||||
⚠️ NO OVERLAP FOUND - This indicates iterations should stop
|
||||
Closing JSON and returning final result
|
||||
|
||||
Closed JSON (31743 chars):
|
||||
==============================================================================
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
"content": {
|
||||
"headers": ["Spalte1", "Spalte2", "Spalte3", "Spalte4", "Spalte5", "Spalte6", "Spalte7", "Spalte8", "Spalte9", "Spalte10"],
|
||||
"rows": [
|
||||
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29],
|
||||
[31, 37, 41, 43, 47, 53, 59, 61, 67, 71],
|
||||
[73, 79, 83, 89, 97, 101, 103, 107, 109, 113],
|
||||
[127, 131, 137, 139, 149, 151, 157, 163, 167, 173],
|
||||
[179, 181, 191, 193, 197, 199, 211, 223, 227, 229],
|
||||
[233, 239, 241, 251, 257, 263, 269, 271, 277, 281],
|
||||
[283, 293, 307, 311, 313, 317, 331, 337, 347, 349],
|
||||
[353, 359, 367, 373, 379, 383, 389, 397, 401, 409],
|
||||
[419, 421, 431, 433, 439, 443, 449, 457, 461, 463],
|
||||
[467, 479, 487, 491, 499, 503, 509, 521, 523, 541],
|
||||
[547, 557, 563, 569, 571, 577, 587, 593, 599, 601],
|
||||
[607, 613, 617, 619, 631, 641, 643, 647, 653, 659],
|
||||
[661, 673, 677, 683, 691, 701, 709, 719, 727, 733],
|
||||
[739, 743, 751, 757, 761, 769, 773, 787, 797, 809],
|
||||
[811, 821, 823, 827, 829, 839, 853, 857, 859, 863],
|
||||
[877, 881, 883, 887, 907, 911, 919, 929, 937, 941],
|
||||
[947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013],
|
||||
[1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069],
|
||||
[1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151],
|
||||
[1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223],
|
||||
[1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291],
|
||||
[1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373],
|
||||
[1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451],
|
||||
[1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511],
|
||||
[1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583],
|
||||
[1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657],
|
||||
[1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733],
|
||||
[1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811],
|
||||
[1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889],
|
||||
[1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987],
|
||||
[1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053],
|
||||
[2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129],
|
||||
[2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213],
|
||||
[2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287],
|
||||
[2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357],
|
||||
[2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423],
|
||||
[2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531],
|
||||
[2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617],
|
||||
[2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687],
|
||||
[2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741],
|
||||
[2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819],
|
||||
[2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903],
|
||||
[2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999],
|
||||
[3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079],
|
||||
[3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181],
|
||||
[3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257],
|
||||
[3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331],
|
||||
[3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413],
|
||||
[3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511],
|
||||
[3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571],
|
||||
[3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643],
|
||||
[3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727],
|
||||
[3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821],
|
||||
[3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907],
|
||||
[3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989],
|
||||
[4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057],
|
||||
[4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139],
|
||||
[4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231],
|
||||
[4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297],
|
||||
[4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409],
|
||||
[4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493],
|
||||
[4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583],
|
||||
[4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657],
|
||||
[4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751],
|
||||
[4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831],
|
||||
[4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937],
|
||||
[4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003],
|
||||
[5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087],
|
||||
[5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179],
|
||||
[5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279],
|
||||
[5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387],
|
||||
[5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443],
|
||||
[5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521],
|
||||
[5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639],
|
||||
[5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693],
|
||||
[5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791],
|
||||
[5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857],
|
||||
[5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939],
|
||||
[5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053],
|
||||
[6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133],
|
||||
[6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221],
|
||||
[6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301],
|
||||
[6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367],
|
||||
[6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473],
|
||||
[6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571],
|
||||
[6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673],
|
||||
[6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761],
|
||||
[6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833],
|
||||
[6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917],
|
||||
[6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997],
|
||||
[7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103],
|
||||
[7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207],
|
||||
[7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297],
|
||||
[7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411],
|
||||
[7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499],
|
||||
[7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561],
|
||||
[7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643],
|
||||
[7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723],
|
||||
[7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829],
|
||||
[7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919],
|
||||
[7927, 7933, 7937, 7949, 7951, 7963, 7993, 8009, 8011, 8017],
|
||||
[8039, 8053, 8059, 8069, 8081, 8087, 8089, 8093, 8101, 8111],
|
||||
[8117, 8123, 8147, 8161, 8167, 8171, 8179, 8191, 8209, 8219],
|
||||
[8221, 8231, 8233, 8237, 8243, 8263, 8269, 8273, 8287, 8291],
|
||||
[8293, 8297, 8311, 8317, 8329, 8353, 8363, 8369, 8377, 8387],
|
||||
[8389, 8419, 8423, 8429, 8431, 8443, 8447, 8461, 8467, 8501],
|
||||
[8513, 8521, 8527, 8537, 8539, 8543, 8563, 8573, 8581, 8597],
|
||||
[8599, 8609, 8623, 8627, 8629, 8641, 8647, 8663, 8669, 8677],
|
||||
[8681, 8689, 8693, 8699, 8707, 8713, 8719, 8731, 8737, 8741],
|
||||
[8747, 8753, 8761, 8779, 8783, 8803, 8807, 8819, 8821, 8831],
|
||||
[8837, 8839, 8849, 8861, 8863, 8867, 8887, 8893, 8923, 8929],
|
||||
[8933, 8941, 8951, 8963, 8969, 8971, 8999, 9001, 9007, 9011],
|
||||
[9013, 9029, 9041, 9043, 9049, 9059, 9067, 9091, 9103, 9109],
|
||||
[9127, 9133, 9137, 9151, 9157, 9161, 9173, 9181, 9187, 9199],
|
||||
[9203, 9209, 9221, 9227, 9239, 9241, 9257, 9277, 9281, 9283],
|
||||
[9293, 9311, 9319, 9323, 9337, 9341, 9343, 9349, 9371, 9377],
|
||||
[9391, 9397, 9403, 9413, 9419, 9421, 9431, 9433, 9437, 9439],
|
||||
[9461, 9463, 9467, 9473, 9479, 9491, 9497, 9511, 9521, 9533],
|
||||
[9539, 9547, 9551, 9587, 9601, 9613, 9619, 9623, 9629, 9631],
|
||||
[9643, 9649, 9661, 9677, 9679, 9689, 9697, 9719, 9721, 9733],
|
||||
[9739, 9743, 9749, 9767, 9769, 9781, 9787, 9791, 9803, 9811],
|
||||
[9817, 9829, 9833, 9839, 9851, 9857, 9859, 9871, 9883, 9887],
|
||||
[9901, 9907, 9923, 9929, 9931, 9941, 9949, 9967, 9973, 10007],
|
||||
[10009, 10037, 10039, 10061, 10067, 10069, 10079, 10091, 10093, 10099],
|
||||
[10103, 10111, 10133, 10139, 10141, 10151, 10159, 10163, 10169, 10177],
|
||||
[10181, 10193, 10211, 10223, 10243, 10247, 10253, 10259, 10267, 10271],
|
||||
[10273, 10289, 10301, 10303, 10313, 10321, 10331, 10333, 10337, 10343],
|
||||
[10357, 10369, 10391, 10399, 10427, 10429, 10433, 10453, 10457, 10459],
|
||||
[10463, 10477, 10487, 10499, 10501, 10513, 10529, 10531, 10559, 10567],
|
||||
[10589, 10597, 10601, 10607, 10613, 10627, 10631, 10639, 10651, 10657],
|
||||
[10663, 10667, 10687, 10691, 10709, 10711, 10723, 10729, 10733, 10739],
|
||||
[10753, 10771, 10781, 10789, 10799, 10831, 10837, 10847, 10853, 10859],
|
||||
[10861, 10867, 10883, 10889, 10891, 10903, 10909, 10937, 10939, 10949],
|
||||
[10957, 10973, 10979, 10987, 10993, 11003, 11027, 11047, 11057, 11059],
|
||||
[11069, 11071, 11083, 11087, 11093, 11113, 11117, 11119, 11131, 11149],
|
||||
[11159, 11161, 11171, 11173, 11177, 11197, 11213, 11239, 11243, 11251],
|
||||
[11257, 11261, 11273, 11279, 11287, 11299, 11311, 11317, 11321, 11329],
|
||||
[11351, 11353, 11369, 11383, 11393, 11399, 11411, 11423, 11437, 11443],
|
||||
[11447, 11467, 11471, 11483, 11489, 11491, 11497, 11503, 11519, 11527],
|
||||
[11549, 11551, 11579, 11587, 11593, 11597, 11617, 11621, 11633, 11657],
|
||||
[11677, 11681, 11689, 11699, 11701, 11717, 11719, 11731, 11743, 11777],
|
||||
[11779, 11783, 11789, 11801, 11807, 11813, 11821, 11827, 11831, 11833],
|
||||
[11839, 11863, 11867, 11887, 11897, 11903, 11909, 11923, 11927, 11933],
|
||||
[11939, 11941, 11953, 11959, 11969, 11971, 11981, 11987, 12007, 12011],
|
||||
[12037, 12041, 12043, 12049, 12071, 12073, 12097, 12101, 12107, 12109],
|
||||
[12113, 12119, 12143, 12149, 12157, 12161, 12163, 12197, 12203, 12211],
|
||||
[12227, 12239, 12241, 12251, 12253, 12263, 12269, 12277, 12281, 12289],
|
||||
[12301, 12323, 12329, 12343, 12347, 12373, 12377, 12379, 12391, 12401],
|
||||
[12409, 12413, 12421, 12433, 12437, 12451, 12457, 12473, 12479, 12487],
|
||||
[12491, 12497, 12503, 12511, 12517, 12527, 12539, 12541, 12547, 12553],
|
||||
[12569, 12577, 12583, 12589, 12601, 12611, 12613, 12619, 12637, 12641],
|
||||
[12647, 12653, 12659, 12671, 12689, 12697, 12703, 12713, 12721, 12739],
|
||||
[12743, 12757, 12763, 12781, 12791, 12799, 12809, 12821, 12823, 12829],
|
||||
[12841, 12853, 12889, 12893, 12899, 12907, 12911, 12917, 12919, 12923],
|
||||
[12941, 12953, 12959, 12967, 12973, 12979, 12983, 13001, 13003, 13007],
|
||||
[13009, 13033, 13037, 13043, 13049, 13063, 13093, 13099, 13103, 13109],
|
||||
[13121, 13127, 13147, 13151, 13159, 13163, 13171, 13177, 13183, 13187],
|
||||
[13217, 13219, 13229, 13241, 13249, 13259, 13267, 13291, 13297, 13309],
|
||||
[13313, 13327, 13331, 13337, 13339, 13367, 13381, 13397, 13399, 13411],
|
||||
[13417, 13421, 13441, 13451, 13457, 13463, 13469, 13477, 13487, 13499],
|
||||
[13513, 13523, 13537, 13553, 13567, 13577, 13591, 13597, 13613, 13619],
|
||||
[13627, 13633, 13649, 13669, 13679, 13681, 13687, 13691, 13693, 13697],
|
||||
[13709, 13711, 13721, 13723, 13729, 13751, 13757, 13759, 13763, 13781],
|
||||
[13789, 13799, 13807, 13829, 13831, 13841, 13859, 13873, 13877, 13879],
|
||||
[13883, 13901, 13903, 13907, 13913, 13921, 13931, 13933, 13963, 13967],
|
||||
[13997, 13999, 14009, 14011, 14029, 14033, 14051, 14057, 14071, 14081],
|
||||
[14083, 14087, 14107, 14143, 14149, 14153, 14159, 14173, 14177, 14197],
|
||||
[14207, 14221, 14243, 14249, 14251, 14281, 14293, 14303, 14321, 14323],
|
||||
[14327, 14341, 14347, 14369, 14387, 14389, 14401, 14407, 14411, 14419],
|
||||
[14423, 14431, 14437, 14447, 14449, 14461, 14479, 14489, 14503, 14519],
|
||||
[14533, 14537, 14543, 14549, 14551, 14557, 14561, 14563, 14591, 14593],
|
||||
[14621, 14627, 14629, 14633, 14639, 14653, 14657, 14669, 14683, 14699],
|
||||
[14713, 14717, 14723, 14731, 14737, 14741, 14747, 14753, 14759, 14767],
|
||||
[14771, 14779, 14783, 14797, 14813, 14821, 14827, 14831, 14843, 14851],
|
||||
[14867, 14869, 14879, 14887, 14891, 14897, 14923, 14929, 14939, 14947],
|
||||
[14951, 14957, 14969, 14983, 15013, 15017, 15031, 15053, 15061, 15073],
|
||||
[15077, 15083, 15091, 15101, 15107, 15121, 15131, 15137, 15139, 15149],
|
||||
[15161, 15173, 15187, 15193, 15199, 15217, 15227, 15233, 15241, 15259],
|
||||
[15263, 15269, 15271, 15277, 15287, 15289, 15299, 15307, 15313, 15319],
|
||||
[15329, 15331, 15349, 15359, 15361, 15373, 15377, 15383, 15391, 15401],
|
||||
[15413, 15427, 15439, 15443, 15451, 15461, 15467, 15473, 15493, 15497],
|
||||
[15511, 15527, 15541, 15551, 15559, 15569, 15581, 15583, 15601, 15607],
|
||||
[15619, 15629, 15641, 15643, 15647, 15649, 15661, 15667, 15671, 15679],
|
||||
[15683, 15727, 15731, 15733, 15737, 15739, 15749, 15761, 15767, 15773],
|
||||
[15787, 15791, 15797, 15803, 15809, 15817, 15823, 15859, 15877, 15881],
|
||||
[15887, 15889, 15901, 15907, 15913, 15919, 15923, 15937, 15959, 15971],
|
||||
[15973, 15991, 16001, 16007, 16033, 16057, 16061, 16063, 16067, 16069],
|
||||
[16073, 16087, 16091, 16097, 16103, 16111, 16127, 16139, 16141, 16183],
|
||||
[16187, 16189, 16193, 16217, 16223, 16229, 16231, 16249, 16253, 16267],
|
||||
[16273, 16301, 16319, 16333, 16339, 16349, 16361, 16363, 16369, 16381],
|
||||
[16411, 16417, 16421, 16427, 16433, 16447, 16451, 16453, 16477, 16481],
|
||||
[16487, 16493, 16519, 16529, 16547, 16553, 16561, 16567, 16573, 16603],
|
||||
[16607, 16619, 16631, 16633, 16649, 16651, 16657, 16661, 16673, 16691],
|
||||
[16693, 16699, 16703, 16729, 16741, 16747, 16759, 16763, 16787, 16811],
|
||||
[16823, 16829, 16831, 16843, 16871, 16879, 16883, 16889, 16901, 16903],
|
||||
[16921, 16927, 16931, 16937, 16943, 16963, 16979, 16981, 16987, 16993],
|
||||
[17011, 17021, 17027, 17029, 17033, 17041, 17047, 17053, 17077, 17093],
|
||||
[17099, 17107, 17117, 17123, 17137, 17159, 17167, 17183, 17189, 17191],
|
||||
[17203, 17207, 17209, 17231, 17239, 17257, 17291, 17293, 17299, 17317],
|
||||
[17321, 17327, 17333, 17341, 17351, 17359, 17377, 17383, 17387, 17389],
|
||||
[17393, 17401, 17417, 17419, 17431, 17443, 17449, 17467, 17471, 17477],
|
||||
[17483, 17489, 17491, 17497, 17509, 17519, 17539, 17551, 17569, 17573],
|
||||
[17579, 17581, 17597, 17599, 17609, 17623, 17627, 17657, 17659, 17669],
|
||||
[17681, 17683, 17707, 17713, 17729, 17737, 17747, 17749, 17761, 17783],
|
||||
[17789, 17791, 17807, 17827, 17837, 17839, 17851, 17863, 17881, 17891],
|
||||
[17903, 17909, 17911, 17921, 17923, 17929, 17939, 17957, 17959, 17971],
|
||||
[17977, 17981, 17987, 17989, 18013, 18041, 18043, 18047, 18049, 18059],
|
||||
[18061, 18077, 18089, 18097, 18119, 18121, 18127, 18131, 18133, 18143],
|
||||
[18149, 18169, 18181, 18191, 18199, 18211, 18217, 18223, 18229, 18233],
|
||||
[18251, 18253, 18257, 18269, 18287, 18289, 18301, 18307, 18311, 18313],
|
||||
[18329, 18341, 18353, 18367, 18371, 18379, 18397, 18401, 18413, 18427],
|
||||
[18433, 18439, 18443, 18451, 18457, 18461, 18481, 18493, 18503, 18517],
|
||||
[18521, 18523, 18539, 18541, 18553, 18583, 18587, 18593, 18617, 18637],
|
||||
[18661, 18671, 18679, 18691, 18701, 18713, 18719, 18731, 18743, 18749],
|
||||
[18757, 18773, 18787, 18793, 18797, 18803, 18839, 18859, 18869, 18899],
|
||||
[18911, 18913, 18917, 18919, 18947, 18959, 18973, 18979, 19001, 19009],
|
||||
[19013, 19031, 19037, 19051, 19069, 19073, 19079, 19081, 19087, 19121],
|
||||
[19139, 19141, 19157, 19163, 19181, 19183, 19207, 19211, 19213, 19219],
|
||||
[19231, 19237, 19249, 19259, 19267, 19273, 19289, 19301, 19309, 19319],
|
||||
[19333, 19373, 19379, 19381, 19387, 19391, 19403, 19417, 19421, 19423],
|
||||
[19427, 19429, 19433, 19441, 19447, 19457, 19463, 19469, 19471, 19477],
|
||||
[19483, 19489, 19501, 19507, 19531, 19541, 19543, 19553, 19559, 19571],
|
||||
[19577, 19583, 19597, 19603, 19609, 19661, 19681, 19687, 19697, 19699],
|
||||
[19709, 19717, 19727, 19739, 19751, 19753, 19759, 19763, 19777, 19793],
|
||||
[19801, 19813, 19819, 19841, 19843, 19853, 19861, 19867, 19889, 19891],
|
||||
[19913, 19919, 19927, 19937, 19949, 19961, 19963, 19973, 19979, 19991],
|
||||
[19993, 19997, 20011, 20021, 20023, 20029, 20047, 20051, 20063, 20071],
|
||||
[20089, 20101, 20107, 20113, 20117, 20123, 20129, 20143, 20147, 20149],
|
||||
[20161, 20173, 20177, 20183, 20201, 20219, 20231, 20233, 20249, 20261],
|
||||
[20269, 20287, 20297, 20323, 20327, 20333, 20341, 20347, 20353, 20357],
|
||||
[20359, 20369, 20389, 20393, 20399, 20407, 20411, 20431, 20441, 20443],
|
||||
[20477, 20479, 20483, 20507, 20509, 20521, 20533, 20543, 20549, 20551],
|
||||
[20563, 20593, 20599, 20611, 20627, 20639, 20641, 20663, 20681, 20693],
|
||||
[20707, 20717, 20719, 20731, 20743, 20747, 20749, 20753, 20759, 20771],
|
||||
[20773, 20789, 20807, 20809, 20849, 20857, 20873, 20879, 20887, 20897],
|
||||
[20899, 20903, 20921, 20929, 20939, 20947, 20959, 20963, 20981, 20983],
|
||||
[21001, 21011, 21013, 21017, 21019, 21023, 21031, 21059, 21061, 21067],
|
||||
[21089, 21101, 21107, 21121, 21139, 21143, 21149, 21157, 21163, 21169],
|
||||
[21179, 21187, 21191, 21193, 21211, 21221, 21227, 21247, 21269, 21277],
|
||||
[21283, 21313, 21317, 21319, 21323, 21341, 21347, 21377, 21379, 21383],
|
||||
[21391, 21397, 21401, 21407, 21419, 21433, 21467, 21481, 21487, 21491],
|
||||
[21493, 21499, 21503, 21517, 21521, 21523, 21529, 21557, 21559, 21563],
|
||||
[21569, 21577, 21587, 21589, 21599, 21601, 21611, 21613, 21617, 21647],
|
||||
[21649, 21661, 21673, 21683, 21701, 21713, 21727, 21737, 21739, 21751],
|
||||
[21757, 21767, 21773, 21787, 21799, 21803, 21817, 21821, 21839, 21841],
|
||||
[21851, 21859, 21863, 21871, 21881, 21893, 21911, 21929, 21937, 21943],
|
||||
[21961, 21977, 21991, 21997, 22003, 22013, 22027, 22031, 22037, 22039],
|
||||
[22051, 22063, 22067, 22073, 22079, 22091, 22093, 22109, 22111, 22123],
|
||||
[22129, 22133, 22147, 22153, 22157, 22159, 22171, 22189, 22193, 22229],
|
||||
[22247, 22259, 22271, 22273, 22277, 22279, 22283, 22291, 22303, 22307],
|
||||
[22343, 22349, 22367, 22369, 22381, 22391, 22397, 22409, 22433, 22441],
|
||||
[22447, 22453, 22469, 22481, 22483, 22501, 22511, 22531, 22541, 22543],
|
||||
[22549, 22567, 22571, 22573, 22613, 22619, 22621, 22637, 22639, 22643],
|
||||
[22651, 22669, 22679, 22691, 22697, 22699, 22709, 22717, 22721, 22727],
|
||||
[22739, 22741, 22751, 22769, 22777, 22783, 22787, 22807, 22811, 22817],
|
||||
[22853, 22859, 22861, 22871, 22877, 22901, 22907, 22921, 22937, 22943],
|
||||
[22961, 22963, 22973, 22993, 23003, 23011, 23017, 23021, 23027, 23029],
|
||||
[23039, 23041, 23053, 23057, 23059, 23063, 23071, 23081, 23087, 23099],
|
||||
[23117, 23131, 23143, 23159, 23167, 23173, 23189, 23197, 23201, 23203],
|
||||
[23209, 23227, 23251, 23269, 23279, 23291, 23293, 23297, 23311, 23321],
|
||||
[23327, 23333, 23339, 23357, 23369, 23371, 23399, 23417, 23431, 23447],
|
||||
[23459, 23473, 23497, 23509, 23531, 23537, 23539, 23549, 23557, 23561],
|
||||
[23563, 23567, 23581, 23593, 23599, 23603, 23609, 23623, 23627, 23629],
|
||||
[23633, 23663, 23669, 23671, 23677, 23687, 23689, 23719, 23741, 23743],
|
||||
[23747, 23753, 23761, 23767, 23773, 23789, 23801, 23813, 23819, 23827],
|
||||
[23831, 23833, 23857, 23869, 23873, 23879, 23887, 23893, 23899, 23909],
|
||||
[23911, 23917, 23929, 23957, 23971, 23977, 23981, 23993, 24001, 24007],
|
||||
[24019, 24023, 24029, 24043, 24049, 24061, 24071, 24077, 24083, 24091],
|
||||
[24097, 24103, 24107, 24109, 24113, 24121, 24133, 24137, 24151, 24169],
|
||||
[24179, 24181, 24197, 24203, 24223, 24229, 24239, 24247, 24251, 24281],
|
||||
[24317, 24329, 24337, 24359, 24371, 24373, 24379, 24391, 24407, 24413],
|
||||
[24419, 24421, 24439, 24443, 24469, 24473, 24481, 24499, 24509, 24517],
|
||||
[24527, 24533, 24547, 24551, 24571, 24593, 24611, 24623, 24631, 24659],
|
||||
[24671, 24677, 24683, 24691, 24697, 24709, 24733, 24749, 24763, 24767],
|
||||
[24781, 24793, 24799, 24809, 24821, 24841, 24847, 24851, 24859, 24877],
|
||||
[24889, 24907, 24917, 24919, 24923, 24943, 24953, 24967, 24971, 24977],
|
||||
[24979, 24989, 25013, 25031, 25033, 25037, 25057, 25073, 25087, 25097],
|
||||
[25111, 25117, 25121, 25127, 25147, 25153, 25163, 25169, 25171, 25183],
|
||||
[25189, 25219, 25229, 25237, 25243, 25247, 25253, 25261, 25301, 25303],
|
||||
[25307, 25309, 25321, 25339, 25343, 25349, 25357, 25367, 25373, 25391],
|
||||
[25409, 25411, 25423, 25439, 25447, 25453, 25457, 25463, 25469, 25471],
|
||||
[25523, 25537, 25541, 25561, 25577, 25579, 25583, 25589, 25601, 25603],
|
||||
[25609, 25621, 25633, 25639, 25643, 25657, 25667, 25673, 25679, 25693],
|
||||
[25703, 25717, 25733, 25741, 25747, 25759, 25763, 25771, 25793, 25799],
|
||||
[25801, 25819, 25841, 25847, 25849, 25867, 25873, 25889, 25903, 25913],
|
||||
[25919, 25931, 25933, 25939, 25943, 25951, 25969, 25981, 25997, 25999],
|
||||
[26003, 26017, 26021, 26029, 26041, 26053, 26083, 26099, 26107, 26111],
|
||||
[26113, 26119, 26141, 26153, 26161, 26171, 26177, 26183, 26189, 26203],
|
||||
[26209, 26227, 26237, 26249, 26251, 26261, 26263, 26267, 26293, 26297],
|
||||
[26309, 26317, 26321, 26339, 26347, 26357, 26371, 26387, 26393, 26399],
|
||||
[26407, 26417, 26423, 26431, 26437, 26449, 26459, 26479, 26489, 26497],
|
||||
[26501, 26513, 26539, 26557, 26561, 26573, 26591, 26597, 26627, 26633],
|
||||
[26641, 26647, 26669, 26681, 26683, 26687, 26693, 26699, 26701, 26711],
|
||||
[26713, 26717, 26723, 26729, 26731, 26737, 26759, 26777, 26783, 26801],
|
||||
[26813, 26821, 26833, 26839, 26849, 26861, 26863, 26879, 26881, 26891],
|
||||
[26893, 26903, 26921, 26927, 26947, 26951, 26953, 26959, 26981, 26987],
|
||||
[26993, 27011, 27017, 27031, 27043, 27059, 27061, 27067, 27073, 27077],
|
||||
[27091, 27103, 27107, 27109, 27127, 27143, 27179, 27191, 27197, 27211],
|
||||
[27239, 27241, 27253, 27259, 27271, 27277, 27281, 27283, 27299, 27329],
|
||||
[27337, 27361, 27367, 27397, 27407, 27409, 27427, 27431, 27437, 27449],
|
||||
[27457, 27479, 27481, 27487, 27509, 27527, 27529, 27539, 27541, 27551],
|
||||
[27581, 27583, 27611, 27617, 27631, 27647, 27653, 27673, 27689, 27691],
|
||||
[27697, 27701, 27733, 27737, 27739, 27743, 27749, 27751, 27763, 27767],
|
||||
[27773, 27779, 27791, 27793, 27799, 27803, 27809, 27817, 27823, 27827],
|
||||
[27847, 27851, 27883, 27893, 27901, 27917, 27919, 27941, 27943, 27947],
|
||||
[27953, 27961, 27967, 27983, 27997, 28001, 28019, 28027, 28031, 28051],
|
||||
[28057, 28069, 28081, 28087, 28097, 28099, 28109, 28111, 28123, 28151],
|
||||
[28163, 28181, 28183, 28201, 28211, 28219, 28229, 28277, 28279, 28283],
|
||||
[28289, 28297, 28307, 28309, 28319, 28349, 28351, 28387, 28393, 28403],
|
||||
[28409, 28411, 28429, 28433, 28439, 28447, 28463, 28477, 28493, 28499],
|
||||
[28513, 28517, 28537, 28541, 28547, 28549, 28559, 28571, 28573, 28579],
|
||||
[28591, 28597, 28603, 28607, 28619, 28621, 28627, 28631, 28643, 28649],
|
||||
[28657, 28661, 28663, 28669, 28687, 28697, 28703, 28711, 28723, 28729],
|
||||
[28751, 28753, 28759, 28771, 28789, 28793, 28807, 28813, 28817, 28837],
|
||||
[28843, 28859, 28867, 28871, 28879, 28901, 28909, 28921, 28927, 28933],
|
||||
[28949, 28961, 28979, 29009, 29017, 29021, 29023, 29027, 29033, 29059],
|
||||
[29063, 29077, 29101, 29123, 29129, 29131, 29137, 29147, 29153, 29167],
|
||||
[29173, 29179, 29191, 29201, 29207, 29209, 29221, 29231, 29243, 29251],
|
||||
[29269, 29287, 29297, 29303, 29311, 29327, 29333, 29339, 29347, 29363],
|
||||
[29383, 29387, 29389, 29399, 29401, 29411, 29423, 29429, 29437, 29443],
|
||||
[29453, 29473, 29483, 29501, 29527, 29531, 29537, 29567, 29569, 29573],
|
||||
[29581, 29587, 29599, 29611, 29629, 29633, 29641, 29663, 29669, 29671],
|
||||
[29683, 29717, 29723, 29741, 29753, 29759, 29761, 29789, 29803, 29819],
|
||||
[29833, 29837, 29851, 29863, 29867, 29873, 29879, 29881, 29917, 29921],
|
||||
[29927, 29947, 29959, 29983, 29989, 30011, 30013, 30029, 30047, 30059],
|
||||
[30071, 30089, 30091, 30097, 30103, 30109, 30113, 30119, 30133, 30137],
|
||||
[30139, 30161, 30169, 30181, 30187, 30197, 30203, 30211, 30223, 30241],
|
||||
[30253, 30259, 30269, 30271, 30293, 30307, 30313, 30319, 30323, 30341],
|
||||
[30347, 30367, 30389, 30391, 30403, 30427, 30431, 30449, 30467, 30469],
|
||||
[30491, 30493, 30497, 30509, 30517, 30529, 30539, 30553, 30557, 30559],
|
||||
[30577, 30593, 30631, 30637, 30643, 30649, 30661, 30671, 30677, 30689],
|
||||
[30697, 30703, 30707, 30713, 30727, 30757, 30763, 30773, 30781, 30803],
|
||||
[30809, 30817, 30829, 30839, 30841, 30851, 30853, 30859, 30869, 30871],
|
||||
[30881, 30893, 30911, 30931, 30937, 30941, 30949, 30971, 30977, 30983],
|
||||
[31013, 31019, 31033, 31039, 31051, 31063, 31069, 31079, 31081, 31091],
|
||||
[31121, 31123, 31139, 31147, 31151, 31153, 31159, 31177, 31181, 31189],
|
||||
[31193, 31219, 31223, 31231, 31237, 31247, 31249, 31253, 31259, 31267],
|
||||
[31271, 31277, 31307, 31319, 31321, 31327, 31333, 31337, 31357, 31379],
|
||||
[31387, 31391, 31393, 31397, 31469, 31477, 31481, 31489, 31511, 31513],
|
||||
[31517, 31531, 31541, 31543, 31547, 31567, 31573, 31583, 31601, 31607],
|
||||
[31627, 31643, 31649, 31657, 31663, 31667, 31687, 31699, 31721, 31723],
|
||||
[31727, 31729, 31741, 31751, 31769, 31771, 31793, 31799, 31817, 31847],
|
||||
[31849, 31859, 31873, 31883, 31891, 31907, 31957, 31963, 31973, 31981],
|
||||
[31991, 32003, 32009, 32027, 32029, 32051, 32057, 32059, 32063, 32069],
|
||||
[32077, 32083, 32089, 32099, 32117, 32119, 32141, 32143, 32159, 32173],
|
||||
[32183, 32189, 32191, 32203, 32213, 32233, 32237, 32251, 32257, 32261],
|
||||
[32297, 32299, 32303, 32309, 32321, 32323, 32327, 32341, 32353, 32359],
|
||||
[32363, 32369, 32371, 32377, 32381, 32401, 32411, 32413, 32423, 32429],
|
||||
[32441, 32443, 32467, 32479, 32491, 32497, 32503, 32507, 32531, 32533],
|
||||
[32537, 32561, 32563, 32569, 32573, 32579, 32587, 32603, 32609, 32611],
|
||||
[32621, 32633, 32647, 32653, 32687, 32693, 32707, 32713, 32717, 32719],
|
||||
[32749, 32771, 32779, 32783, 32789, 32797, 32801, 32803, 32831, 32833],
|
||||
[32839, 32843, 32869, 32887, 32909, 32911, 32917, 32933, 32939, 32941],
|
||||
[32957, 32969, 32971, 32983, 32987, 32993, 32999, 33013, 33023, 33029],
|
||||
[33037, 33049, 33053, 33071, 33073, 33083, 33091, 33107, 33113, 33119],
|
||||
[33149, 33151, 33161, 33179, 33181, 33191, 33199, 33203, 33211, 33223],
|
||||
[33247, 33287, 33289, 33301, 33311, 33317, 33329, 33331, 33343, 33347],
|
||||
[33349, 33353, 33359, 33377, 33391, 33403, 33409, 33413, 33427, 33457],
|
||||
[33461, 33469, 33479, 33487, 33493, 33503, 33521, 33529, 33533, 33547],
|
||||
[33563, 33569, 33577, 33581, 33587, 33589, 33599, 33601, 33613, 33617],
|
||||
[33619, 33623, 33629, 33637, 33641, 33647, 33679, 33703, 33713, 33721],
|
||||
[33739, 33749, 33751, 33757, 33767, 33769, 33773, 33791, 33797, 33809],
|
||||
[33811, 33827, 33829, 33851, 33857, 33863, 33871, 33889, 33893, 33911],
|
||||
[33923, 33931, 33937, 33941, 33961, 33967, 33997, 34019, 34031, 34033],
|
||||
[34039, 34057, 34061, 34123, 34127, 34129, 34141, 34147, 34157, 34159],
|
||||
[34171, 34183, 34211, 34213, 34217, 34231, 34253, 34259, 34261, 34267],
|
||||
[34273, 34283, 34297, 34301, 34303, 34313, 34319, 34327, 34337, 34351],
|
||||
[34361, 34367, 34369, 34381, 34403, 34421, 34429, 34439, 34457, 34469],
|
||||
[34471, 34483, 34487, 34499, 34501, 34511, 34513, 34519, 34537, 34543],
|
||||
[34549, 34583, 34589, 34591, 34603, 34607, 34613, 34631, 34649, 34651],
|
||||
[34667, 34673, 34679, 34687, 34693, 34703, 34721, 34729, 34739, 34747],
|
||||
[34757, 34759, 34763, 34781, 34807, 34819, 34841, 34843, 34847, 34849],
|
||||
[34871, 34877, 34883, 34897, 34913, 34919, 34939, 34949, 34961, 34963],
|
||||
[34981, 35023, 35027, 35051, 35053, 35059, 35069, 35081, 35083, 35089],
|
||||
[35099, 35107, 35111, 35117, 35129, 35141, 35149, 35153, 35159, 35171],
|
||||
[35201, 35221, 35227, 35251, 35257, 35267, 35279, 35281, 35291, 35311],
|
||||
[35317, 35323, 35327, 35339, 35353, 35363, 35381, 35393, 35401, 35407],
|
||||
[35419, 35423, 35437, 35447, 35449, 35461, 35491, 35507, 35509, 35521],
|
||||
[35527, 35531, 35533, 35537, 35543, 35569, 35573, 35591, 35593, 35597],
|
||||
[35603, 35617, 35671, 35677, 35729, 35731, 35747, 35753, 35759, 35771],
|
||||
[35797, 35801, 35803, 35809, 35831, 35837, 35839, 35851, 35863, 35869],
|
||||
[35879, 35897, 35899, 35911, 35923, 35933, 35951, 35963, 35969, 35977],
|
||||
[35983, 35993, 35999, 36007, 36011, 36013, 36017, 36037, 36061, 36067],
|
||||
[36073, 36083, 36097, 36107, 36109, 36131, 36137, 36151, 36161, 36187],
|
||||
[36191, 36209, 36217, 36229, 36241, 36251, 36263, 36269, 36277, 36293],
|
||||
[36299, 36307, 36313, 36319, 36341, 36343, 36353, 36373, 36383, 36389],
|
||||
[36433, 36451, 36457, 36467, 36469, 36473, 36479, 36493, 36497, 36523],
|
||||
[36527, 36529, 36541, 36551, 36559, 36563, 36571, 36583, 36587, 36599],
|
||||
[36607, 36629, 36637, 36643, 36653, 36671, 36677, 36683, 36691, 36697],
|
||||
[36709, 36713, 36721, 36739, 36749, 36761, 36767, 36779, 36781, 36787],
|
||||
[36791, 36793, 36809, 36821, 36833, 36847, 36857, 36871, 36877, 36887],
|
||||
[36899, 36901, 36913, 36919, 36923, 36929, 36931, 36943, 36947, 36973],
|
||||
[36979, 36997, 37003, 37013, 37019, 37021, 37039, 37049, 37057, 37061],
|
||||
[37087, 37097, 37117, 37123, 37139, 37159, 37171, 37181, 37189, 37199],
|
||||
[37201, 37217, 37223, 37243, 37253, 37273, 37277, 37307, 37309, 37313],
|
||||
[37321, 37337, 37339, 37357, 37361, 37363, 37369, 37379, 37397, 37409],
|
||||
[37423, 37441, 37447, 37463, 37483, 37489, 37493, 37501, 37507, 37511],
|
||||
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
|
||||
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
|
||||
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
|
||||
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
|
||||
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039]]}}]}
|
||||
==============================================================================
|
||||
|
||||
================================================================================
|
||||
MERGE RESULT: ✅ SUCCESS
|
||||
================================================================================
|
||||
Final result length: 31743 chars
|
||||
Final result (COMPLETE):
|
||||
================================================================================
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"type": "table",
|
||||
"content": {
|
||||
"headers": ["Spalte1", "Spalte2", "Spalte3", "Spalte4", "Spalte5", "Spalte6", "Spalte7", "Spalte8", "Spalte9", "Spalte10"],
|
||||
"rows": [
|
||||
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29],
|
||||
[31, 37, 41, 43, 47, 53, 59, 61, 67, 71],
|
||||
[73, 79, 83, 89, 97, 101, 103, 107, 109, 113],
|
||||
[127, 131, 137, 139, 149, 151, 157, 163, 167, 173],
|
||||
[179, 181, 191, 193, 197, 199, 211, 223, 227, 229],
|
||||
[233, 239, 241, 251, 257, 263, 269, 271, 277, 281],
|
||||
[283, 293, 307, 311, 313, 317, 331, 337, 347, 349],
|
||||
[353, 359, 367, 373, 379, 383, 389, 397, 401, 409],
|
||||
[419, 421, 431, 433, 439, 443, 449, 457, 461, 463],
|
||||
[467, 479, 487, 491, 499, 503, 509, 521, 523, 541],
|
||||
[547, 557, 563, 569, 571, 577, 587, 593, 599, 601],
|
||||
[607, 613, 617, 619, 631, 641, 643, 647, 653, 659],
|
||||
[661, 673, 677, 683, 691, 701, 709, 719, 727, 733],
|
||||
[739, 743, 751, 757, 761, 769, 773, 787, 797, 809],
|
||||
[811, 821, 823, 827, 829, 839, 853, 857, 859, 863],
|
||||
[877, 881, 883, 887, 907, 911, 919, 929, 937, 941],
|
||||
[947, 953, 967, 971, 977, 983, 991, 997, 1009, 1013],
|
||||
[1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069],
|
||||
[1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151],
|
||||
[1153, 1163, 1171, 1181, 1187, 1193, 1201, 1213, 1217, 1223],
|
||||
[1229, 1231, 1237, 1249, 1259, 1277, 1279, 1283, 1289, 1291],
|
||||
[1297, 1301, 1303, 1307, 1319, 1321, 1327, 1361, 1367, 1373],
|
||||
[1381, 1399, 1409, 1423, 1427, 1429, 1433, 1439, 1447, 1451],
|
||||
[1453, 1459, 1471, 1481, 1483, 1487, 1489, 1493, 1499, 1511],
|
||||
[1523, 1531, 1543, 1549, 1553, 1559, 1567, 1571, 1579, 1583],
|
||||
[1597, 1601, 1607, 1609, 1613, 1619, 1621, 1627, 1637, 1657],
|
||||
[1663, 1667, 1669, 1693, 1697, 1699, 1709, 1721, 1723, 1733],
|
||||
[1741, 1747, 1753, 1759, 1777, 1783, 1787, 1789, 1801, 1811],
|
||||
[1823, 1831, 1847, 1861, 1867, 1871, 1873, 1877, 1879, 1889],
|
||||
[1901, 1907, 1913, 1931, 1933, 1949, 1951, 1973, 1979, 1987],
|
||||
[1993, 1997, 1999, 2003, 2011, 2017, 2027, 2029, 2039, 2053],
|
||||
[2063, 2069, 2081, 2083, 2087, 2089, 2099, 2111, 2113, 2129],
|
||||
[2131, 2137, 2141, 2143, 2153, 2161, 2179, 2203, 2207, 2213],
|
||||
[2221, 2237, 2239, 2243, 2251, 2267, 2269, 2273, 2281, 2287],
|
||||
[2293, 2297, 2309, 2311, 2333, 2339, 2341, 2347, 2351, 2357],
|
||||
[2371, 2377, 2381, 2383, 2389, 2393, 2399, 2411, 2417, 2423],
|
||||
[2437, 2441, 2447, 2459, 2467, 2473, 2477, 2503, 2521, 2531],
|
||||
[2539, 2543, 2549, 2551, 2557, 2579, 2591, 2593, 2609, 2617],
|
||||
[2621, 2633, 2647, 2657, 2659, 2663, 2671, 2677, 2683, 2687],
|
||||
[2689, 2693, 2699, 2707, 2711, 2713, 2719, 2729, 2731, 2741],
|
||||
[2749, 2753, 2767, 2777, 2789, 2791, 2797, 2801, 2803, 2819],
|
||||
[2833, 2837, 2843, 2851, 2857, 2861, 2879, 2887, 2897, 2903],
|
||||
[2909, 2917, 2927, 2939, 2953, 2957, 2963, 2969, 2971, 2999],
|
||||
[3001, 3011, 3019, 3023, 3037, 3041, 3049, 3061, 3067, 3079],
|
||||
[3083, 3089, 3109, 3119, 3121, 3137, 3163, 3167, 3169, 3181],
|
||||
[3187, 3191, 3203, 3209, 3217, 3221, 3229, 3251, 3253, 3257],
|
||||
[3259, 3271, 3299, 3301, 3307, 3313, 3319, 3323, 3329, 3331],
|
||||
[3343, 3347, 3359, 3361, 3371, 3373, 3389, 3391, 3407, 3413],
|
||||
[3433, 3449, 3457, 3461, 3463, 3467, 3469, 3491, 3499, 3511],
|
||||
[3517, 3527, 3529, 3533, 3539, 3541, 3547, 3557, 3559, 3571],
|
||||
[3581, 3583, 3593, 3607, 3613, 3617, 3623, 3631, 3637, 3643],
|
||||
[3659, 3671, 3673, 3677, 3691, 3697, 3701, 3709, 3719, 3727],
|
||||
[3733, 3739, 3761, 3767, 3769, 3779, 3793, 3797, 3803, 3821],
|
||||
[3823, 3833, 3847, 3851, 3853, 3863, 3877, 3881, 3889, 3907],
|
||||
[3911, 3917, 3919, 3923, 3929, 3931, 3943, 3947, 3967, 3989],
|
||||
[4001, 4003, 4007, 4013, 4019, 4021, 4027, 4049, 4051, 4057],
|
||||
[4073, 4079, 4091, 4093, 4099, 4111, 4127, 4129, 4133, 4139],
|
||||
[4153, 4157, 4159, 4177, 4201, 4211, 4217, 4219, 4229, 4231],
|
||||
[4241, 4243, 4253, 4259, 4261, 4271, 4273, 4283, 4289, 4297],
|
||||
[4327, 4337, 4339, 4349, 4357, 4363, 4373, 4391, 4397, 4409],
|
||||
[4421, 4423, 4441, 4447, 4451, 4457, 4463, 4481, 4483, 4493],
|
||||
[4507, 4513, 4517, 4519, 4523, 4547, 4549, 4561, 4567, 4583],
|
||||
[4591, 4597, 4603, 4621, 4637, 4639, 4643, 4649, 4651, 4657],
|
||||
[4663, 4673, 4679, 4691, 4703, 4721, 4723, 4729, 4733, 4751],
|
||||
[4759, 4783, 4787, 4789, 4793, 4799, 4801, 4813, 4817, 4831],
|
||||
[4861, 4871, 4877, 4889, 4903, 4909, 4919, 4931, 4933, 4937],
|
||||
[4943, 4951, 4957, 4967, 4969, 4973, 4987, 4993, 4999, 5003],
|
||||
[5009, 5011, 5021, 5023, 5039, 5051, 5059, 5077, 5081, 5087],
|
||||
[5099, 5101, 5107, 5113, 5119, 5147, 5153, 5167, 5171, 5179],
|
||||
[5189, 5197, 5209, 5227, 5231, 5233, 5237, 5261, 5273, 5279],
|
||||
[5281, 5297, 5303, 5309, 5323, 5333, 5347, 5351, 5381, 5387],
|
||||
[5393, 5399, 5407, 5413, 5417, 5419, 5431, 5437, 5441, 5443],
|
||||
[5449, 5471, 5477, 5479, 5483, 5501, 5503, 5507, 5519, 5521],
|
||||
[5527, 5531, 5557, 5563, 5569, 5573, 5581, 5591, 5623, 5639],
|
||||
[5641, 5647, 5651, 5653, 5657, 5659, 5669, 5683, 5689, 5693],
|
||||
[5701, 5711, 5717, 5737, 5741, 5743, 5749, 5779, 5783, 5791],
|
||||
[5801, 5807, 5813, 5821, 5827, 5839, 5843, 5849, 5851, 5857],
|
||||
[5861, 5867, 5869, 5879, 5881, 5897, 5903, 5923, 5927, 5939],
|
||||
[5953, 5981, 5987, 6007, 6011, 6029, 6037, 6043, 6047, 6053],
|
||||
[6067, 6073, 6079, 6089, 6091, 6101, 6113, 6121, 6131, 6133],
|
||||
[6143, 6151, 6163, 6173, 6197, 6199, 6203, 6211, 6217, 6221],
|
||||
[6229, 6247, 6257, 6263, 6269, 6271, 6277, 6287, 6299, 6301],
|
||||
[6311, 6317, 6323, 6329, 6337, 6343, 6353, 6359, 6361, 6367],
|
||||
[6373, 6379, 6389, 6397, 6421, 6427, 6449, 6451, 6469, 6473],
|
||||
[6481, 6491, 6521, 6529, 6547, 6551, 6553, 6563, 6569, 6571],
|
||||
[6577, 6581, 6599, 6607, 6619, 6637, 6653, 6659, 6661, 6673],
|
||||
[6679, 6689, 6691, 6701, 6703, 6709, 6719, 6733, 6737, 6761],
|
||||
[6763, 6779, 6781, 6791, 6793, 6803, 6823, 6827, 6829, 6833],
|
||||
[6841, 6857, 6863, 6869, 6871, 6883, 6899, 6907, 6911, 6917],
|
||||
[6947, 6949, 6959, 6961, 6967, 6971, 6977, 6983, 6991, 6997],
|
||||
[7001, 7013, 7019, 7027, 7039, 7043, 7057, 7069, 7079, 7103],
|
||||
[7109, 7121, 7127, 7129, 7151, 7159, 7177, 7187, 7193, 7207],
|
||||
[7211, 7213, 7219, 7229, 7237, 7243, 7247, 7253, 7283, 7297],
|
||||
[7307, 7309, 7321, 7331, 7333, 7349, 7351, 7369, 7393, 7411],
|
||||
[7417, 7433, 7451, 7457, 7459, 7477, 7481, 7487, 7489, 7499],
|
||||
[7507, 7517, 7523, 7529, 7537, 7541, 7547, 7549, 7559, 7561],
|
||||
[7573, 7577, 7583, 7589, 7591, 7603, 7607, 7621, 7639, 7643],
|
||||
[7649, 7669, 7673, 7681, 7687, 7691, 7699, 7703, 7717, 7723],
|
||||
[7727, 7741, 7753, 7757, 7759, 7789, 7793, 7817, 7823, 7829],
|
||||
[7841, 7853, 7867, 7873, 7877, 7879, 7883, 7901, 7907, 7919],
|
||||
[7927, 7933, 7937, 7949, 7951, 7963, 7993, 8009, 8011, 8017],
|
||||
[8039, 8053, 8059, 8069, 8081, 8087, 8089, 8093, 8101, 8111],
|
||||
[8117, 8123, 8147, 8161, 8167, 8171, 8179, 8191, 8209, 8219],
|
||||
[8221, 8231, 8233, 8237, 8243, 8263, 8269, 8273, 8287, 8291],
|
||||
[8293, 8297, 8311, 8317, 8329, 8353, 8363, 8369, 8377, 8387],
|
||||
[8389, 8419, 8423, 8429, 8431, 8443, 8447, 8461, 8467, 8501],
|
||||
[8513, 8521, 8527, 8537, 8539, 8543, 8563, 8573, 8581, 8597],
|
||||
[8599, 8609, 8623, 8627, 8629, 8641, 8647, 8663, 8669, 8677],
|
||||
[8681, 8689, 8693, 8699, 8707, 8713, 8719, 8731, 8737, 8741],
|
||||
[8747, 8753, 8761, 8779, 8783, 8803, 8807, 8819, 8821, 8831],
|
||||
[8837, 8839, 8849, 8861, 8863, 8867, 8887, 8893, 8923, 8929],
|
||||
[8933, 8941, 8951, 8963, 8969, 8971, 8999, 9001, 9007, 9011],
|
||||
[9013, 9029, 9041, 9043, 9049, 9059, 9067, 9091, 9103, 9109],
|
||||
[9127, 9133, 9137, 9151, 9157, 9161, 9173, 9181, 9187, 9199],
|
||||
[9203, 9209, 9221, 9227, 9239, 9241, 9257, 9277, 9281, 9283],
|
||||
[9293, 9311, 9319, 9323, 9337, 9341, 9343, 9349, 9371, 9377],
|
||||
[9391, 9397, 9403, 9413, 9419, 9421, 9431, 9433, 9437, 9439],
|
||||
[9461, 9463, 9467, 9473, 9479, 9491, 9497, 9511, 9521, 9533],
|
||||
[9539, 9547, 9551, 9587, 9601, 9613, 9619, 9623, 9629, 9631],
|
||||
[9643, 9649, 9661, 9677, 9679, 9689, 9697, 9719, 9721, 9733],
|
||||
[9739, 9743, 9749, 9767, 9769, 9781, 9787, 9791, 9803, 9811],
|
||||
[9817, 9829, 9833, 9839, 9851, 9857, 9859, 9871, 9883, 9887],
|
||||
[9901, 9907, 9923, 9929, 9931, 9941, 9949, 9967, 9973, 10007],
|
||||
[10009, 10037, 10039, 10061, 10067, 10069, 10079, 10091, 10093, 10099],
|
||||
[10103, 10111, 10133, 10139, 10141, 10151, 10159, 10163, 10169, 10177],
|
||||
[10181, 10193, 10211, 10223, 10243, 10247, 10253, 10259, 10267, 10271],
|
||||
[10273, 10289, 10301, 10303, 10313, 10321, 10331, 10333, 10337, 10343],
|
||||
[10357, 10369, 10391, 10399, 10427, 10429, 10433, 10453, 10457, 10459],
|
||||
[10463, 10477, 10487, 10499, 10501, 10513, 10529, 10531, 10559, 10567],
|
||||
[10589, 10597, 10601, 10607, 10613, 10627, 10631, 10639, 10651, 10657],
|
||||
[10663, 10667, 10687, 10691, 10709, 10711, 10723, 10729, 10733, 10739],
|
||||
[10753, 10771, 10781, 10789, 10799, 10831, 10837, 10847, 10853, 10859],
|
||||
[10861, 10867, 10883, 10889, 10891, 10903, 10909, 10937, 10939, 10949],
|
||||
[10957, 10973, 10979, 10987, 10993, 11003, 11027, 11047, 11057, 11059],
|
||||
[11069, 11071, 11083, 11087, 11093, 11113, 11117, 11119, 11131, 11149],
|
||||
[11159, 11161, 11171, 11173, 11177, 11197, 11213, 11239, 11243, 11251],
|
||||
[11257, 11261, 11273, 11279, 11287, 11299, 11311, 11317, 11321, 11329],
|
||||
[11351, 11353, 11369, 11383, 11393, 11399, 11411, 11423, 11437, 11443],
|
||||
[11447, 11467, 11471, 11483, 11489, 11491, 11497, 11503, 11519, 11527],
|
||||
[11549, 11551, 11579, 11587, 11593, 11597, 11617, 11621, 11633, 11657],
|
||||
[11677, 11681, 11689, 11699, 11701, 11717, 11719, 11731, 11743, 11777],
|
||||
[11779, 11783, 11789, 11801, 11807, 11813, 11821, 11827, 11831, 11833],
|
||||
[11839, 11863, 11867, 11887, 11897, 11903, 11909, 11923, 11927, 11933],
|
||||
[11939, 11941, 11953, 11959, 11969, 11971, 11981, 11987, 12007, 12011],
|
||||
[12037, 12041, 12043, 12049, 12071, 12073, 12097, 12101, 12107, 12109],
|
||||
[12113, 12119, 12143, 12149, 12157, 12161, 12163, 12197, 12203, 12211],
|
||||
[12227, 12239, 12241, 12251, 12253, 12263, 12269, 12277, 12281, 12289],
|
||||
[12301, 12323, 12329, 12343, 12347, 12373, 12377, 12379, 12391, 12401],
|
||||
[12409, 12413, 12421, 12433, 12437, 12451, 12457, 12473, 12479, 12487],
|
||||
[12491, 12497, 12503, 12511, 12517, 12527, 12539, 12541, 12547, 12553],
|
||||
[12569, 12577, 12583, 12589, 12601, 12611, 12613, 12619, 12637, 12641],
|
||||
[12647, 12653, 12659, 12671, 12689, 12697, 12703, 12713, 12721, 12739],
|
||||
[12743, 12757, 12763, 12781, 12791, 12799, 12809, 12821, 12823, 12829],
|
||||
[12841, 12853, 12889, 12893, 12899, 12907, 12911, 12917, 12919, 12923],
|
||||
[12941, 12953, 12959, 12967, 12973, 12979, 12983, 13001, 13003, 13007],
|
||||
[13009, 13033, 13037, 13043, 13049, 13063, 13093, 13099, 13103, 13109],
|
||||
[13121, 13127, 13147, 13151, 13159, 13163, 13171, 13177, 13183, 13187],
|
||||
[13217, 13219, 13229, 13241, 13249, 13259, 13267, 13291, 13297, 13309],
|
||||
[13313, 13327, 13331, 13337, 13339, 13367, 13381, 13397, 13399, 13411],
|
||||
[13417, 13421, 13441, 13451, 13457, 13463, 13469, 13477, 13487, 13499],
|
||||
[13513, 13523, 13537, 13553, 13567, 13577, 13591, 13597, 13613, 13619],
|
||||
[13627, 13633, 13649, 13669, 13679, 13681, 13687, 13691, 13693, 13697],
|
||||
[13709, 13711, 13721, 13723, 13729, 13751, 13757, 13759, 13763, 13781],
|
||||
[13789, 13799, 13807, 13829, 13831, 13841, 13859, 13873, 13877, 13879],
|
||||
[13883, 13901, 13903, 13907, 13913, 13921, 13931, 13933, 13963, 13967],
|
||||
[13997, 13999, 14009, 14011, 14029, 14033, 14051, 14057, 14071, 14081],
|
||||
[14083, 14087, 14107, 14143, 14149, 14153, 14159, 14173, 14177, 14197],
|
||||
[14207, 14221, 14243, 14249, 14251, 14281, 14293, 14303, 14321, 14323],
|
||||
[14327, 14341, 14347, 14369, 14387, 14389, 14401, 14407, 14411, 14419],
|
||||
[14423, 14431, 14437, 14447, 14449, 14461, 14479, 14489, 14503, 14519],
|
||||
[14533, 14537, 14543, 14549, 14551, 14557, 14561, 14563, 14591, 14593],
|
||||
[14621, 14627, 14629, 14633, 14639, 14653, 14657, 14669, 14683, 14699],
|
||||
[14713, 14717, 14723, 14731, 14737, 14741, 14747, 14753, 14759, 14767],
|
||||
[14771, 14779, 14783, 14797, 14813, 14821, 14827, 14831, 14843, 14851],
|
||||
[14867, 14869, 14879, 14887, 14891, 14897, 14923, 14929, 14939, 14947],
|
||||
[14951, 14957, 14969, 14983, 15013, 15017, 15031, 15053, 15061, 15073],
|
||||
[15077, 15083, 15091, 15101, 15107, 15121, 15131, 15137, 15139, 15149],
|
||||
[15161, 15173, 15187, 15193, 15199, 15217, 15227, 15233, 15241, 15259],
|
||||
[15263, 15269, 15271, 15277, 15287, 15289, 15299, 15307, 15313, 15319],
|
||||
[15329, 15331, 15349, 15359, 15361, 15373, 15377, 15383, 15391, 15401],
|
||||
[15413, 15427, 15439, 15443, 15451, 15461, 15467, 15473, 15493, 15497],
|
||||
[15511, 15527, 15541, 15551, 15559, 15569, 15581, 15583, 15601, 15607],
|
||||
[15619, 15629, 15641, 15643, 15647, 15649, 15661, 15667, 15671, 15679],
|
||||
[15683, 15727, 15731, 15733, 15737, 15739, 15749, 15761, 15767, 15773],
|
||||
[15787, 15791, 15797, 15803, 15809, 15817, 15823, 15859, 15877, 15881],
|
||||
[15887, 15889, 15901, 15907, 15913, 15919, 15923, 15937, 15959, 15971],
|
||||
[15973, 15991, 16001, 16007, 16033, 16057, 16061, 16063, 16067, 16069],
|
||||
[16073, 16087, 16091, 16097, 16103, 16111, 16127, 16139, 16141, 16183],
|
||||
[16187, 16189, 16193, 16217, 16223, 16229, 16231, 16249, 16253, 16267],
|
||||
[16273, 16301, 16319, 16333, 16339, 16349, 16361, 16363, 16369, 16381],
|
||||
[16411, 16417, 16421, 16427, 16433, 16447, 16451, 16453, 16477, 16481],
|
||||
[16487, 16493, 16519, 16529, 16547, 16553, 16561, 16567, 16573, 16603],
|
||||
[16607, 16619, 16631, 16633, 16649, 16651, 16657, 16661, 16673, 16691],
|
||||
[16693, 16699, 16703, 16729, 16741, 16747, 16759, 16763, 16787, 16811],
|
||||
[16823, 16829, 16831, 16843, 16871, 16879, 16883, 16889, 16901, 16903],
|
||||
[16921, 16927, 16931, 16937, 16943, 16963, 16979, 16981, 16987, 16993],
|
||||
[17011, 17021, 17027, 17029, 17033, 17041, 17047, 17053, 17077, 17093],
|
||||
[17099, 17107, 17117, 17123, 17137, 17159, 17167, 17183, 17189, 17191],
|
||||
[17203, 17207, 17209, 17231, 17239, 17257, 17291, 17293, 17299, 17317],
|
||||
[17321, 17327, 17333, 17341, 17351, 17359, 17377, 17383, 17387, 17389],
|
||||
[17393, 17401, 17417, 17419, 17431, 17443, 17449, 17467, 17471, 17477],
|
||||
[17483, 17489, 17491, 17497, 17509, 17519, 17539, 17551, 17569, 17573],
|
||||
[17579, 17581, 17597, 17599, 17609, 17623, 17627, 17657, 17659, 17669],
|
||||
[17681, 17683, 17707, 17713, 17729, 17737, 17747, 17749, 17761, 17783],
|
||||
[17789, 17791, 17807, 17827, 17837, 17839, 17851, 17863, 17881, 17891],
|
||||
[17903, 17909, 17911, 17921, 17923, 17929, 17939, 17957, 17959, 17971],
|
||||
[17977, 17981, 17987, 17989, 18013, 18041, 18043, 18047, 18049, 18059],
|
||||
[18061, 18077, 18089, 18097, 18119, 18121, 18127, 18131, 18133, 18143],
|
||||
[18149, 18169, 18181, 18191, 18199, 18211, 18217, 18223, 18229, 18233],
|
||||
[18251, 18253, 18257, 18269, 18287, 18289, 18301, 18307, 18311, 18313],
|
||||
[18329, 18341, 18353, 18367, 18371, 18379, 18397, 18401, 18413, 18427],
|
||||
[18433, 18439, 18443, 18451, 18457, 18461, 18481, 18493, 18503, 18517],
|
||||
[18521, 18523, 18539, 18541, 18553, 18583, 18587, 18593, 18617, 18637],
|
||||
[18661, 18671, 18679, 18691, 18701, 18713, 18719, 18731, 18743, 18749],
|
||||
[18757, 18773, 18787, 18793, 18797, 18803, 18839, 18859, 18869, 18899],
|
||||
[18911, 18913, 18917, 18919, 18947, 18959, 18973, 18979, 19001, 19009],
|
||||
[19013, 19031, 19037, 19051, 19069, 19073, 19079, 19081, 19087, 19121],
|
||||
[19139, 19141, 19157, 19163, 19181, 19183, 19207, 19211, 19213, 19219],
|
||||
[19231, 19237, 19249, 19259, 19267, 19273, 19289, 19301, 19309, 19319],
|
||||
[19333, 19373, 19379, 19381, 19387, 19391, 19403, 19417, 19421, 19423],
|
||||
[19427, 19429, 19433, 19441, 19447, 19457, 19463, 19469, 19471, 19477],
|
||||
[19483, 19489, 19501, 19507, 19531, 19541, 19543, 19553, 19559, 19571],
|
||||
[19577, 19583, 19597, 19603, 19609, 19661, 19681, 19687, 19697, 19699],
|
||||
[19709, 19717, 19727, 19739, 19751, 19753, 19759, 19763, 19777, 19793],
|
||||
[19801, 19813, 19819, 19841, 19843, 19853, 19861, 19867, 19889, 19891],
|
||||
[19913, 19919, 19927, 19937, 19949, 19961, 19963, 19973, 19979, 19991],
|
||||
[19993, 19997, 20011, 20021, 20023, 20029, 20047, 20051, 20063, 20071],
|
||||
[20089, 20101, 20107, 20113, 20117, 20123, 20129, 20143, 20147, 20149],
|
||||
[20161, 20173, 20177, 20183, 20201, 20219, 20231, 20233, 20249, 20261],
|
||||
[20269, 20287, 20297, 20323, 20327, 20333, 20341, 20347, 20353, 20357],
|
||||
[20359, 20369, 20389, 20393, 20399, 20407, 20411, 20431, 20441, 20443],
|
||||
[20477, 20479, 20483, 20507, 20509, 20521, 20533, 20543, 20549, 20551],
|
||||
[20563, 20593, 20599, 20611, 20627, 20639, 20641, 20663, 20681, 20693],
|
||||
[20707, 20717, 20719, 20731, 20743, 20747, 20749, 20753, 20759, 20771],
|
||||
[20773, 20789, 20807, 20809, 20849, 20857, 20873, 20879, 20887, 20897],
|
||||
[20899, 20903, 20921, 20929, 20939, 20947, 20959, 20963, 20981, 20983],
|
||||
[21001, 21011, 21013, 21017, 21019, 21023, 21031, 21059, 21061, 21067],
|
||||
[21089, 21101, 21107, 21121, 21139, 21143, 21149, 21157, 21163, 21169],
|
||||
[21179, 21187, 21191, 21193, 21211, 21221, 21227, 21247, 21269, 21277],
|
||||
[21283, 21313, 21317, 21319, 21323, 21341, 21347, 21377, 21379, 21383],
|
||||
[21391, 21397, 21401, 21407, 21419, 21433, 21467, 21481, 21487, 21491],
|
||||
[21493, 21499, 21503, 21517, 21521, 21523, 21529, 21557, 21559, 21563],
|
||||
[21569, 21577, 21587, 21589, 21599, 21601, 21611, 21613, 21617, 21647],
|
||||
[21649, 21661, 21673, 21683, 21701, 21713, 21727, 21737, 21739, 21751],
|
||||
[21757, 21767, 21773, 21787, 21799, 21803, 21817, 21821, 21839, 21841],
|
||||
[21851, 21859, 21863, 21871, 21881, 21893, 21911, 21929, 21937, 21943],
|
||||
[21961, 21977, 21991, 21997, 22003, 22013, 22027, 22031, 22037, 22039],
|
||||
[22051, 22063, 22067, 22073, 22079, 22091, 22093, 22109, 22111, 22123],
|
||||
[22129, 22133, 22147, 22153, 22157, 22159, 22171, 22189, 22193, 22229],
|
||||
[22247, 22259, 22271, 22273, 22277, 22279, 22283, 22291, 22303, 22307],
|
||||
[22343, 22349, 22367, 22369, 22381, 22391, 22397, 22409, 22433, 22441],
|
||||
[22447, 22453, 22469, 22481, 22483, 22501, 22511, 22531, 22541, 22543],
|
||||
[22549, 22567, 22571, 22573, 22613, 22619, 22621, 22637, 22639, 22643],
|
||||
[22651, 22669, 22679, 22691, 22697, 22699, 22709, 22717, 22721, 22727],
|
||||
[22739, 22741, 22751, 22769, 22777, 22783, 22787, 22807, 22811, 22817],
|
||||
[22853, 22859, 22861, 22871, 22877, 22901, 22907, 22921, 22937, 22943],
|
||||
[22961, 22963, 22973, 22993, 23003, 23011, 23017, 23021, 23027, 23029],
|
||||
[23039, 23041, 23053, 23057, 23059, 23063, 23071, 23081, 23087, 23099],
|
||||
[23117, 23131, 23143, 23159, 23167, 23173, 23189, 23197, 23201, 23203],
|
||||
[23209, 23227, 23251, 23269, 23279, 23291, 23293, 23297, 23311, 23321],
|
||||
[23327, 23333, 23339, 23357, 23369, 23371, 23399, 23417, 23431, 23447],
|
||||
[23459, 23473, 23497, 23509, 23531, 23537, 23539, 23549, 23557, 23561],
|
||||
[23563, 23567, 23581, 23593, 23599, 23603, 23609, 23623, 23627, 23629],
|
||||
[23633, 23663, 23669, 23671, 23677, 23687, 23689, 23719, 23741, 23743],
|
||||
[23747, 23753, 23761, 23767, 23773, 23789, 23801, 23813, 23819, 23827],
|
||||
[23831, 23833, 23857, 23869, 23873, 23879, 23887, 23893, 23899, 23909],
|
||||
[23911, 23917, 23929, 23957, 23971, 23977, 23981, 23993, 24001, 24007],
|
||||
[24019, 24023, 24029, 24043, 24049, 24061, 24071, 24077, 24083, 24091],
|
||||
[24097, 24103, 24107, 24109, 24113, 24121, 24133, 24137, 24151, 24169],
|
||||
[24179, 24181, 24197, 24203, 24223, 24229, 24239, 24247, 24251, 24281],
|
||||
[24317, 24329, 24337, 24359, 24371, 24373, 24379, 24391, 24407, 24413],
|
||||
[24419, 24421, 24439, 24443, 24469, 24473, 24481, 24499, 24509, 24517],
|
||||
[24527, 24533, 24547, 24551, 24571, 24593, 24611, 24623, 24631, 24659],
|
||||
[24671, 24677, 24683, 24691, 24697, 24709, 24733, 24749, 24763, 24767],
|
||||
[24781, 24793, 24799, 24809, 24821, 24841, 24847, 24851, 24859, 24877],
|
||||
[24889, 24907, 24917, 24919, 24923, 24943, 24953, 24967, 24971, 24977],
|
||||
[24979, 24989, 25013, 25031, 25033, 25037, 25057, 25073, 25087, 25097],
|
||||
[25111, 25117, 25121, 25127, 25147, 25153, 25163, 25169, 25171, 25183],
|
||||
[25189, 25219, 25229, 25237, 25243, 25247, 25253, 25261, 25301, 25303],
|
||||
[25307, 25309, 25321, 25339, 25343, 25349, 25357, 25367, 25373, 25391],
|
||||
[25409, 25411, 25423, 25439, 25447, 25453, 25457, 25463, 25469, 25471],
|
||||
[25523, 25537, 25541, 25561, 25577, 25579, 25583, 25589, 25601, 25603],
|
||||
[25609, 25621, 25633, 25639, 25643, 25657, 25667, 25673, 25679, 25693],
|
||||
[25703, 25717, 25733, 25741, 25747, 25759, 25763, 25771, 25793, 25799],
|
||||
[25801, 25819, 25841, 25847, 25849, 25867, 25873, 25889, 25903, 25913],
|
||||
[25919, 25931, 25933, 25939, 25943, 25951, 25969, 25981, 25997, 25999],
|
||||
[26003, 26017, 26021, 26029, 26041, 26053, 26083, 26099, 26107, 26111],
|
||||
[26113, 26119, 26141, 26153, 26161, 26171, 26177, 26183, 26189, 26203],
|
||||
[26209, 26227, 26237, 26249, 26251, 26261, 26263, 26267, 26293, 26297],
|
||||
[26309, 26317, 26321, 26339, 26347, 26357, 26371, 26387, 26393, 26399],
|
||||
[26407, 26417, 26423, 26431, 26437, 26449, 26459, 26479, 26489, 26497],
|
||||
[26501, 26513, 26539, 26557, 26561, 26573, 26591, 26597, 26627, 26633],
|
||||
[26641, 26647, 26669, 26681, 26683, 26687, 26693, 26699, 26701, 26711],
|
||||
[26713, 26717, 26723, 26729, 26731, 26737, 26759, 26777, 26783, 26801],
|
||||
[26813, 26821, 26833, 26839, 26849, 26861, 26863, 26879, 26881, 26891],
|
||||
[26893, 26903, 26921, 26927, 26947, 26951, 26953, 26959, 26981, 26987],
|
||||
[26993, 27011, 27017, 27031, 27043, 27059, 27061, 27067, 27073, 27077],
|
||||
[27091, 27103, 27107, 27109, 27127, 27143, 27179, 27191, 27197, 27211],
|
||||
[27239, 27241, 27253, 27259, 27271, 27277, 27281, 27283, 27299, 27329],
|
||||
[27337, 27361, 27367, 27397, 27407, 27409, 27427, 27431, 27437, 27449],
|
||||
[27457, 27479, 27481, 27487, 27509, 27527, 27529, 27539, 27541, 27551],
|
||||
[27581, 27583, 27611, 27617, 27631, 27647, 27653, 27673, 27689, 27691],
|
||||
[27697, 27701, 27733, 27737, 27739, 27743, 27749, 27751, 27763, 27767],
|
||||
[27773, 27779, 27791, 27793, 27799, 27803, 27809, 27817, 27823, 27827],
|
||||
[27847, 27851, 27883, 27893, 27901, 27917, 27919, 27941, 27943, 27947],
|
||||
[27953, 27961, 27967, 27983, 27997, 28001, 28019, 28027, 28031, 28051],
|
||||
[28057, 28069, 28081, 28087, 28097, 28099, 28109, 28111, 28123, 28151],
|
||||
[28163, 28181, 28183, 28201, 28211, 28219, 28229, 28277, 28279, 28283],
|
||||
[28289, 28297, 28307, 28309, 28319, 28349, 28351, 28387, 28393, 28403],
|
||||
[28409, 28411, 28429, 28433, 28439, 28447, 28463, 28477, 28493, 28499],
|
||||
[28513, 28517, 28537, 28541, 28547, 28549, 28559, 28571, 28573, 28579],
|
||||
[28591, 28597, 28603, 28607, 28619, 28621, 28627, 28631, 28643, 28649],
|
||||
[28657, 28661, 28663, 28669, 28687, 28697, 28703, 28711, 28723, 28729],
|
||||
[28751, 28753, 28759, 28771, 28789, 28793, 28807, 28813, 28817, 28837],
|
||||
[28843, 28859, 28867, 28871, 28879, 28901, 28909, 28921, 28927, 28933],
|
||||
[28949, 28961, 28979, 29009, 29017, 29021, 29023, 29027, 29033, 29059],
|
||||
[29063, 29077, 29101, 29123, 29129, 29131, 29137, 29147, 29153, 29167],
|
||||
[29173, 29179, 29191, 29201, 29207, 29209, 29221, 29231, 29243, 29251],
|
||||
[29269, 29287, 29297, 29303, 29311, 29327, 29333, 29339, 29347, 29363],
|
||||
[29383, 29387, 29389, 29399, 29401, 29411, 29423, 29429, 29437, 29443],
|
||||
[29453, 29473, 29483, 29501, 29527, 29531, 29537, 29567, 29569, 29573],
|
||||
[29581, 29587, 29599, 29611, 29629, 29633, 29641, 29663, 29669, 29671],
|
||||
[29683, 29717, 29723, 29741, 29753, 29759, 29761, 29789, 29803, 29819],
|
||||
[29833, 29837, 29851, 29863, 29867, 29873, 29879, 29881, 29917, 29921],
|
||||
[29927, 29947, 29959, 29983, 29989, 30011, 30013, 30029, 30047, 30059],
|
||||
[30071, 30089, 30091, 30097, 30103, 30109, 30113, 30119, 30133, 30137],
|
||||
[30139, 30161, 30169, 30181, 30187, 30197, 30203, 30211, 30223, 30241],
|
||||
[30253, 30259, 30269, 30271, 30293, 30307, 30313, 30319, 30323, 30341],
|
||||
[30347, 30367, 30389, 30391, 30403, 30427, 30431, 30449, 30467, 30469],
|
||||
[30491, 30493, 30497, 30509, 30517, 30529, 30539, 30553, 30557, 30559],
|
||||
[30577, 30593, 30631, 30637, 30643, 30649, 30661, 30671, 30677, 30689],
|
||||
[30697, 30703, 30707, 30713, 30727, 30757, 30763, 30773, 30781, 30803],
|
||||
[30809, 30817, 30829, 30839, 30841, 30851, 30853, 30859, 30869, 30871],
|
||||
[30881, 30893, 30911, 30931, 30937, 30941, 30949, 30971, 30977, 30983],
|
||||
[31013, 31019, 31033, 31039, 31051, 31063, 31069, 31079, 31081, 31091],
|
||||
[31121, 31123, 31139, 31147, 31151, 31153, 31159, 31177, 31181, 31189],
|
||||
[31193, 31219, 31223, 31231, 31237, 31247, 31249, 31253, 31259, 31267],
|
||||
[31271, 31277, 31307, 31319, 31321, 31327, 31333, 31337, 31357, 31379],
|
||||
[31387, 31391, 31393, 31397, 31469, 31477, 31481, 31489, 31511, 31513],
|
||||
[31517, 31531, 31541, 31543, 31547, 31567, 31573, 31583, 31601, 31607],
|
||||
[31627, 31643, 31649, 31657, 31663, 31667, 31687, 31699, 31721, 31723],
|
||||
[31727, 31729, 31741, 31751, 31769, 31771, 31793, 31799, 31817, 31847],
|
||||
[31849, 31859, 31873, 31883, 31891, 31907, 31957, 31963, 31973, 31981],
|
||||
[31991, 32003, 32009, 32027, 32029, 32051, 32057, 32059, 32063, 32069],
|
||||
[32077, 32083, 32089, 32099, 32117, 32119, 32141, 32143, 32159, 32173],
|
||||
[32183, 32189, 32191, 32203, 32213, 32233, 32237, 32251, 32257, 32261],
|
||||
[32297, 32299, 32303, 32309, 32321, 32323, 32327, 32341, 32353, 32359],
|
||||
[32363, 32369, 32371, 32377, 32381, 32401, 32411, 32413, 32423, 32429],
|
||||
[32441, 32443, 32467, 32479, 32491, 32497, 32503, 32507, 32531, 32533],
|
||||
[32537, 32561, 32563, 32569, 32573, 32579, 32587, 32603, 32609, 32611],
|
||||
[32621, 32633, 32647, 32653, 32687, 32693, 32707, 32713, 32717, 32719],
|
||||
[32749, 32771, 32779, 32783, 32789, 32797, 32801, 32803, 32831, 32833],
|
||||
[32839, 32843, 32869, 32887, 32909, 32911, 32917, 32933, 32939, 32941],
|
||||
[32957, 32969, 32971, 32983, 32987, 32993, 32999, 33013, 33023, 33029],
|
||||
[33037, 33049, 33053, 33071, 33073, 33083, 33091, 33107, 33113, 33119],
|
||||
[33149, 33151, 33161, 33179, 33181, 33191, 33199, 33203, 33211, 33223],
|
||||
[33247, 33287, 33289, 33301, 33311, 33317, 33329, 33331, 33343, 33347],
|
||||
[33349, 33353, 33359, 33377, 33391, 33403, 33409, 33413, 33427, 33457],
|
||||
[33461, 33469, 33479, 33487, 33493, 33503, 33521, 33529, 33533, 33547],
|
||||
[33563, 33569, 33577, 33581, 33587, 33589, 33599, 33601, 33613, 33617],
|
||||
[33619, 33623, 33629, 33637, 33641, 33647, 33679, 33703, 33713, 33721],
|
||||
[33739, 33749, 33751, 33757, 33767, 33769, 33773, 33791, 33797, 33809],
|
||||
[33811, 33827, 33829, 33851, 33857, 33863, 33871, 33889, 33893, 33911],
|
||||
[33923, 33931, 33937, 33941, 33961, 33967, 33997, 34019, 34031, 34033],
|
||||
[34039, 34057, 34061, 34123, 34127, 34129, 34141, 34147, 34157, 34159],
|
||||
[34171, 34183, 34211, 34213, 34217, 34231, 34253, 34259, 34261, 34267],
|
||||
[34273, 34283, 34297, 34301, 34303, 34313, 34319, 34327, 34337, 34351],
|
||||
[34361, 34367, 34369, 34381, 34403, 34421, 34429, 34439, 34457, 34469],
|
||||
[34471, 34483, 34487, 34499, 34501, 34511, 34513, 34519, 34537, 34543],
|
||||
[34549, 34583, 34589, 34591, 34603, 34607, 34613, 34631, 34649, 34651],
|
||||
[34667, 34673, 34679, 34687, 34693, 34703, 34721, 34729, 34739, 34747],
|
||||
[34757, 34759, 34763, 34781, 34807, 34819, 34841, 34843, 34847, 34849],
|
||||
[34871, 34877, 34883, 34897, 34913, 34919, 34939, 34949, 34961, 34963],
|
||||
[34981, 35023, 35027, 35051, 35053, 35059, 35069, 35081, 35083, 35089],
|
||||
[35099, 35107, 35111, 35117, 35129, 35141, 35149, 35153, 35159, 35171],
|
||||
[35201, 35221, 35227, 35251, 35257, 35267, 35279, 35281, 35291, 35311],
|
||||
[35317, 35323, 35327, 35339, 35353, 35363, 35381, 35393, 35401, 35407],
|
||||
[35419, 35423, 35437, 35447, 35449, 35461, 35491, 35507, 35509, 35521],
|
||||
[35527, 35531, 35533, 35537, 35543, 35569, 35573, 35591, 35593, 35597],
|
||||
[35603, 35617, 35671, 35677, 35729, 35731, 35747, 35753, 35759, 35771],
|
||||
[35797, 35801, 35803, 35809, 35831, 35837, 35839, 35851, 35863, 35869],
|
||||
[35879, 35897, 35899, 35911, 35923, 35933, 35951, 35963, 35969, 35977],
|
||||
[35983, 35993, 35999, 36007, 36011, 36013, 36017, 36037, 36061, 36067],
|
||||
[36073, 36083, 36097, 36107, 36109, 36131, 36137, 36151, 36161, 36187],
|
||||
[36191, 36209, 36217, 36229, 36241, 36251, 36263, 36269, 36277, 36293],
|
||||
[36299, 36307, 36313, 36319, 36341, 36343, 36353, 36373, 36383, 36389],
|
||||
[36433, 36451, 36457, 36467, 36469, 36473, 36479, 36493, 36497, 36523],
|
||||
[36527, 36529, 36541, 36551, 36559, 36563, 36571, 36583, 36587, 36599],
|
||||
[36607, 36629, 36637, 36643, 36653, 36671, 36677, 36683, 36691, 36697],
|
||||
[36709, 36713, 36721, 36739, 36749, 36761, 36767, 36779, 36781, 36787],
|
||||
[36791, 36793, 36809, 36821, 36833, 36847, 36857, 36871, 36877, 36887],
|
||||
[36899, 36901, 36913, 36919, 36923, 36929, 36931, 36943, 36947, 36973],
|
||||
[36979, 36997, 37003, 37013, 37019, 37021, 37039, 37049, 37057, 37061],
|
||||
[37087, 37097, 37117, 37123, 37139, 37159, 37171, 37181, 37189, 37199],
|
||||
[37201, 37217, 37223, 37243, 37253, 37273, 37277, 37307, 37309, 37313],
|
||||
[37321, 37337, 37339, 37357, 37361, 37363, 37369, 37379, 37397, 37409],
|
||||
[37423, 37441, 37447, 37463, 37483, 37489, 37493, 37501, 37507, 37511],
|
||||
[37517, 37529, 37537, 37547, 37549, 37561, 37567, 37571, 37573, 37579],
|
||||
[37589, 37591, 37607, 37619, 37633, 37643, 37649, 37657, 37663, 37691],
|
||||
[37693, 37699, 37717, 37747, 37781, 37783, 37799, 37811, 37813, 37831],
|
||||
[37847, 37853, 37861, 37871, 37879, 37889, 37897, 37907, 37951, 37957],
|
||||
[37963, 37967, 37987, 37991, 37993, 37997, 38011, 38039]]}}]}
|
||||
================================================================================
|
||||
File diff suppressed because it is too large
Load diff
237
modules/services/serviceAi/subAiCallLooping-flow.md
Normal file
237
modules/services/serviceAi/subAiCallLooping-flow.md
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
# AI Call Iteration Flow - JSON Merging System
|
||||
|
||||
This document describes the iteration flow for handling large JSON responses from AI that may be truncated and need to be merged across multiple iterations.
|
||||
|
||||
## Overview
|
||||
|
||||
When an AI response is too large, it may be truncated (cut) at an arbitrary point. The iteration system:
|
||||
1. Detects incomplete JSON
|
||||
2. Requests continuation from the AI
|
||||
3. Merges the continuation with the existing JSON
|
||||
4. Repeats until complete or max failures reached
|
||||
|
||||
---
|
||||
|
||||
## Key Variables
|
||||
|
||||
| Variable | Type | Purpose |
|
||||
|----------|------|---------|
|
||||
| `jsonBase` | `str \| None` | The merged JSON string (CUT version for overlap matching) |
|
||||
| `candidateJson` | `str` | Temporary holder for merged result until validated |
|
||||
| `lastValidCompletePart` | `str \| None` | Fallback - last successfully parsed CLOSED JSON |
|
||||
| `lastOverlapContext` | `str` | Context for retry/continuation prompts |
|
||||
| `lastHierarchyContextForPrompt` | `str` | Context for retry/continuation prompts |
|
||||
| `mergeFailCount` | `int` | Global counter (max 3 failures) |
|
||||
|
||||
---
|
||||
|
||||
## Key Distinction: hierarchyContext vs completePart
|
||||
|
||||
| Field | Description | Use Case |
|
||||
|-------|-------------|----------|
|
||||
| `hierarchyContext` | **CUT JSON** - truncated at cut point | Used as `jsonBase` for merging with next AI fragment |
|
||||
| `completePart` | **CLOSED JSON** - all structures properly closed | Used for validation, parsing, and fallback |
|
||||
|
||||
**Why this matters:**
|
||||
- The next AI fragment starts with an **overlap** that matches the CUT point
|
||||
- If we used `completePart` (closed), the overlap detection would FAIL
|
||||
- We must use `hierarchyContext` (cut) so overlap matching works correctly
|
||||
|
||||
---
|
||||
|
||||
## Flow Steps
|
||||
|
||||
### Step 1: BUILD PROMPT
|
||||
|
||||
**Location:** `subAiCallLooping.py` lines 163-212
|
||||
**Function:** `buildContinuationContext()` from `modules/shared/jsonUtils.py`
|
||||
|
||||
- **First iteration:** Use original prompt
|
||||
- **Continuation:** `buildContinuationContext(allSections, lastRawResponse, ...)`
|
||||
- Internally calls `getContexts(lastRawResponse)` to get overlap/hierarchy
|
||||
- Builds continuation prompt with `overlapContext` + `hierarchyContextForPrompt`
|
||||
|
||||
### Step 2: CALL AI
|
||||
|
||||
**Location:** `subAiCallLooping.py` lines 214-299
|
||||
**Function:** `self.aiService.callAi(request)`
|
||||
|
||||
- Returns `response.content` as `result`
|
||||
- Store raw response: `lastRawResponse = result`
|
||||
|
||||
### Step 4: MERGE
|
||||
|
||||
**Location:** `subAiCallLooping.py` lines 338-396
|
||||
**Function:** `JsonResponseHandler.mergeJsonStringsWithOverlap()` from `modules/services/serviceAi/subJsonResponseHandling.py`
|
||||
|
||||
```
|
||||
IF first iteration (jsonBase is None):
|
||||
→ candidateJson = result
|
||||
ELSE:
|
||||
→ mergedJsonString, hasOverlap = mergeJsonStringsWithOverlap(jsonBase, result)
|
||||
|
||||
IF hasOverlap = False (MERGE FAILED):
|
||||
→ mergeFailCount++
|
||||
→ If mergeFailCount >= 3: return lastValidCompletePart (fallback)
|
||||
→ Else: continue (retry with unchanged jsonBase)
|
||||
ELSE:
|
||||
→ candidateJson = mergedJsonString (don't update jsonBase yet!)
|
||||
|
||||
TRY DIRECT PARSE of candidateJson:
|
||||
IF parse succeeds:
|
||||
→ jsonBase = candidateJson (commit)
|
||||
→ FINISHED! Return normalized result
|
||||
ELSE:
|
||||
→ Proceed to Step 5
|
||||
```
|
||||
|
||||
### Step 5: GET CONTEXTS
|
||||
|
||||
**Location:** `subAiCallLooping.py` lines 420-427
|
||||
**Function:** `getContexts()` from `modules/shared/jsonContinuation.py`
|
||||
|
||||
```python
|
||||
contexts = getContexts(candidateJson)
|
||||
```
|
||||
|
||||
Returns `JsonContinuationContexts`:
|
||||
- `overlapContext`: `""` if JSON is complete (no cut point)
|
||||
- `hierarchyContext`: CUT JSON (for merging with next fragment)
|
||||
- `hierarchyContextForPrompt`: CUT JSON with budget limits (for prompts)
|
||||
- `completePart`: CLOSED JSON (repaired if needed)
|
||||
- `jsonParsingSuccess`: `True` if completePart is valid JSON
|
||||
|
||||
**Enhancement:** If original JSON is already complete → `overlapContext = ""`
|
||||
This signals "JSON is complete, no more continuation needed"
|
||||
|
||||
### Step 6: DECIDE
|
||||
|
||||
**Location:** `subAiCallLooping.py` lines 429-528
|
||||
|
||||
#### Case A: `jsonParsingSuccess=true` AND `overlapContext=""`
|
||||
**→ FINISHED**
|
||||
- JSON is complete (no cut point)
|
||||
- `jsonBase = contexts.completePart` (use CLOSED version for final result)
|
||||
- Return `completePart` as result
|
||||
|
||||
#### Case B: `jsonParsingSuccess=true` AND `overlapContext!=""`
|
||||
**→ CONTINUE to next iteration**
|
||||
- JSON parseable but has cut point
|
||||
- `jsonBase = contexts.hierarchyContext` ← **CUT version for next merge!**
|
||||
- `lastValidCompletePart = contexts.completePart` ← **CLOSED version for fallback**
|
||||
- Store contexts for next prompt
|
||||
- `mergeFailCount = 0` (reset on success)
|
||||
- `lastRawResponse = jsonBase`
|
||||
- Continue to next iteration
|
||||
|
||||
#### Case C: `jsonParsingSuccess=false`
|
||||
**→ RETRY with same prompt**
|
||||
- Do NOT update `jsonBase` (keep previous valid state)
|
||||
- `mergeFailCount++`
|
||||
- If `mergeFailCount >= 3`: return `lastValidCompletePart` (fallback)
|
||||
- Else: continue (retry with unchanged jsonBase/lastRawResponse)
|
||||
|
||||
---
|
||||
|
||||
## Flow Diagram
|
||||
|
||||
```
|
||||
┌───────────────────────────────────────────────────────────────┐
|
||||
│ ITERATION START │
|
||||
└───────────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌───────────────────────────▼───────────────────────────────────┐
|
||||
│ STEP 1: BUILD PROMPT │
|
||||
│ - First: original prompt │
|
||||
│ - Next: buildContinuationContext(lastRawResponse) │
|
||||
└───────────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌───────────────────────────▼───────────────────────────────────┐
|
||||
│ STEP 2: CALL AI → result │
|
||||
└───────────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌───────────────────────────▼───────────────────────────────────┐
|
||||
│ STEP 4: MERGE jsonBase + result → candidateJson │
|
||||
└───────────────────────────┬───────────────────────────────────┘
|
||||
│
|
||||
┌────────────▼────────────┐
|
||||
│ Merge OK? │
|
||||
└────────────┬────────────┘
|
||||
│
|
||||
┌─────────────────────┼─────────────────────┐
|
||||
│ NO │ YES │
|
||||
▼ ▼ │
|
||||
┌──────────────┐ ┌──────────────────┐ │
|
||||
│ fails++ │ │ TRY DIRECT PARSE │ │
|
||||
│ if >=3: │ │ of candidateJson │ │
|
||||
│ RETURN │ └────────┬─────────┘ │
|
||||
│ fallback │ │ │
|
||||
│ else: RETRY │ ┌────────▼─────────┐ │
|
||||
│ (continue) │ │ Parse OK? │ │
|
||||
└──────────────┘ └────────┬─────────┘ │
|
||||
│ │
|
||||
┌─────────────────────┼─────────────────────┐
|
||||
│ YES │ NO │
|
||||
▼ ▼ │
|
||||
┌──────────────┐ ┌──────────────────────────────┐
|
||||
│ FINISHED ✓ │ │ STEP 5: getContexts() │
|
||||
│ Return │ │ → jsonParsingSuccess │
|
||||
│ normalized │ │ → overlapContext │
|
||||
│ result │ └────────────┬─────────────────┘
|
||||
└──────────────┘ │
|
||||
┌────────────▼────────────────────┐
|
||||
│ STEP 6: DECIDE │
|
||||
└────────────┬────────────────────┘
|
||||
│
|
||||
┌────────────────────────────┼────────────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌───────────────────┐ ┌───────────────────────┐ ┌───────────────────┐
|
||||
│ success=true │ │ success=true │ │ success=false │
|
||||
│ overlap="" │ │ overlap!="" │ │ │
|
||||
│ ───────────── │ │ ───────────────── │ │ ───────────── │
|
||||
│ FINISHED ✓ │ │ CONTINUE │ │ RETRY │
|
||||
│ │ │ │ │ │
|
||||
│ jsonBase = │ │ jsonBase = │ │ jsonBase unchanged│
|
||||
│ completePart │ │ hierarchyContext │ │ fails++ │
|
||||
│ (CLOSED) │ │ (CUT for merge!) │ │ │
|
||||
│ │ │ │ │ if >=3: fallback │
|
||||
│ Return result │ │ fallback = │ │ else: retry │
|
||||
│ │ │ completePart │ │ │
|
||||
│ │ │ (CLOSED) │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ Next iteration → │ │ │
|
||||
└───────────────────┘ └───────────────────────┘ └───────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Files Involved
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `modules/services/serviceAi/subAiCallLooping.py` | Main iteration loop |
|
||||
| `modules/shared/jsonContinuation.py` | `getContexts()` - context extraction & repair |
|
||||
| `modules/shared/jsonUtils.py` | `buildContinuationContext()` - prompt building |
|
||||
| `modules/services/serviceAi/subJsonResponseHandling.py` | `mergeJsonStringsWithOverlap()` |
|
||||
| `modules/services/serviceAi/subJsonMerger.py` | `ModularJsonMerger` - actual merge logic |
|
||||
| `modules/datamodels/datamodelAi.py` | `JsonContinuationContexts` model |
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Merge Failures
|
||||
- Max 3 consecutive failures allowed
|
||||
- On failure: retry with unchanged `jsonBase` (previous valid state)
|
||||
- After 3 failures: return `lastValidCompletePart` as fallback
|
||||
|
||||
### Parse Failures
|
||||
- If `getContexts()` cannot produce valid JSON: increment fail counter
|
||||
- Retry with same prompt (don't update jsonBase)
|
||||
- After 3 failures: return `lastValidCompletePart` as fallback
|
||||
|
||||
### Fallback Strategy
|
||||
- `lastValidCompletePart` stores the last successfully parsed CLOSED JSON
|
||||
- Always available as fallback when things go wrong
|
||||
- Ensures we return valid JSON even after multiple failures
|
||||
|
|
@ -7,7 +7,44 @@ Handles AI calls with looping and repair logic, including:
|
|||
- Looping with JSON repair and continuation
|
||||
- KPI definition and tracking
|
||||
- Progress tracking and iteration management
|
||||
|
||||
FLOW LOGIC
|
||||
|
||||
VARIABLES:
|
||||
- jsonBase: str (merged JSON so far, starts empty)
|
||||
- lastValidCompletePart: str (fallback for failures)
|
||||
- mergeFailCount: int = 0 (max 3)
|
||||
|
||||
FLOW:
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 1. BUILD PROMPT │
|
||||
│ - First: original prompt │
|
||||
│ - Next: buildContinuationContext(lastRawResponse) │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ 2. CALL AI → response fragment │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ 4. MERGE jsonBase + response │
|
||||
│ ├─ FAILS: repeat prompt, fails++ (if >=3 return fallback) │
|
||||
│ └─ SUCCEEDS: try parse │
|
||||
│ ├─ SUCCEEDS: FINISHED │
|
||||
│ └─ FAILS: → step 5 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ 5. GET CONTEXTS (merge OK, parse failed) │
|
||||
│ getContexts(mergedJson) → │
|
||||
│ - If no cut point: overlapContext = "" │
|
||||
│ - Store contexts for next iteration │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ 6. DECIDE │
|
||||
│ ├─ jsonParsingSuccess=true AND overlapContext="": │
|
||||
│ │ FINISHED. return completePart │
|
||||
│ ├─ jsonParsingSuccess=true AND overlapContext!="": │
|
||||
│ │ CONTINUE, fails=0 │
|
||||
│ └─ ELSE: repeat prompt, fails++ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
|
|
@ -16,10 +53,14 @@ from modules.datamodels.datamodelAi import (
|
|||
AiCallRequest, AiCallOptions
|
||||
)
|
||||
from modules.datamodels.datamodelExtraction import ContentPart
|
||||
from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson
|
||||
from modules.services.serviceAi.subJsonResponseHandling import JsonResponseHandler
|
||||
from modules.services.serviceAi.subLoopingUseCases import LoopingUseCaseRegistry
|
||||
from modules.workflows.processing.shared.stateTools import checkWorkflowStopped
|
||||
from modules.shared.jsonContinuation import getContexts
|
||||
from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson
|
||||
from modules.shared.jsonUtils import tryParseJson
|
||||
from modules.shared.jsonUtils import closeJsonStructures
|
||||
from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -88,7 +129,18 @@ class AiCallLooper:
|
|||
iteration = 0
|
||||
allSections = [] # Accumulate all sections across iterations
|
||||
lastRawResponse = None # Store last raw JSON response for continuation
|
||||
accumulatedDirectJson = [] # Accumulate JSON strings for direct return use cases (chapter_structure, code_structure)
|
||||
|
||||
# JSON Base Iteration System:
|
||||
# - jsonBase: the merged JSON string (replaces accumulatedDirectJson array)
|
||||
# - After each iteration, new response is merged with jsonBase
|
||||
# - On merge success: check if complete, store contexts for next iteration
|
||||
# - On merge fail: retry with same prompt, increment fails
|
||||
jsonBase = None # Merged JSON string (starts None, set on first response)
|
||||
|
||||
# Merge fail tracking - stop after 3 consecutive merge failures
|
||||
MAX_MERGE_FAILS = 3
|
||||
mergeFailCount = 0 # Global counter for merge failures across entire loop
|
||||
lastValidCompletePart = None # Store last successfully parsed completePart for fallback
|
||||
|
||||
# Get parent operation ID for iteration operations (parentId should be operationId, not log entry ID)
|
||||
parentOperationId = operationId # Use the parent's operationId directly
|
||||
|
|
@ -136,6 +188,17 @@ class AiCallLooper:
|
|||
if not lastRawResponse:
|
||||
logger.warning(f"Iteration {iteration}: No previous response available for continuation!")
|
||||
|
||||
# Store valid completePart from continuation context for fallback on merge failures
|
||||
# Use getContexts to check if completePart is parseable and store it
|
||||
if lastRawResponse and not lastValidCompletePart:
|
||||
try:
|
||||
contexts = getContexts(lastRawResponse)
|
||||
if contexts.jsonParsingSuccess and contexts.completePart:
|
||||
lastValidCompletePart = contexts.completePart
|
||||
logger.debug(f"Iteration {iteration}: Stored initial valid completePart ({len(lastValidCompletePart)} chars)")
|
||||
except Exception as e:
|
||||
logger.debug(f"Iteration {iteration}: Failed to extract completePart: {e}")
|
||||
|
||||
# Unified prompt builder call: Continuation builders only need continuationContext, templateStructure, and basePrompt
|
||||
# All initial context (section, userPrompt, etc.) is already in basePrompt, so promptArgs is not needed
|
||||
# Extract templateStructure and basePrompt from promptArgs (they're explicit parameters)
|
||||
|
|
@ -247,238 +310,220 @@ class AiCallLooper:
|
|||
# Store raw response for continuation (even if broken)
|
||||
lastRawResponse = result
|
||||
|
||||
# Parse JSON for use case handling
|
||||
parsedJsonForUseCase = None
|
||||
extractedJsonForUseCase = None
|
||||
|
||||
try:
|
||||
extractedJsonForUseCase = extractJsonString(result)
|
||||
parsedJson, parseError, _ = tryParseJson(extractedJsonForUseCase)
|
||||
if parseError is None and parsedJson:
|
||||
parsedJsonForUseCase = parsedJson
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Handle use cases that return JSON directly (no section extraction needed)
|
||||
# Check if use case supports direct return (all registered use cases do)
|
||||
if useCase and not useCase.requiresExtraction:
|
||||
# For all direct return use cases, check completeness and support looping
|
||||
if True: # All registered use cases support looping
|
||||
# If parsing failed (e.g., invalid JSON with comments or truncated JSON), continue looping to get valid JSON
|
||||
if not parsedJsonForUseCase:
|
||||
logger.info(f"Iteration {iteration}: Use case '{useCaseId}' - JSON parsing failed (likely incomplete/truncated), continuing iteration to complete")
|
||||
# Accumulate response for merging in next iteration
|
||||
accumulatedDirectJson.append(result)
|
||||
# =====================================================================
|
||||
# ITERATION FLOW (Simplified)
|
||||
# =====================================================================
|
||||
# Step 4: MERGE jsonBase + new response
|
||||
# - FAILS: repeat prompt, increment fails cont (if >=3 return fallback)
|
||||
# - SUCCEEDS: try parse
|
||||
# - SUCCEEDS: FINISHED
|
||||
# - FAILS: proceed to Step 5
|
||||
# Step 5: GET CONTEXTS (merge OK, parse failed)
|
||||
# - getContexts() with repair
|
||||
# - If no cut point: overlapContext = ""
|
||||
# Step 6: DECIDE
|
||||
# - jsonParsingSuccess=true AND overlapContext="": FINISHED
|
||||
# - jsonParsingSuccess=true AND overlapContext!="": continue, fails=0
|
||||
# - ELSE: repeat prompt, increment fails count
|
||||
# =====================================================================
|
||||
|
||||
# STEP 4: MERGE jsonBase + new response
|
||||
# Use candidateJson to hold merged result until we confirm it's valid
|
||||
candidateJson = None
|
||||
|
||||
if jsonBase is None:
|
||||
# First iteration - candidate is the current result
|
||||
candidateJson = result
|
||||
logger.debug(f"Iteration {iteration}: First response, candidateJson ({len(candidateJson)} chars)")
|
||||
else:
|
||||
# Merge jsonBase with new response
|
||||
logger.info(f"Iteration {iteration}: Merging jsonBase ({len(jsonBase)} chars) with new response ({len(result)} chars)")
|
||||
mergedJsonString, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap(jsonBase, result)
|
||||
|
||||
if not hasOverlap:
|
||||
# MERGE FAILED - repeat prompt with unchanged jsonBase
|
||||
mergeFailCount += 1
|
||||
logger.warning(
|
||||
f"Iteration {iteration}: Merge failed, no overlap found "
|
||||
f"(fail {mergeFailCount}/{MAX_MERGE_FAILS})"
|
||||
)
|
||||
|
||||
# Continue to next iteration - continuation prompt builder will handle the rest
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogUpdate(iterationOperationId, 0.7, "JSON incomplete, requesting continuation")
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
continue
|
||||
|
||||
# If we successfully parsed JSON, check completeness using parsed structure
|
||||
# CRITICAL: If parsing succeeded, trust the parsed JSON structure check
|
||||
# The string-based check can have false positives on valid JSON (e.g., due to normalization issues)
|
||||
# Only use string-based check when parsing fails (already handled above)
|
||||
isComplete = JsonResponseHandler.isJsonComplete(parsedJsonForUseCase)
|
||||
|
||||
# If parsed check says complete, trust it - don't override with string check
|
||||
# String check is only reliable when parsing fails (truncated JSON that gets closed for parsing)
|
||||
# For successfully parsed JSON, the structure check is definitive
|
||||
|
||||
if not isComplete:
|
||||
logger.warning(f"Iteration {iteration}: Use case '{useCaseId}' - JSON is incomplete, continuing for continuation")
|
||||
# Accumulate response for merging in next iteration
|
||||
accumulatedDirectJson.append(result)
|
||||
|
||||
# Continue to next iteration - continuation prompt builder will handle the rest
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogUpdate(iterationOperationId, 0.7, "JSON incomplete, requesting continuation")
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
continue
|
||||
else:
|
||||
# JSON is complete - merge accumulated responses if any
|
||||
if accumulatedDirectJson:
|
||||
logger.info(f"Iteration {iteration}: Merging {len(accumulatedDirectJson) + 1} accumulated responses")
|
||||
if mergeFailCount >= MAX_MERGE_FAILS:
|
||||
# Max failures reached - return last valid completePart
|
||||
logger.error(
|
||||
f"Iteration {iteration}: Max merge failures ({MAX_MERGE_FAILS}) reached, "
|
||||
"returning last valid completePart"
|
||||
)
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, False)
|
||||
|
||||
# Use generic data-based merging for all use cases
|
||||
try:
|
||||
# Strategy: Merge strings first for incomplete JSON, then parse and merge parsed objects
|
||||
# This ensures incomplete JSON from part 1 is preserved
|
||||
allJsonStrings = accumulatedDirectJson + [result]
|
||||
|
||||
# Step 1: Merge all JSON strings using existing overlap detection
|
||||
mergedJsonString = allJsonStrings[0] if allJsonStrings else ""
|
||||
hasOverlap = True # Track if any overlap was found
|
||||
for jsonStr in allJsonStrings[1:]:
|
||||
mergedJsonString, hasOverlapInMerge = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, jsonStr)
|
||||
# If no overlap found in any merge, stop iterations
|
||||
if not hasOverlapInMerge:
|
||||
hasOverlap = False
|
||||
logger.info(f"Iteration {iteration}: No overlap found during merge - stopping iterations and closing JSON")
|
||||
break
|
||||
|
||||
# If no overlap was found, mark as complete and use closed JSON
|
||||
if not hasOverlap:
|
||||
isComplete = True
|
||||
# JSON is already closed by mergeJsonStringsWithOverlap when no overlap
|
||||
# Use the merged (closed) JSON string directly
|
||||
result = mergedJsonString
|
||||
# CRITICAL: Update lastRawResponse with merged result for next iteration
|
||||
lastRawResponse = mergedJsonString
|
||||
# Try to parse it to get parsedJsonForUseCase
|
||||
try:
|
||||
extracted = extractJsonString(mergedJsonString)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
# Use callback to normalize JSON structure
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
parsedJsonForUseCase = normalized
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
# CRITICAL: Update lastRawResponse with final result
|
||||
lastRawResponse = result
|
||||
else:
|
||||
# Parsing failed - try to repair JSON
|
||||
from modules.shared.jsonUtils import repairBrokenJson
|
||||
logger.warning(
|
||||
f"Iteration {iteration}: JSON parse failed after no-overlap merge, "
|
||||
f"attempting repair: {str(parseErr) if parseErr else 'Unknown error'}"
|
||||
)
|
||||
repairedJson = repairBrokenJson(extracted)
|
||||
if repairedJson and isinstance(repairedJson, dict):
|
||||
# repairBrokenJson returns a dict directly - use it
|
||||
normalized = self._normalizeJsonStructure(repairedJson, useCase)
|
||||
parsedJsonForUseCase = normalized
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
# CRITICAL: Update lastRawResponse with final result
|
||||
lastRawResponse = result
|
||||
logger.info(f"Iteration {iteration}: Successfully repaired JSON after no-overlap merge")
|
||||
except Exception as e:
|
||||
# Last resort: try repair on the original merged string
|
||||
logger.warning(
|
||||
f"Iteration {iteration}: Exception during no-overlap JSON processing, "
|
||||
f"attempting repair: {str(e)}"
|
||||
)
|
||||
try:
|
||||
from modules.shared.jsonUtils import repairBrokenJson
|
||||
repairedJson = repairBrokenJson(mergedJsonString)
|
||||
if repairedJson and isinstance(repairedJson, dict):
|
||||
normalized = self._normalizeJsonStructure(repairedJson, useCase)
|
||||
parsedJsonForUseCase = normalized
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
logger.info(f"Iteration {iteration}: Successfully repaired JSON after exception")
|
||||
else:
|
||||
logger.error(f"Iteration {iteration}: JSON repair failed, using string result as-is")
|
||||
except Exception as repairError:
|
||||
logger.error(
|
||||
f"Iteration {iteration}: JSON repair also failed: {str(repairError)}, "
|
||||
"using string result as-is"
|
||||
)
|
||||
else:
|
||||
# Overlap found - continue with normal processing
|
||||
# Step 2: Try to parse the merged string
|
||||
extracted = extractJsonString(mergedJsonString)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
|
||||
if parseErr is None and parsed:
|
||||
# Parsing succeeded - normalize and use (via callback)
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
parsedJsonForUseCase = normalized
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
# CRITICAL: Update lastRawResponse with merged result
|
||||
lastRawResponse = result
|
||||
else:
|
||||
# Parsing failed - try to extract partial data using Deep-Structure-Merging
|
||||
# This fallback works for all use cases: parse what we can from each part
|
||||
allParsed = []
|
||||
for jsonStr in allJsonStrings:
|
||||
extracted = extractJsonString(jsonStr)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
# Use callback to normalize JSON structure
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
allParsed.append(normalized)
|
||||
|
||||
if allParsed:
|
||||
# Use mergeDeepStructures for intelligent merging across all use cases
|
||||
if len(allParsed) > 1:
|
||||
mergedJsonObj = allParsed[0]
|
||||
for nextObj in allParsed[1:]:
|
||||
mergedJsonObj = JsonResponseHandler.mergeDeepStructures(
|
||||
mergedJsonObj, nextObj, iteration, f"{useCaseId}.merge"
|
||||
)
|
||||
else:
|
||||
mergedJsonObj = allParsed[0]
|
||||
|
||||
parsedJsonForUseCase = mergedJsonObj
|
||||
result = json.dumps(mergedJsonObj, indent=2, ensure_ascii=False)
|
||||
# CRITICAL: Update lastRawResponse with merged result
|
||||
lastRawResponse = result
|
||||
else:
|
||||
# All parsing failed - use string merge result
|
||||
result = mergedJsonString
|
||||
# CRITICAL: Update lastRawResponse with merged result
|
||||
lastRawResponse = mergedJsonString
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed data-based merge, falling back to string merging: {e}")
|
||||
# Fallback to string merging
|
||||
mergedJsonString = accumulatedDirectJson[0] if accumulatedDirectJson else result
|
||||
hasOverlap = True # Track if any overlap was found
|
||||
for prevJson in accumulatedDirectJson[1:]:
|
||||
mergedJsonString, hasOverlapInMerge = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, prevJson)
|
||||
if not hasOverlapInMerge:
|
||||
hasOverlap = False
|
||||
logger.info(f"Iteration {iteration}: No overlap found during fallback merge - stopping iterations")
|
||||
break
|
||||
if hasOverlap:
|
||||
mergedJsonString, hasOverlapInMerge = JsonResponseHandler.mergeJsonStringsWithOverlap(mergedJsonString, result)
|
||||
if not hasOverlapInMerge:
|
||||
hasOverlap = False
|
||||
logger.info(f"Iteration {iteration}: No overlap found in final fallback merge - stopping iterations")
|
||||
result = mergedJsonString
|
||||
# CRITICAL: Update lastRawResponse with merged result
|
||||
lastRawResponse = mergedJsonString
|
||||
|
||||
# If no overlap was found, mark as complete and use closed JSON
|
||||
if not hasOverlap:
|
||||
isComplete = True
|
||||
# JSON is already closed by mergeJsonStringsWithOverlap when no overlap
|
||||
# Try to parse it to get parsedJsonForUseCase
|
||||
try:
|
||||
extractedMerged = extractJsonString(result)
|
||||
parsedMerged, parseError, _ = tryParseJson(extractedMerged)
|
||||
if parseError is None and parsedMerged:
|
||||
parsedJsonForUseCase = parsedMerged
|
||||
except Exception:
|
||||
pass # Use string result if parsing fails
|
||||
|
||||
# Try to parse the string-merged result
|
||||
if lastValidCompletePart:
|
||||
try:
|
||||
extractedMerged = extractJsonString(result)
|
||||
parsedMerged, parseError, _ = tryParseJson(extractedMerged)
|
||||
if parseError is None and parsedMerged:
|
||||
parsedJsonForUseCase = parsedMerged
|
||||
extracted = extractJsonString(lastValidCompletePart)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
return json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
return lastValidCompletePart
|
||||
else:
|
||||
# No valid fallback - return whatever we have
|
||||
return jsonBase if jsonBase else ""
|
||||
|
||||
logger.info(f"Iteration {iteration}: Use case '{useCaseId}' - JSON is complete")
|
||||
# Not at max failures - retry with same prompt (jsonBase unchanged)
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogUpdate(
|
||||
iterationOperationId, 0.7,
|
||||
f"Merge failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying"
|
||||
)
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
continue
|
||||
|
||||
# MERGE SUCCEEDED - set candidate (don't update jsonBase yet!)
|
||||
candidateJson = mergedJsonString
|
||||
logger.debug(f"Iteration {iteration}: Merge succeeded, candidateJson ({len(candidateJson)} chars)")
|
||||
|
||||
logger.info(f"Iteration {iteration}: Use case '{useCaseId}' - returning JSON directly")
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
# Try direct parse of candidate
|
||||
try:
|
||||
extracted = extractJsonString(candidateJson)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
# Direct parse succeeded - FINISHED
|
||||
# Commit candidate to jsonBase
|
||||
jsonBase = candidateJson
|
||||
logger.info(f"Iteration {iteration}: Direct parse succeeded, JSON is complete")
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
|
||||
if not useCase.finalResultHandler:
|
||||
raise ValueError(
|
||||
f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback."
|
||||
)
|
||||
return useCase.finalResultHandler(
|
||||
result, normalized, extracted, debugPrefix, self.services
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Iteration {iteration}: Direct parse failed: {e}")
|
||||
|
||||
# Use callback to handle final result formatting and debug file writing (REQUIRED - no fallback)
|
||||
if not useCase.finalResultHandler:
|
||||
raise ValueError(
|
||||
f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback. "
|
||||
"All use cases must provide a finalResultHandler function."
|
||||
)
|
||||
final_json = useCase.finalResultHandler(
|
||||
result, parsedJsonForUseCase, extractedJsonForUseCase,
|
||||
debugPrefix, self.services
|
||||
# STEP 5: GET CONTEXTS (merge OK, parse failed = cut JSON)
|
||||
# Use candidateJson for context extraction
|
||||
contexts = getContexts(candidateJson)
|
||||
logger.debug(
|
||||
f"Iteration {iteration}: getContexts() -> "
|
||||
f"jsonParsingSuccess={contexts.jsonParsingSuccess}, "
|
||||
f"overlapContext={'\"\"' if not contexts.overlapContext else f'({len(contexts.overlapContext)} chars)'}"
|
||||
)
|
||||
|
||||
return final_json
|
||||
# STEP 6: DECIDE based on jsonParsingSuccess and overlapContext
|
||||
if contexts.jsonParsingSuccess and contexts.overlapContext == "":
|
||||
# JSON is complete (no cut point) - FINISHED
|
||||
# Use completePart for final result (closed, repaired JSON)
|
||||
# No more merging needed, so we don't need the cut version
|
||||
jsonBase = contexts.completePart
|
||||
logger.info(f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext='', JSON complete")
|
||||
|
||||
# Store and parse completePart
|
||||
lastValidCompletePart = contexts.completePart
|
||||
|
||||
try:
|
||||
extracted = extractJsonString(contexts.completePart)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
result = json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
|
||||
if not useCase.finalResultHandler:
|
||||
raise ValueError(
|
||||
f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback."
|
||||
)
|
||||
return useCase.finalResultHandler(
|
||||
result, normalized, extracted, debugPrefix, self.services
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Iteration {iteration}: Failed to parse completePart: {e}")
|
||||
|
||||
# Fallback: return completePart as-is
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
return contexts.completePart
|
||||
|
||||
elif contexts.jsonParsingSuccess and contexts.overlapContext != "":
|
||||
# JSON parseable but has cut point - CONTINUE to next iteration
|
||||
# CRITICAL: Use hierarchyContext (CUT json) as jsonBase for next merge!
|
||||
# - hierarchyContext = the truncated JSON at cut point (needed for overlap matching)
|
||||
# - completePart = closed JSON (for validation/fallback only)
|
||||
# The next AI fragment's overlap must match the CUT point, not closed structures
|
||||
jsonBase = contexts.hierarchyContext
|
||||
logger.info(
|
||||
f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext not empty, "
|
||||
f"continuing iteration (jsonBase updated to hierarchyContext: {len(jsonBase)} chars)"
|
||||
)
|
||||
|
||||
# Store valid completePart as fallback (different from jsonBase!)
|
||||
lastValidCompletePart = contexts.completePart
|
||||
|
||||
# Reset fail counter on successful progress
|
||||
mergeFailCount = 0
|
||||
|
||||
# Update lastRawResponse for continuation prompt building
|
||||
# Use the CUT version for prompt context as well
|
||||
lastRawResponse = jsonBase
|
||||
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogUpdate(iterationOperationId, 0.7, "JSON incomplete, requesting continuation")
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
continue
|
||||
|
||||
else:
|
||||
# JSON not parseable after repair - repeat prompt, increment fails
|
||||
# Do NOT update jsonBase - keep previous valid state
|
||||
mergeFailCount += 1
|
||||
logger.warning(
|
||||
f"Iteration {iteration}: jsonParsingSuccess=false, "
|
||||
f"repeat prompt (fail {mergeFailCount}/{MAX_MERGE_FAILS})"
|
||||
)
|
||||
|
||||
if mergeFailCount >= MAX_MERGE_FAILS:
|
||||
# Max failures reached - return last valid completePart
|
||||
logger.error(
|
||||
f"Iteration {iteration}: Max failures ({MAX_MERGE_FAILS}) reached, "
|
||||
"returning last valid completePart"
|
||||
)
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogFinish(iterationOperationId, False)
|
||||
|
||||
if lastValidCompletePart:
|
||||
try:
|
||||
extracted = extractJsonString(lastValidCompletePart)
|
||||
parsed, parseErr, _ = tryParseJson(extracted)
|
||||
if parseErr is None and parsed:
|
||||
normalized = self._normalizeJsonStructure(parsed, useCase)
|
||||
return json.dumps(normalized, indent=2, ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
return lastValidCompletePart
|
||||
else:
|
||||
return jsonBase if jsonBase else ""
|
||||
|
||||
# Not at max - retry with same prompt
|
||||
# Do NOT update jsonBase or lastRawResponse - keep previous for retry
|
||||
if iterationOperationId:
|
||||
self.services.chat.progressLogUpdate(
|
||||
iterationOperationId, 0.7,
|
||||
f"Parse failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying"
|
||||
)
|
||||
self.services.chat.progressLogFinish(iterationOperationId, True)
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in AI call iteration {iteration}: {str(e)}")
|
||||
|
|
@ -510,9 +555,7 @@ class AiCallLooper:
|
|||
"""
|
||||
if not jsonString or not jsonString.strip():
|
||||
return False
|
||||
|
||||
from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText
|
||||
|
||||
|
||||
# Normalize JSON string
|
||||
normalized = stripCodeFences(normalizeJsonText(jsonString)).strip()
|
||||
if not normalized:
|
||||
|
|
@ -587,14 +630,12 @@ class AiCallLooper:
|
|||
# - Ends with incomplete object property (e.g., {"key": "val)
|
||||
|
||||
# If JSON parses successfully without closing, it's complete
|
||||
from modules.shared.jsonUtils import tryParseJson
|
||||
parsed, parseErr, _ = tryParseJson(jsonContent)
|
||||
if parseErr is None:
|
||||
# Parses successfully - it's complete
|
||||
return False
|
||||
|
||||
# If it doesn't parse, try closing it and see if that helps
|
||||
from modules.shared.jsonUtils import closeJsonStructures
|
||||
closed = closeJsonStructures(jsonContent)
|
||||
parsedClosed, parseErrClosed, _ = tryParseJson(closed)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
# Budget Rendering Requirement for JSON Continuation Context
|
||||
|
||||
## Problem Statement
|
||||
When rendering the `hierarchyContextForPrompt` with budget constraints, we need to prioritize data near the cut point (where JSON was truncated) over data near the root. Currently, the implementation renders top-down (root → children), causing root-level data to consume budget before cut-level data gets rendered.
|
||||
|
||||
## Requirement
|
||||
|
||||
### Core Logic
|
||||
1. **Find the cut element**: Identify the element in the JSON structure where truncation occurred (the cut point).
|
||||
|
||||
2. **Build path from cut to root**: Create an ordered list of nodes from the cut element back to the root element. The path should be: `[cut_element, parent_of_cut, grandparent_of_cut, ..., root]`.
|
||||
|
||||
3. **Walk backwards from cut to root, consuming budget**:
|
||||
- Start at the cut element
|
||||
- Walk backwards along the path to root
|
||||
- For each value node encountered on the path:
|
||||
- If value size ≤ remaining budget: render full value and deduct budget
|
||||
- If value size > remaining budget: render type hint (`<str>`, `<number>`, etc.) and **DO NOT** deduct budget (never let budget go below 0)
|
||||
- If remaining budget < 50: set budget to 0 and enable "summary mode"
|
||||
|
||||
4. **Summary mode (budget = 0)**:
|
||||
- Continue walking towards root
|
||||
- For elements on the **same level** as current position: render with type hints
|
||||
- For elements on **higher levels** (closer to root): render only structure (keys/attributes) without values - just show the level hierarchy
|
||||
|
||||
### Critical Constraints
|
||||
- **Never let budget go below 0**: If a value is bigger than remaining budget, use type hint instead of rendering data
|
||||
- **Budget allocation order**: Cut element → parent → grandparent → ... → root (bottom-up)
|
||||
- **Rendering order**: Can be top-down (root → children) for structure, but budget must be allocated bottom-up
|
||||
- **When budget < 50**: Set to 0 and enable summary mode immediately
|
||||
|
||||
### Example Flow
|
||||
|
||||
Given JSON structure (truncated at cut point):
|
||||
```json
|
||||
{
|
||||
"document": {
|
||||
"metadata": {
|
||||
"title": "My Document",
|
||||
"author": "John Doe",
|
||||
"version": 1
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "section1",
|
||||
"title": "Introduction",
|
||||
"content": "This is the introduction content..."
|
||||
},
|
||||
{
|
||||
"id": "section2",
|
||||
"title": "Main Content",
|
||||
"content": "This is a very long content that gets cut right here in the middle of this sentence and the JSON is truncated..."
|
||||
```
|
||||
|
||||
**Cut point**: The JSON is truncated in the middle of `sections[1].content` value.
|
||||
|
||||
**Path from cut to root**:
|
||||
```
|
||||
[
|
||||
sections[1].content (value - CUT ELEMENT),
|
||||
sections[1].content (key-value pair),
|
||||
sections[1] (object),
|
||||
sections (array),
|
||||
document (object),
|
||||
root (object)
|
||||
]
|
||||
```
|
||||
|
||||
**Budget allocation order (walking backwards from cut → root)**:
|
||||
|
||||
1. **`sections[1].content` (value)** - CUT ELEMENT
|
||||
- Check: value size = 120 chars, budget = 500
|
||||
- Action: Render full value "This is a very long content that gets cut right here in the middle of this sentence and the JSON is truncated..."
|
||||
- Deduct: 120 chars from budget → remaining = 380
|
||||
|
||||
2. **`sections[1]` (object)** - Parent of cut
|
||||
- Action: Render structure only (no budget needed for `{`, `}`, keys)
|
||||
- Render: `"id": "section2", "title": "Main Content", "content": <already rendered>`
|
||||
|
||||
3. **`sections` (array)** - Grandparent
|
||||
- Action: Render structure only
|
||||
- Render: `[<section1>, <section2>]` where section2 already has content rendered
|
||||
|
||||
4. **`document` (object)** - Great-grandparent
|
||||
- Action: Render structure only
|
||||
- Render: `"metadata": {...}, "sections": <already rendered>`
|
||||
|
||||
5. **`root` (object)** - Root
|
||||
- Action: Render structure only
|
||||
- Render: `{"document": <already rendered>}`
|
||||
|
||||
**If budget becomes 0 during step 1** (e.g., value size = 600, budget = 500):
|
||||
- `sections[1].content` gets type hint `<str>` (value too big, don't deduct budget)
|
||||
- Budget remains 500, but if < 50, set to 0 and enable summary mode
|
||||
- Continue to root with summary mode:
|
||||
- **Same level elements** (e.g., `sections[1].id`, `sections[1].title`): type hints (`<str>`)
|
||||
- **Higher level elements** (e.g., `sections[0]`, `metadata`): structure only (keys, braces, no values)
|
||||
|
||||
**Expected output with budget = 500** (sufficient budget):
|
||||
```json
|
||||
{
|
||||
"document": {
|
||||
"metadata": {
|
||||
"title": "My Document",
|
||||
"author": "John Doe",
|
||||
"version": 1
|
||||
},
|
||||
"sections": [
|
||||
{
|
||||
"id": "section1",
|
||||
"title": "Introduction",
|
||||
"content": "This is the introduction content..."
|
||||
},
|
||||
{
|
||||
"id": "section2",
|
||||
"title": "Main Content",
|
||||
"content": "This is a very long content that gets cut right here in the middle of this sentence and the JSON is truncated..."
|
||||
```
|
||||
*All values rendered because budget is sufficient.*
|
||||
|
||||
**Expected output if ONE value is too big and budget running out** (budget = 100, cut value = 120):
|
||||
```json
|
||||
{
|
||||
"document": {
|
||||
"metadata": <object>>,
|
||||
"sections": [
|
||||
{
|
||||
"id": "section1",
|
||||
"title": "Introduction",
|
||||
"content": "This is the introduction content..."
|
||||
},
|
||||
{
|
||||
"id": "section2",
|
||||
"title": <str>,
|
||||
"content": "This is a very long content that gets cut right here in the middle of this sentence and the JSON is truncated..."
|
||||
```
|
||||
|
||||
|
||||
|
||||
**Key Points:**
|
||||
- **Single value too big**: Only that value gets type hint, continue rendering other data
|
||||
- **Budget > 0**: Render side paths (siblings, other branches) as long as budget allows
|
||||
- **Budget = 0**: Stop rendering side paths, only render path from cut element to root (structure only for higher levels, type hints for same level)
|
||||
|
||||
## Current Implementation Issues
|
||||
|
||||
The current implementation in `jsonContinuation.py`:
|
||||
- Pre-allocates budget to path elements before rendering
|
||||
- But rendering still happens top-down, so root elements consume budget first
|
||||
- Path elements check for pre-allocated budget, but non-path elements also consume budget during top-down rendering
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
**Before budget runs out:**
|
||||
- Cut element and path to root: full values rendered
|
||||
- Other elements: full values if budget allows
|
||||
|
||||
**After budget < 50 (summary mode):**
|
||||
- Cut element and path to root: full values (if budget was allocated)
|
||||
- Same level elements: type hints (`<str>`, `<object>`, etc.)
|
||||
- Higher level elements: structure only (keys, braces, brackets - no values)
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The path should be built using `_findPathToRoot()` which walks from root to find cut element
|
||||
- Budget should be consumed during a separate pass that walks the path (cut → root)
|
||||
- During rendering, path elements should check if they have pre-allocated budget
|
||||
- Non-path elements should only consume leftover budget after path elements are processed
|
||||
- Structure elements (objects, arrays) don't consume budget - only values do
|
||||
|
|
@ -1,305 +0,0 @@
|
|||
"""
|
||||
FINAL CORRECTED _renderWithBudgetFromStructure - Version 3
|
||||
|
||||
This file contains the CORRECT implementation that should REPLACE
|
||||
the existing methods in jsonContinuation.py
|
||||
|
||||
KEY BEHAVIOR:
|
||||
1. Budget is allocated from CUT → ROOT (not top-down)
|
||||
2. Cut-near values get priority
|
||||
3. When budget < 50: summary_mode enabled, non-path containers → <object>/<array>
|
||||
4. Path containers always render their structure
|
||||
|
||||
COPY THESE METHODS INTO YOUR JsonAnalyzer CLASS:
|
||||
- _renderWithBudgetFromStructure (REPLACE existing)
|
||||
- _buildPathFromCutToRootV3 (ADD)
|
||||
- _collectAllValuesWithDistance (ADD)
|
||||
- _renderNodeV3 (ADD)
|
||||
- _renderObjectV3 (ADD)
|
||||
- _renderArrayV3 (ADD)
|
||||
- _renderValueV3 (ADD)
|
||||
"""
|
||||
|
||||
from typing import List, Set
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class BudgetAllocation:
|
||||
"""Tracks which nodes have been allocated budget"""
|
||||
allocated_node_ids: Set[int] = field(default_factory=set)
|
||||
path_node_ids: Set[int] = field(default_factory=set)
|
||||
summary_mode: bool = False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# METHODS TO COPY INTO JsonAnalyzer CLASS
|
||||
# =============================================================================
|
||||
|
||||
def _renderWithBudgetFromStructure(self, structure: dict, cutPos: int) -> str:
|
||||
"""
|
||||
Render structure with budget logic - allocate from CUT to ROOT.
|
||||
|
||||
ALGORITHM:
|
||||
|
||||
Phase 1: Build path from cut to root
|
||||
- Find the cut element (truncated value or deepest incomplete node)
|
||||
- Build ordered path: [cut_element, parent, grandparent, ..., root]
|
||||
|
||||
Phase 2: Allocate budget
|
||||
- Collect ALL value nodes with their distance to cut
|
||||
- Sort by distance (smaller = closer to cut = higher priority)
|
||||
- Allocate budget to values in this order
|
||||
- When budget < 50: enable summary_mode (affects containers only)
|
||||
|
||||
Phase 3: Render
|
||||
- PATH containers: always render structure
|
||||
- NON-PATH containers in summary_mode: render as <object>/<array>
|
||||
- Values: render if allocated, else type hint
|
||||
|
||||
Returns:
|
||||
Rendered JSON string with budget constraints applied
|
||||
"""
|
||||
# Phase 1: Build path from cut to root
|
||||
pathFromCutToRoot = []
|
||||
self._buildPathFromCutToRootV3(structure, cutPos, [], pathFromCutToRoot)
|
||||
|
||||
pathNodeIds = set(id(node) for node in pathFromCutToRoot)
|
||||
|
||||
# Phase 2: Collect ALL values and allocate budget
|
||||
allValues = []
|
||||
self._collectAllValuesWithDistance(structure, cutPos, allValues)
|
||||
|
||||
# Sort by distance (smaller = closer to cut = higher priority)
|
||||
allValues.sort(key=lambda x: x['distance'])
|
||||
|
||||
# Initialize allocation tracker
|
||||
allocation = BudgetAllocation(
|
||||
path_node_ids=pathNodeIds,
|
||||
allocated_node_ids=set(),
|
||||
summary_mode=False
|
||||
)
|
||||
|
||||
remainingBudget = self.budgetLimit
|
||||
|
||||
# Phase 2a: Allocate PATH values first (truncated values are always rendered)
|
||||
pathValues = [item for item in allValues if id(item['node']) in pathNodeIds]
|
||||
for item in pathValues:
|
||||
node = item['node']
|
||||
nodeType = node.get('type')
|
||||
|
||||
if nodeType == 'truncated_value':
|
||||
allocation.allocated_node_ids.add(id(node))
|
||||
continue
|
||||
|
||||
if nodeType != 'value':
|
||||
continue
|
||||
|
||||
rawValue = node.get('raw', '')
|
||||
valueSize = len(rawValue)
|
||||
|
||||
if valueSize <= remainingBudget:
|
||||
allocation.allocated_node_ids.add(id(node))
|
||||
remainingBudget -= valueSize
|
||||
|
||||
if remainingBudget < 50:
|
||||
allocation.summary_mode = True
|
||||
|
||||
# Phase 2b: Allocate NON-PATH values (skip if path already triggered summary mode)
|
||||
if not allocation.summary_mode:
|
||||
nonPathValues = [item for item in allValues if id(item['node']) not in pathNodeIds]
|
||||
for item in nonPathValues:
|
||||
node = item['node']
|
||||
nodeType = node.get('type')
|
||||
|
||||
if nodeType != 'value':
|
||||
continue
|
||||
|
||||
rawValue = node.get('raw', '')
|
||||
valueSize = len(rawValue)
|
||||
|
||||
if valueSize <= remainingBudget:
|
||||
allocation.allocated_node_ids.add(id(node))
|
||||
remainingBudget -= valueSize
|
||||
|
||||
if remainingBudget < 50 and not allocation.summary_mode:
|
||||
allocation.summary_mode = True
|
||||
|
||||
# Phase 3: Render with allocation info
|
||||
return self._renderNodeV3(structure, 0, allocation)
|
||||
|
||||
|
||||
def _buildPathFromCutToRootV3(self, node: dict, cutPos: int, currentPath: list, resultPath: list) -> bool:
|
||||
"""
|
||||
Recursively find the path from root to cut element, then reverse it.
|
||||
Result path is ordered: [cut_element, parent, ..., root]
|
||||
"""
|
||||
nodeType = node.get('type')
|
||||
startPos = node.get('start_pos', 0)
|
||||
endPos = node.get('end_pos', cutPos + 1)
|
||||
|
||||
pathWithCurrent = currentPath + [node]
|
||||
|
||||
for child in node.get('children', []):
|
||||
if self._buildPathFromCutToRootV3(child, cutPos, pathWithCurrent, resultPath):
|
||||
return True
|
||||
|
||||
if nodeType == 'truncated_value':
|
||||
resultPath.clear()
|
||||
resultPath.extend(reversed(pathWithCurrent))
|
||||
return True
|
||||
|
||||
if nodeType == 'value' and startPos <= cutPos <= endPos:
|
||||
resultPath.clear()
|
||||
resultPath.extend(reversed(pathWithCurrent))
|
||||
return True
|
||||
|
||||
if nodeType in ('object', 'array') and not node.get('complete') and startPos <= cutPos:
|
||||
resultPath.clear()
|
||||
resultPath.extend(reversed(pathWithCurrent))
|
||||
return True
|
||||
|
||||
if nodeType == 'root' and not resultPath:
|
||||
resultPath.clear()
|
||||
resultPath.extend(reversed(pathWithCurrent))
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _collectAllValuesWithDistance(self, node: dict, cutPos: int, result: list, depth: int = 0):
|
||||
"""Collect ALL value nodes with their distance to cut point."""
|
||||
nodeType = node.get('type')
|
||||
|
||||
if nodeType in ('value', 'truncated_value'):
|
||||
endPos = node.get('end_pos', cutPos)
|
||||
distance = cutPos - endPos
|
||||
result.append({
|
||||
'node': node,
|
||||
'distance': distance,
|
||||
'depth': depth
|
||||
})
|
||||
|
||||
for child in node.get('children', []):
|
||||
self._collectAllValuesWithDistance(child, cutPos, result, depth + 1)
|
||||
|
||||
|
||||
def _renderNodeV3(self, node: dict, depth: int, allocation) -> str:
|
||||
"""Render a node with budget allocation info."""
|
||||
nodeType = node.get('type')
|
||||
|
||||
if nodeType == 'root':
|
||||
parts = []
|
||||
for child in node.get('children', []):
|
||||
parts.append(self._renderNodeV3(child, depth, allocation))
|
||||
return '\n'.join(parts)
|
||||
|
||||
elif nodeType == 'object':
|
||||
return self._renderObjectV3(node, depth, allocation)
|
||||
|
||||
elif nodeType == 'array':
|
||||
return self._renderArrayV3(node, depth, allocation)
|
||||
|
||||
elif nodeType == 'value':
|
||||
return self._renderValueV3(node, depth, allocation)
|
||||
|
||||
elif nodeType == 'truncated_value':
|
||||
keyPrefix = f'"{node.get("key")}": ' if node.get('key') else ''
|
||||
return f"{keyPrefix}{node.get('raw', '')}"
|
||||
|
||||
return ''
|
||||
|
||||
|
||||
def _renderObjectV3(self, node: dict, depth: int, allocation) -> str:
|
||||
"""Render object - summary mode non-path objects become <object>."""
|
||||
indentStr = " " * depth
|
||||
innerIndent = " " * (depth + 1)
|
||||
|
||||
keyPrefix = f'"{node.get("key")}": ' if node.get('key') else ''
|
||||
children = node.get('children', [])
|
||||
isOnPath = id(node) in allocation.path_node_ids
|
||||
|
||||
if allocation.summary_mode and not isOnPath:
|
||||
return f"{keyPrefix}<object>"
|
||||
|
||||
if not children:
|
||||
return f"{keyPrefix}{{}}" if node.get('complete') else f"{keyPrefix}{{"
|
||||
|
||||
parts = [f"{keyPrefix}{{"]
|
||||
|
||||
for i, child in enumerate(children):
|
||||
childRendered = self._renderNodeV3(child, depth + 1, allocation)
|
||||
isLast = (i == len(children) - 1)
|
||||
isTruncated = child.get('type') == 'truncated_value'
|
||||
|
||||
if isLast or isTruncated:
|
||||
parts.append(f"{innerIndent}{childRendered}")
|
||||
else:
|
||||
parts.append(f"{innerIndent}{childRendered},")
|
||||
|
||||
if node.get('complete'):
|
||||
parts.append(f"{indentStr}}}")
|
||||
|
||||
return '\n'.join(parts)
|
||||
|
||||
|
||||
def _renderArrayV3(self, node: dict, depth: int, allocation) -> str:
|
||||
"""Render array - summary mode non-path arrays become <array>."""
|
||||
indentStr = " " * depth
|
||||
innerIndent = " " * (depth + 1)
|
||||
|
||||
keyPrefix = f'"{node.get("key")}": ' if node.get('key') else ''
|
||||
children = node.get('children', [])
|
||||
isOnPath = id(node) in allocation.path_node_ids
|
||||
|
||||
if allocation.summary_mode and not isOnPath:
|
||||
return f"{keyPrefix}<array>"
|
||||
|
||||
if not children:
|
||||
return f"{keyPrefix}[]" if node.get('complete') else f"{keyPrefix}["
|
||||
|
||||
parts = [f"{keyPrefix}["]
|
||||
|
||||
for i, child in enumerate(children):
|
||||
childRendered = self._renderNodeV3(child, depth + 1, allocation)
|
||||
isLast = (i == len(children) - 1)
|
||||
isTruncated = child.get('type') == 'truncated_value'
|
||||
|
||||
if isLast or isTruncated:
|
||||
parts.append(f"{innerIndent}{childRendered}")
|
||||
else:
|
||||
parts.append(f"{innerIndent}{childRendered},")
|
||||
|
||||
if node.get('complete'):
|
||||
parts.append(f"{indentStr}]")
|
||||
|
||||
return '\n'.join(parts)
|
||||
|
||||
|
||||
def _renderValueV3(self, node: dict, depth: int, allocation) -> str:
|
||||
"""Render value - if allocated render full, else type hint."""
|
||||
keyPrefix = f'"{node.get("key")}": ' if node.get('key') else ''
|
||||
rawValue = node.get('raw', '""')
|
||||
valueType = node.get('value_type', 'string')
|
||||
|
||||
typeHints = {
|
||||
'string': '<str>',
|
||||
'number': '<number>',
|
||||
'boolean': '<boolean>',
|
||||
'null': '<null>'
|
||||
}
|
||||
typeHint = typeHints.get(valueType, '<value>')
|
||||
|
||||
if id(node) in allocation.allocated_node_ids:
|
||||
return f"{keyPrefix}{rawValue}"
|
||||
else:
|
||||
return f"{keyPrefix}{typeHint}"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ALSO ADD THIS IMPORT AT THE TOP OF YOUR FILE
|
||||
# =============================================================================
|
||||
# from dataclasses import dataclass, field
|
||||
# from typing import Set
|
||||
|
||||
# And add the BudgetAllocation class inside your file or as a nested class
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -32,11 +32,16 @@ Autor: Claude
|
|||
Version: 2.0
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Tuple, List, Optional, Any, Set
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from modules.datamodels.datamodelAi import JsonContinuationContexts
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MODULE CONSTANTS
|
||||
|
|
@ -1442,12 +1447,7 @@ class JsonAnalyzer:
|
|||
return ''
|
||||
|
||||
def _renderObjectV3(self, node: dict, depth: int, allocation: BudgetAllocation) -> str:
|
||||
"""
|
||||
Render object.
|
||||
- PATH containers: always render with structure
|
||||
- NON-PATH containers: only render as <object> if summary_mode AND no allocated children
|
||||
- Otherwise: render with full details (children may have allocated values)
|
||||
"""
|
||||
"""Render object - summary mode non-path objects become <object>."""
|
||||
indentStr = " " * depth
|
||||
innerIndent = " " * (depth + 1)
|
||||
|
||||
|
|
@ -1455,26 +1455,19 @@ class JsonAnalyzer:
|
|||
children = node.get('children', [])
|
||||
isOnPath = id(node) in allocation.path_node_ids
|
||||
|
||||
# Check if any child has allocated values
|
||||
hasAllocatedChildren = False
|
||||
if children:
|
||||
for child in children:
|
||||
if child.get('type') in ('value', 'truncated_value'):
|
||||
if id(child) in allocation.allocated_node_ids:
|
||||
hasAllocatedChildren = True
|
||||
break
|
||||
else:
|
||||
# For containers, check recursively if they have allocated descendants
|
||||
if self._hasAllocatedDescendants(child, allocation):
|
||||
hasAllocatedChildren = True
|
||||
break
|
||||
|
||||
# NON-PATH containers: only render as <object> if summary_mode AND no allocated children
|
||||
# PATH containers always render with structure
|
||||
if not isOnPath and allocation.summary_mode and not hasAllocatedChildren:
|
||||
if allocation.summary_mode and not isOnPath:
|
||||
return f"{keyPrefix}<object>"
|
||||
|
||||
# Render with details (either PATH container or NON-PATH with allocated children)
|
||||
# If object is incomplete and cut is directly here (no incomplete child),
|
||||
# extract exact string from original JSON to preserve formatting
|
||||
if not node.get('complete') and node.get('start_pos') is not None:
|
||||
hasIncompleteChild = any(
|
||||
child.get('type') in ('object', 'array') and not child.get('complete')
|
||||
for child in children
|
||||
)
|
||||
if not hasIncompleteChild:
|
||||
return self.jsonStr[node.get('start_pos'):]
|
||||
|
||||
if not children:
|
||||
return f"{keyPrefix}{{}}" if node.get('complete') else f"{keyPrefix}{{"
|
||||
|
||||
|
|
@ -1495,26 +1488,8 @@ class JsonAnalyzer:
|
|||
|
||||
return '\n'.join(parts)
|
||||
|
||||
def _hasAllocatedDescendants(self, node: dict, allocation: BudgetAllocation) -> bool:
|
||||
"""Check if node or any of its descendants have allocated values."""
|
||||
nodeType = node.get('type')
|
||||
|
||||
if nodeType in ('value', 'truncated_value'):
|
||||
return id(node) in allocation.allocated_node_ids
|
||||
|
||||
for child in node.get('children', []):
|
||||
if self._hasAllocatedDescendants(child, allocation):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _renderArrayV3(self, node: dict, depth: int, allocation: BudgetAllocation) -> str:
|
||||
"""
|
||||
Render array.
|
||||
- PATH containers: always render with structure
|
||||
- NON-PATH containers: only render as <array> if summary_mode AND no allocated children
|
||||
- Otherwise: render with full details (children may have allocated values)
|
||||
"""
|
||||
"""Render array - summary mode non-path arrays become <array>."""
|
||||
indentStr = " " * depth
|
||||
innerIndent = " " * (depth + 1)
|
||||
|
||||
|
|
@ -1522,26 +1497,19 @@ class JsonAnalyzer:
|
|||
children = node.get('children', [])
|
||||
isOnPath = id(node) in allocation.path_node_ids
|
||||
|
||||
# Check if any child has allocated values
|
||||
hasAllocatedChildren = False
|
||||
if children:
|
||||
for child in children:
|
||||
if child.get('type') in ('value', 'truncated_value'):
|
||||
if id(child) in allocation.allocated_node_ids:
|
||||
hasAllocatedChildren = True
|
||||
break
|
||||
else:
|
||||
# For containers, check recursively if they have allocated descendants
|
||||
if self._hasAllocatedDescendants(child, allocation):
|
||||
hasAllocatedChildren = True
|
||||
break
|
||||
|
||||
# NON-PATH containers: only render as <array> if summary_mode AND no allocated children
|
||||
# PATH containers always render with structure
|
||||
if not isOnPath and allocation.summary_mode and not hasAllocatedChildren:
|
||||
if allocation.summary_mode and not isOnPath:
|
||||
return f"{keyPrefix}<array>"
|
||||
|
||||
# Render with details (either PATH container or NON-PATH with allocated children)
|
||||
# If array is incomplete and cut is directly here (no incomplete child),
|
||||
# extract exact string from original JSON to preserve formatting
|
||||
if not node.get('complete') and node.get('start_pos') is not None:
|
||||
hasIncompleteChild = any(
|
||||
child.get('type') in ('object', 'array') and not child.get('complete')
|
||||
for child in children
|
||||
)
|
||||
if not hasIncompleteChild:
|
||||
return self.jsonStr[node.get('start_pos'):]
|
||||
|
||||
if not children:
|
||||
return f"{keyPrefix}[]" if node.get('complete') else f"{keyPrefix}["
|
||||
|
||||
|
|
@ -1669,6 +1637,276 @@ def extractContinuationContexts(
|
|||
return getJsonContinuationContext(truncatedJson)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# JSON REPAIR FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
def _repairInternalJsonErrors(jsonStr: str) -> str:
|
||||
"""
|
||||
Repair internal JSON errors WITHOUT touching incomplete structures at cut point.
|
||||
|
||||
This function fixes common internal JSON issues:
|
||||
- Invalid escape sequences (e.g., \\x, \\u without proper hex)
|
||||
- Unescaped control characters
|
||||
- Invalid Unicode characters
|
||||
- Trailing commas before closing brackets/braces
|
||||
- Comments (// and /* */)
|
||||
- Single quotes instead of double quotes (outside of string values)
|
||||
- Unquoted keys
|
||||
|
||||
IMPORTANT: Does NOT modify incomplete structures at the end of the JSON.
|
||||
Those are handled separately by structure closing logic.
|
||||
|
||||
Args:
|
||||
jsonStr: JSON string that may have internal errors
|
||||
|
||||
Returns:
|
||||
Repaired JSON string with internal errors fixed
|
||||
"""
|
||||
if not jsonStr or not jsonStr.strip():
|
||||
return jsonStr
|
||||
|
||||
result = jsonStr
|
||||
|
||||
# Fix 1: Remove BOM and normalize whitespace at start
|
||||
if result.startswith('\ufeff'):
|
||||
result = result[1:]
|
||||
|
||||
# Fix 2: Normalize smart quotes to straight quotes
|
||||
result = result.replace('"', '"').replace('"', '"')
|
||||
result = result.replace(''', "'").replace(''', "'")
|
||||
|
||||
# Fix 3: Remove JavaScript-style comments (but be careful not to break strings)
|
||||
result = _removeJsonComments(result)
|
||||
|
||||
# Fix 4: Fix invalid escape sequences
|
||||
result = _fixInvalidEscapeSequences(result)
|
||||
|
||||
# Fix 5: Remove trailing commas before ] or }
|
||||
result = _removeTrailingCommas(result)
|
||||
|
||||
# Fix 6: Fix unquoted keys (simple cases only)
|
||||
result = _fixUnquotedKeys(result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _removeJsonComments(jsonStr: str) -> str:
|
||||
"""Remove JavaScript-style comments from JSON, preserving strings."""
|
||||
result = []
|
||||
i = 0
|
||||
inString = False
|
||||
escaped = False
|
||||
|
||||
while i < len(jsonStr):
|
||||
char = jsonStr[i]
|
||||
|
||||
if escaped:
|
||||
result.append(char)
|
||||
escaped = False
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if char == '\\' and inString:
|
||||
result.append(char)
|
||||
escaped = True
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
inString = not inString
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if inString:
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Check for // comment
|
||||
if char == '/' and i + 1 < len(jsonStr) and jsonStr[i + 1] == '/':
|
||||
# Skip until end of line
|
||||
while i < len(jsonStr) and jsonStr[i] != '\n':
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Check for /* */ comment
|
||||
if char == '/' and i + 1 < len(jsonStr) and jsonStr[i + 1] == '*':
|
||||
i += 2
|
||||
while i + 1 < len(jsonStr):
|
||||
if jsonStr[i] == '*' and jsonStr[i + 1] == '/':
|
||||
i += 2
|
||||
break
|
||||
i += 1
|
||||
continue
|
||||
|
||||
result.append(char)
|
||||
i += 1
|
||||
|
||||
return ''.join(result)
|
||||
|
||||
|
||||
def _fixInvalidEscapeSequences(jsonStr: str) -> str:
|
||||
"""Fix invalid escape sequences in JSON strings."""
|
||||
result = []
|
||||
i = 0
|
||||
inString = False
|
||||
|
||||
while i < len(jsonStr):
|
||||
char = jsonStr[i]
|
||||
|
||||
if char == '"' and (i == 0 or jsonStr[i - 1] != '\\'):
|
||||
inString = not inString
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if inString and char == '\\' and i + 1 < len(jsonStr):
|
||||
nextChar = jsonStr[i + 1]
|
||||
|
||||
# Valid JSON escape sequences: \", \\, \/, \b, \f, \n, \r, \t, \uXXXX
|
||||
validEscapes = ['"', '\\', '/', 'b', 'f', 'n', 'r', 't', 'u']
|
||||
|
||||
if nextChar in validEscapes:
|
||||
if nextChar == 'u':
|
||||
# Check if followed by 4 hex digits
|
||||
if i + 5 < len(jsonStr) and all(c in '0123456789abcdefABCDEF' for c in jsonStr[i + 2:i + 6]):
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
# Invalid \u sequence - escape the backslash
|
||||
result.append('\\')
|
||||
result.append('\\')
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
else:
|
||||
# Invalid escape - escape the backslash
|
||||
result.append('\\')
|
||||
result.append('\\')
|
||||
i += 1
|
||||
continue
|
||||
|
||||
result.append(char)
|
||||
i += 1
|
||||
|
||||
return ''.join(result)
|
||||
|
||||
|
||||
def _removeTrailingCommas(jsonStr: str) -> str:
|
||||
"""Remove trailing commas before ] or } (not valid in JSON)."""
|
||||
# Pattern: comma followed by whitespace and ] or }
|
||||
result = re.sub(r',(\s*[}\]])', r'\1', jsonStr)
|
||||
return result
|
||||
|
||||
|
||||
def _fixUnquotedKeys(jsonStr: str) -> str:
|
||||
"""
|
||||
Fix simple unquoted keys in JSON objects.
|
||||
Only handles simple cases to avoid breaking valid JSON.
|
||||
"""
|
||||
# Pattern: { or , followed by whitespace and an unquoted identifier and :
|
||||
# Be conservative - only fix clear cases
|
||||
|
||||
result = []
|
||||
i = 0
|
||||
inString = False
|
||||
escaped = False
|
||||
|
||||
while i < len(jsonStr):
|
||||
char = jsonStr[i]
|
||||
|
||||
if escaped:
|
||||
result.append(char)
|
||||
escaped = False
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if char == '\\' and inString:
|
||||
result.append(char)
|
||||
escaped = True
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if char == '"':
|
||||
inString = not inString
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if inString:
|
||||
result.append(char)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Check for unquoted key after { or ,
|
||||
if char in '{,' and i + 1 < len(jsonStr):
|
||||
result.append(char)
|
||||
i += 1
|
||||
|
||||
# Skip whitespace
|
||||
while i < len(jsonStr) and jsonStr[i] in ' \t\n\r':
|
||||
result.append(jsonStr[i])
|
||||
i += 1
|
||||
|
||||
if i >= len(jsonStr):
|
||||
continue
|
||||
|
||||
# Check if next is an unquoted identifier (starts with letter or _)
|
||||
if jsonStr[i] not in '"{[' and (jsonStr[i].isalpha() or jsonStr[i] == '_'):
|
||||
# Collect the identifier
|
||||
keyStart = i
|
||||
while i < len(jsonStr) and (jsonStr[i].isalnum() or jsonStr[i] == '_'):
|
||||
i += 1
|
||||
key = jsonStr[keyStart:i]
|
||||
|
||||
# Skip whitespace
|
||||
while i < len(jsonStr) and jsonStr[i] in ' \t\n\r':
|
||||
i += 1
|
||||
|
||||
# Check if followed by :
|
||||
if i < len(jsonStr) and jsonStr[i] == ':':
|
||||
# This was an unquoted key - quote it
|
||||
result.append('"')
|
||||
result.append(key)
|
||||
result.append('"')
|
||||
else:
|
||||
# Not a key, put back as-is
|
||||
result.append(key)
|
||||
continue
|
||||
|
||||
result.append(char)
|
||||
i += 1
|
||||
|
||||
return ''.join(result)
|
||||
|
||||
|
||||
def _tryParseJson(jsonStr: str) -> tuple:
|
||||
"""
|
||||
Try to parse JSON string and return (parsed, error).
|
||||
|
||||
Returns:
|
||||
Tuple of (parsed_object, error_string)
|
||||
- If successful: (parsed_object, None)
|
||||
- If failed: (None, error_message)
|
||||
"""
|
||||
if not jsonStr or not jsonStr.strip():
|
||||
return None, "Empty JSON string"
|
||||
|
||||
try:
|
||||
parsed = json.loads(jsonStr)
|
||||
return parsed, None
|
||||
except json.JSONDecodeError as e:
|
||||
return None, str(e)
|
||||
except Exception as e:
|
||||
return None, str(e)
|
||||
|
||||
|
||||
# Convenience function with named results
|
||||
def getContexts(
|
||||
truncatedJson: str
|
||||
|
|
@ -1678,28 +1916,86 @@ def getContexts(
|
|||
|
||||
Uses module constants BUDGET_LIMIT and OVERLAP_MAX_CHARS.
|
||||
|
||||
This function:
|
||||
1. Extracts continuation contexts (overlap, hierarchy, completePart)
|
||||
2. Tries to parse completePart as JSON
|
||||
3. If parsing fails, repairs internal errors and retries
|
||||
4. Sets jsonParsingSuccess to indicate if completePart is valid JSON
|
||||
5. Sets overlapContext="" if JSON is complete (no cut point)
|
||||
|
||||
IMPORTANT: overlapContext="" signals that JSON is complete (no more data expected).
|
||||
This happens when the original JSON is already valid (no structures needed closing).
|
||||
|
||||
Args:
|
||||
truncatedJson: The truncated JSON string
|
||||
|
||||
Returns:
|
||||
JsonContinuationContexts Pydantic model with:
|
||||
- overlapContext: The innermost object/element containing the cut
|
||||
Empty string "" if JSON is complete (no cut point)
|
||||
- hierarchyContext: Full structure WITHOUT budget limitations (for internal use)
|
||||
- hierarchyContextForPrompt: Full structure WITH budget limitations (for prompts)
|
||||
- completePart: Valid JSON with all structures properly closed
|
||||
- jsonParsingSuccess: True if completePart is valid parseable JSON
|
||||
|
||||
Example:
|
||||
>>> json_str = '{"users": [{"name": "John", "bio": "Hello Wor'
|
||||
>>> contexts = getContexts(json_str)
|
||||
>>> print(contexts.overlapContext)
|
||||
>>> print(contexts.hierarchyContext)
|
||||
>>> print(contexts.hierarchyContextForPrompt)
|
||||
>>> print(contexts.completePart)
|
||||
>>> print(contexts.overlapContext) # Contains cut point context
|
||||
>>> print(contexts.jsonParsingSuccess)
|
||||
|
||||
>>> complete_json = '{"users": [{"name": "John"}]}'
|
||||
>>> contexts = getContexts(complete_json)
|
||||
>>> print(contexts.overlapContext) # "" (empty - JSON is complete)
|
||||
>>> print(contexts.jsonParsingSuccess) # True
|
||||
"""
|
||||
# First, check if original JSON is already complete (parseable without modification)
|
||||
jsonIsComplete = False
|
||||
if truncatedJson and truncatedJson.strip():
|
||||
parsed, error = _tryParseJson(truncatedJson.strip())
|
||||
if error is None:
|
||||
jsonIsComplete = True
|
||||
logger.debug("Original JSON is already complete (no cut point)")
|
||||
|
||||
# Extract contexts
|
||||
overlap, hierarchy, hierarchyForPrompt, completePart = extractContinuationContexts(truncatedJson)
|
||||
|
||||
# If JSON is complete (no cut point), set overlapContext to empty string
|
||||
# This signals that no more continuation is needed
|
||||
if jsonIsComplete:
|
||||
overlap = ""
|
||||
logger.debug("Setting overlapContext='' (JSON is complete)")
|
||||
|
||||
# Try to parse completePart as JSON
|
||||
jsonParsingSuccess = False
|
||||
|
||||
if completePart and completePart.strip():
|
||||
# First attempt: parse as-is
|
||||
parsed, error = _tryParseJson(completePart)
|
||||
|
||||
if error is None:
|
||||
jsonParsingSuccess = True
|
||||
else:
|
||||
# Second attempt: repair internal errors and retry
|
||||
logger.debug(f"Initial parse failed: {error}, attempting repair")
|
||||
repairedCompletePart = _repairInternalJsonErrors(completePart)
|
||||
|
||||
parsed, error = _tryParseJson(repairedCompletePart)
|
||||
|
||||
if error is None:
|
||||
# Repair succeeded - use repaired version
|
||||
completePart = repairedCompletePart
|
||||
jsonParsingSuccess = True
|
||||
logger.debug("JSON repair successful")
|
||||
else:
|
||||
# Repair also failed - keep original completePart, mark as failed
|
||||
logger.debug(f"JSON repair also failed: {error}")
|
||||
jsonParsingSuccess = False
|
||||
|
||||
return JsonContinuationContexts(
|
||||
overlapContext=overlap,
|
||||
hierarchyContext=hierarchy,
|
||||
hierarchyContextForPrompt=hierarchyForPrompt,
|
||||
completePart=completePart
|
||||
completePart=completePart,
|
||||
jsonParsingSuccess=jsonParsingSuccess
|
||||
)
|
||||
|
|
|
|||
|
|
@ -126,170 +126,97 @@ class JsonSplitMergeTester12:
|
|||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "large_customers.json",
|
||||
"data": self._createLargeCustomersData()
|
||||
},
|
||||
{
|
||||
"name": "large_products.json",
|
||||
"data": self._createLargeProductsData()
|
||||
},
|
||||
{
|
||||
"name": "large_documents.json",
|
||||
"data": self._createLargeDocumentsData()
|
||||
},
|
||||
{
|
||||
"name": "table_example.json",
|
||||
"data": self._loadTableJsonExample()
|
||||
},
|
||||
{
|
||||
"name": "complete_json.json",
|
||||
"data": {
|
||||
"status": "complete",
|
||||
"message": "This is a complete, valid JSON object",
|
||||
"data": {
|
||||
"items": [1, 2, 3, 4, 5],
|
||||
"metadata": {
|
||||
"version": "1.0",
|
||||
"timestamp": "2025-01-05T12:00:00Z"
|
||||
}
|
||||
}
|
||||
},
|
||||
"isComplete": True # Flag to indicate this is complete JSON (not cut)
|
||||
},
|
||||
{
|
||||
"name": "json_with_comments.json",
|
||||
"data": None, # Will be set as string with comments
|
||||
"jsonString": '''{
|
||||
// This is a single-line comment
|
||||
"name": "Test",
|
||||
"value": 42,
|
||||
/* This is a multi-line comment
|
||||
spanning multiple lines */
|
||||
"items": [1, 2, 3],
|
||||
"nested": {
|
||||
// Another comment
|
||||
"key": "value"
|
||||
}
|
||||
}''',
|
||||
"hasComments": True
|
||||
},
|
||||
{
|
||||
"name": "json_with_trailing_comma.json",
|
||||
"data": None, # Will be set as string with trailing comma
|
||||
"jsonString": '''{
|
||||
"name": "Test",
|
||||
"value": 42,
|
||||
"items": [1, 2, 3,],
|
||||
"nested": {
|
||||
"key": "value",
|
||||
}
|
||||
}''',
|
||||
"hasTrailingComma": True
|
||||
},
|
||||
{
|
||||
"name": "json_with_unquoted_keys.json",
|
||||
"data": None, # Will be set as string with unquoted keys
|
||||
"jsonString": '''{
|
||||
name: "Test",
|
||||
value: 42,
|
||||
items: [1, 2, 3],
|
||||
nested: {
|
||||
key: "value"
|
||||
}
|
||||
}''',
|
||||
"hasUnquotedKeys": True
|
||||
},
|
||||
{
|
||||
"name": "json_with_invalid_escape.json",
|
||||
"data": None, # Will be set as string with invalid escape
|
||||
"jsonString": '''{
|
||||
"name": "Test\\xInvalid",
|
||||
"value": 42,
|
||||
"description": "This has \\u invalid escape"
|
||||
}''',
|
||||
"hasInvalidEscape": True
|
||||
},
|
||||
{
|
||||
"name": "json_mixed_errors.json",
|
||||
"data": None, # Will be set as string with multiple errors
|
||||
"jsonString": '''{
|
||||
// Comment here
|
||||
name: "Test", // Unquoted key
|
||||
"value": 42,
|
||||
"items": [1, 2, 3,], // Trailing comma
|
||||
"description": "Has \\x invalid escape",
|
||||
"nested": {
|
||||
key: "value", // Unquoted key and trailing comma
|
||||
}
|
||||
}''',
|
||||
"hasMixedErrors": True
|
||||
}
|
||||
]
|
||||
|
||||
return testFiles
|
||||
|
||||
def _createLargeCustomersData(self) -> Dict[str, Any]:
|
||||
"""Create a large customers dataset for budget testing."""
|
||||
customers = []
|
||||
# Create 100 customers with long descriptions
|
||||
for i in range(100):
|
||||
customers.append({
|
||||
"id": i + 1,
|
||||
"name": f"Customer {i + 1}",
|
||||
"email": f"customer{i+1}@example.com",
|
||||
"phone": f"+1{5550000000 + i}",
|
||||
"address": f"{100 + i} Main Street, City {i % 10}, State {i % 5}, ZIP {10000 + i}",
|
||||
"description": f"This is a detailed description for customer {i + 1}. " * 10 +
|
||||
f"They have been a loyal customer since {2000 + (i % 25)}. " +
|
||||
f"Their preferences include various products and services. " * 5,
|
||||
"orders": [
|
||||
{
|
||||
"orderId": f"ORD-{i+1}-{j+1}",
|
||||
"date": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
|
||||
"total": round(100.0 + (i * 10) + (j * 5), 2),
|
||||
"items": [
|
||||
{
|
||||
"productId": f"PROD-{k+1}",
|
||||
"quantity": (k % 5) + 1,
|
||||
"price": round(10.0 + k * 2, 2)
|
||||
}
|
||||
for k in range(3)
|
||||
]
|
||||
}
|
||||
for j in range(5)
|
||||
],
|
||||
"metadata": {
|
||||
"created": f"2020-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
|
||||
"lastLogin": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
|
||||
"tags": [f"tag-{i % 10}", f"category-{i % 5}", f"segment-{i % 3}"]
|
||||
}
|
||||
})
|
||||
return {"customers": customers}
|
||||
|
||||
def _createLargeProductsData(self) -> Dict[str, Any]:
|
||||
"""Create a large products dataset for budget testing."""
|
||||
products = []
|
||||
# Create 200 products with detailed information
|
||||
categories = ["Electronics", "Clothing", "Books", "Home & Garden", "Sports", "Toys", "Automotive", "Health"]
|
||||
for i in range(200):
|
||||
category = categories[i % len(categories)]
|
||||
products.append({
|
||||
"id": f"PROD-{i+1:04d}",
|
||||
"name": f"Product {i+1} - {category}",
|
||||
"category": category,
|
||||
"price": round(10.0 + (i * 2.5), 2),
|
||||
"cost": round(5.0 + (i * 1.5), 2),
|
||||
"inStock": i % 3 != 0,
|
||||
"stockQuantity": (i % 100) * 10,
|
||||
"description": f"This is a comprehensive product description for Product {i+1}. " * 15 +
|
||||
f"It belongs to the {category} category and offers excellent value. " * 10 +
|
||||
f"Features include: feature-1, feature-2, feature-3, and many more. " * 5,
|
||||
"specifications": {
|
||||
"weight": f"{1.0 + (i % 10)} kg",
|
||||
"dimensions": f"{10 + (i % 20)}x{5 + (i % 15)}x{3 + (i % 10)} cm",
|
||||
"color": ["red", "blue", "green", "black", "white"][i % 5],
|
||||
"material": ["plastic", "metal", "wood", "fabric"][i % 4],
|
||||
"warranty": f"{1 + (i % 5)} years"
|
||||
},
|
||||
"reviews": [
|
||||
{
|
||||
"userId": f"USER-{j+1}",
|
||||
"rating": (j % 5) + 1,
|
||||
"comment": f"Review {j+1} for product {i+1}: " + "This is a detailed review comment. " * 10,
|
||||
"date": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}"
|
||||
}
|
||||
for j in range(3)
|
||||
],
|
||||
"relatedProducts": [f"PROD-{k+1:04d}" for k in range(max(0, i-2), min(200, i+3)) if k != i]
|
||||
})
|
||||
return {"products": products}
|
||||
|
||||
def _createLargeDocumentsData(self) -> Dict[str, Any]:
|
||||
"""Create a large documents dataset for budget testing."""
|
||||
documents = []
|
||||
# Create 50 documents with nested structures
|
||||
for i in range(50):
|
||||
sections = []
|
||||
for j in range(10):
|
||||
elements = []
|
||||
for k in range(5):
|
||||
if k % 2 == 0:
|
||||
elements.append({
|
||||
"type": "heading",
|
||||
"level": (k % 3) + 1,
|
||||
"content": {
|
||||
"text": f"Section {j+1} Heading {k+1} for Document {i+1}",
|
||||
"style": "bold"
|
||||
}
|
||||
})
|
||||
else:
|
||||
elements.append({
|
||||
"type": "paragraph",
|
||||
"content": {
|
||||
"text": f"This is paragraph {k+1} in section {j+1} of document {i+1}. " * 20 +
|
||||
f"It contains detailed information about various topics. " * 15 +
|
||||
f"The content is structured and well-organized. " * 10
|
||||
}
|
||||
})
|
||||
|
||||
sections.append({
|
||||
"id": f"sec-{i+1}-{j+1}",
|
||||
"title": f"Section {j+1}",
|
||||
"content_type": "mixed",
|
||||
"elements": elements,
|
||||
"metadata": {
|
||||
"created": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
|
||||
"modified": f"2024-{(j % 12) + 1:02d}-{(j % 28) + 1:02d}",
|
||||
"author": f"Author-{(i % 10) + 1}",
|
||||
"tags": [f"tag-{j % 10}", f"category-{i % 5}"]
|
||||
}
|
||||
})
|
||||
|
||||
documents.append({
|
||||
"id": f"doc-{i+1:03d}",
|
||||
"title": f"Document {i+1} - Comprehensive Report",
|
||||
"description": f"This is a comprehensive document with detailed information. " * 30 +
|
||||
f"It covers multiple topics and sections. " * 20 +
|
||||
f"The content is extensive and well-structured. " * 15,
|
||||
"sections": sections,
|
||||
"metadata": {
|
||||
"created": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
|
||||
"modified": f"2024-{(i % 12) + 1:02d}-{(i % 28) + 1:02d}",
|
||||
"author": f"Author-{(i % 10) + 1}",
|
||||
"version": f"1.{(i % 10)}",
|
||||
"status": ["draft", "review", "published"][i % 3],
|
||||
"tags": [f"tag-{i % 20}" for _ in range(5)]
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
"metadata": {
|
||||
"title": "Large Document Collection",
|
||||
"description": "A collection of 50 comprehensive documents for testing budget limits",
|
||||
"totalDocuments": 50,
|
||||
"created": "2024-01-01"
|
||||
},
|
||||
"documents": documents
|
||||
}
|
||||
|
||||
def _loadTableJsonExample(self) -> Dict[str, Any]:
|
||||
"""Load the table JSON example from the debug prompts file."""
|
||||
try:
|
||||
|
|
@ -460,23 +387,51 @@ class JsonSplitMergeTester12:
|
|||
async def testJsonSplitMerge(self, jsonFile: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Test splitting and merging a single JSON file."""
|
||||
fileName = jsonFile["name"]
|
||||
originalData = jsonFile["data"]
|
||||
|
||||
# Check if this is a complete JSON test (no cut)
|
||||
isComplete = jsonFile.get("isComplete", False)
|
||||
|
||||
# Check if this is a JSON string with errors (not from data dict)
|
||||
jsonString = jsonFile.get("jsonString")
|
||||
if jsonString:
|
||||
# Use the provided JSON string directly (may have errors)
|
||||
originalJsonString = jsonString
|
||||
originalData = None # No original data for error tests
|
||||
else:
|
||||
# Convert data dict to JSON string
|
||||
originalData = jsonFile["data"]
|
||||
originalJsonString = json.dumps(originalData, indent=2, ensure_ascii=False)
|
||||
|
||||
originalSize = len(originalJsonString)
|
||||
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log(f"TESTING JSON SPLIT AND MERGE: {fileName}")
|
||||
testType = "COMPLETE JSON" if isComplete else ("JSON WITH ERRORS" if jsonString else "SPLIT JSON")
|
||||
self._log(f"TESTING {testType}: {fileName}")
|
||||
self._log("="*80)
|
||||
|
||||
# Convert to JSON string
|
||||
originalJsonString = json.dumps(originalData, indent=2, ensure_ascii=False)
|
||||
originalSize = len(originalJsonString)
|
||||
|
||||
# Log original JSON
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("ORIGINAL JSON")
|
||||
self._log("="*80)
|
||||
self._log(f"JSON length: {originalSize} characters")
|
||||
if isComplete:
|
||||
self._log(" ⚠️ This is COMPLETE JSON (not cut) - testing overlapContext='' detection")
|
||||
if jsonString:
|
||||
errorType = []
|
||||
if jsonFile.get("hasComments"):
|
||||
errorType.append("comments")
|
||||
if jsonFile.get("hasTrailingComma"):
|
||||
errorType.append("trailing commas")
|
||||
if jsonFile.get("hasUnquotedKeys"):
|
||||
errorType.append("unquoted keys")
|
||||
if jsonFile.get("hasInvalidEscape"):
|
||||
errorType.append("invalid escapes")
|
||||
if jsonFile.get("hasMixedErrors"):
|
||||
errorType.append("mixed errors")
|
||||
if errorType:
|
||||
self._log(f" ⚠️ This JSON has errors: {', '.join(errorType)} - testing repair function")
|
||||
self._log("")
|
||||
self._log("Full JSON content:")
|
||||
self._log("-"*80)
|
||||
|
|
@ -491,39 +446,63 @@ class JsonSplitMergeTester12:
|
|||
for line in jsonLines:
|
||||
self._log(line)
|
||||
|
||||
# Split JSON at random position (simulating AI response cut)
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("SPLITTING JSON AT RANDOM POSITION (SIMULATING AI RESPONSE CUT)")
|
||||
self._log("="*80)
|
||||
|
||||
# Find random cut position (not at start or end)
|
||||
import random
|
||||
minCutPos = max(100, originalSize // 10) # At least 10% from start
|
||||
maxCutPos = min(originalSize - 100, originalSize * 9 // 10) # At least 10% from end
|
||||
cutPosition = random.randint(minCutPos, maxCutPos)
|
||||
|
||||
# Get part from start to cut
|
||||
partContent = originalJsonString[:cutPosition]
|
||||
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("PART (from start to cut):")
|
||||
self._log("="*80)
|
||||
self._log(f"Cut position: {cutPosition} characters")
|
||||
self._log(f"Part length: {len(partContent)} characters")
|
||||
self._log("")
|
||||
self._log("Part content:")
|
||||
partLines = partContent.split('\n')
|
||||
if len(partLines) > 30:
|
||||
for line in partLines[:15]:
|
||||
self._log(f" {line}")
|
||||
self._log(f" ... ({len(partLines) - 30} lines omitted) ...")
|
||||
for line in partLines[-15:]:
|
||||
self._log(f" {line}")
|
||||
# Handle complete JSON, JSON with errors, vs split JSON
|
||||
if isComplete or jsonString:
|
||||
# For complete JSON or JSON with errors, use the full string (no cut)
|
||||
# We want to test repair on the full error-containing JSON
|
||||
partContent = originalJsonString
|
||||
cutPosition = None # No cut
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
if isComplete:
|
||||
self._log("COMPLETE JSON TEST (NO CUT)")
|
||||
self._log("="*80)
|
||||
self._log(" Testing that getContexts() detects complete JSON and sets overlapContext=''")
|
||||
else:
|
||||
self._log("JSON WITH ERRORS TEST (NO CUT)")
|
||||
self._log("="*80)
|
||||
self._log(" Testing that getContexts() repairs the errors and produces valid JSON")
|
||||
else:
|
||||
for line in partLines:
|
||||
self._log(f" {line}")
|
||||
# Split JSON at random position (simulating AI response cut)
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("SPLITTING JSON AT RANDOM POSITION (SIMULATING AI RESPONSE CUT)")
|
||||
self._log("="*80)
|
||||
|
||||
# Find random cut position (not at start or end)
|
||||
import random
|
||||
minCutPos = max(100, originalSize // 10) # At least 10% from start
|
||||
maxCutPos = min(originalSize - 100, originalSize * 9 // 10) # At least 10% from end
|
||||
|
||||
# Ensure valid range
|
||||
if maxCutPos <= minCutPos:
|
||||
# For small JSON, just cut in the middle
|
||||
cutPosition = originalSize // 2
|
||||
else:
|
||||
cutPosition = random.randint(minCutPos, maxCutPos)
|
||||
|
||||
# Get part from start to cut
|
||||
partContent = originalJsonString[:cutPosition]
|
||||
|
||||
if not isComplete:
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("PART (from start to cut):")
|
||||
self._log("="*80)
|
||||
self._log(f"Cut position: {cutPosition} characters")
|
||||
self._log(f"Part length: {len(partContent)} characters")
|
||||
self._log("")
|
||||
self._log("Part content:")
|
||||
partLines = partContent.split('\n')
|
||||
if len(partLines) > 30:
|
||||
for line in partLines[:15]:
|
||||
self._log(f" {line}")
|
||||
self._log(f" ... ({len(partLines) - 30} lines omitted) ...")
|
||||
for line in partLines[-15:]:
|
||||
self._log(f" {line}")
|
||||
else:
|
||||
for line in partLines:
|
||||
self._log(f" {line}")
|
||||
|
||||
# Generate contexts using getContexts()
|
||||
self._log("")
|
||||
|
|
@ -590,12 +569,23 @@ class JsonSplitMergeTester12:
|
|||
for line in completeLines:
|
||||
self._log(f" {line}")
|
||||
|
||||
# Validate completePart as JSON
|
||||
# Validate completePart as JSON and check overlapContext
|
||||
self._log("")
|
||||
self._log("="*80)
|
||||
self._log("VALIDATING COMPLETE PART AS JSON:")
|
||||
self._log("VALIDATION RESULTS:")
|
||||
self._log("="*80)
|
||||
|
||||
# Check overlapContext for complete JSON
|
||||
if isComplete:
|
||||
if contexts.overlapContext == "":
|
||||
self._log(" ✅ overlapContext is empty (correct for complete JSON)")
|
||||
else:
|
||||
self._log(f" ❌ overlapContext is NOT empty: '{contexts.overlapContext[:50]}...'")
|
||||
self._log(" Expected empty string for complete JSON")
|
||||
|
||||
# Validate completePart as JSON
|
||||
self._log("")
|
||||
self._log("VALIDATING COMPLETE PART AS JSON:")
|
||||
isValidJson = False
|
||||
parsedCompletePart = None
|
||||
jsonError = None
|
||||
|
|
@ -606,34 +596,40 @@ class JsonSplitMergeTester12:
|
|||
self._log(" ✅ completePart is valid JSON")
|
||||
self._log(f" Parsed type: {type(parsedCompletePart).__name__}")
|
||||
|
||||
# Compare with truncated JSON (not original) - parse the truncated part to compare
|
||||
from modules.shared.jsonUtils import closeJsonStructures, tryParseJson
|
||||
# For error tests, verify repair worked
|
||||
if jsonString:
|
||||
self._log(" ✅ JSON repair successful - errors were fixed")
|
||||
|
||||
# Try to parse the truncated JSON part (with structures closed)
|
||||
truncatedClosed = closeJsonStructures(partContent)
|
||||
truncatedParsed, truncatedError, _ = tryParseJson(truncatedClosed)
|
||||
|
||||
if truncatedParsed is not None:
|
||||
# Compare completePart with the parsed truncated JSON
|
||||
if isinstance(parsedCompletePart, dict) and isinstance(truncatedParsed, dict):
|
||||
comparison = self.compareJson(truncatedParsed, parsedCompletePart)
|
||||
self._log(f" Comparison with truncated JSON (at cut position {cutPosition}):")
|
||||
self._log(f" Exact match: {comparison['exactMatch']}")
|
||||
self._log(f" Size match: {comparison['sizeMatch']}")
|
||||
if comparison['differences']:
|
||||
self._log(f" Differences found: {len(comparison['differences'])}")
|
||||
for diff in comparison['differences'][:10]: # Show first 10 differences
|
||||
self._log(f" - {diff}")
|
||||
if len(comparison['differences']) > 10:
|
||||
self._log(f" ... ({len(comparison['differences']) - 10} more differences)")
|
||||
# For split JSON, compare with truncated JSON
|
||||
if not isComplete and not jsonString:
|
||||
# Compare with truncated JSON (not original) - parse the truncated part to compare
|
||||
from modules.shared.jsonUtils import closeJsonStructures, tryParseJson
|
||||
|
||||
# Try to parse the truncated JSON part (with structures closed)
|
||||
truncatedClosed = closeJsonStructures(partContent)
|
||||
truncatedParsed, truncatedError, _ = tryParseJson(truncatedClosed)
|
||||
|
||||
if truncatedParsed is not None:
|
||||
# Compare completePart with the parsed truncated JSON
|
||||
if isinstance(parsedCompletePart, dict) and isinstance(truncatedParsed, dict):
|
||||
comparison = self.compareJson(truncatedParsed, parsedCompletePart)
|
||||
self._log(f" Comparison with truncated JSON (at cut position {cutPosition}):")
|
||||
self._log(f" Exact match: {comparison['exactMatch']}")
|
||||
self._log(f" Size match: {comparison['sizeMatch']}")
|
||||
if comparison['differences']:
|
||||
self._log(f" Differences found: {len(comparison['differences'])}")
|
||||
for diff in comparison['differences'][:10]: # Show first 10 differences
|
||||
self._log(f" - {diff}")
|
||||
if len(comparison['differences']) > 10:
|
||||
self._log(f" ... ({len(comparison['differences']) - 10} more differences)")
|
||||
else:
|
||||
self._log(" No differences found - completePart matches truncated JSON structure")
|
||||
elif isinstance(parsedCompletePart, list) and isinstance(truncatedParsed, list):
|
||||
self._log(f" Both are lists: truncated={len(truncatedParsed)} items, completePart={len(parsedCompletePart)} items")
|
||||
else:
|
||||
self._log(" No differences found - completePart matches truncated JSON structure")
|
||||
elif isinstance(parsedCompletePart, list) and isinstance(truncatedParsed, list):
|
||||
self._log(f" Both are lists: truncated={len(truncatedParsed)} items, completePart={len(parsedCompletePart)} items")
|
||||
self._log(f" Different types: truncated={type(truncatedParsed).__name__}, completePart={type(parsedCompletePart).__name__}")
|
||||
else:
|
||||
self._log(f" Different types: truncated={type(truncatedParsed).__name__}, completePart={type(parsedCompletePart).__name__}")
|
||||
else:
|
||||
self._log(f" Could not parse truncated JSON for comparison (error: {truncatedError})")
|
||||
self._log(f" Could not parse truncated JSON for comparison (error: {truncatedError})")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
isValidJson = False
|
||||
|
|
@ -641,13 +637,15 @@ class JsonSplitMergeTester12:
|
|||
self._log(f" ❌ completePart is NOT valid JSON")
|
||||
self._log(f" Error: {jsonError}")
|
||||
self._log(f" Error position: line {e.lineno}, column {e.colno}")
|
||||
if jsonString:
|
||||
self._log(" ❌ JSON repair FAILED - errors were not fixed")
|
||||
|
||||
# Return test results
|
||||
return {
|
||||
result = {
|
||||
"success": isValidJson,
|
||||
"fileName": fileName,
|
||||
"originalSize": originalSize,
|
||||
"cutPosition": cutPosition,
|
||||
"cutPosition": cutPosition if not isComplete else None,
|
||||
"partSize": len(partContent),
|
||||
"overlapContextSize": len(contexts.overlapContext),
|
||||
"hierarchyContextSize": len(contexts.hierarchyContext),
|
||||
|
|
@ -655,8 +653,23 @@ class JsonSplitMergeTester12:
|
|||
"completePartSize": len(contexts.completePart),
|
||||
"isValidJson": isValidJson,
|
||||
"jsonError": jsonError,
|
||||
"parsedCompletePart": parsedCompletePart is not None
|
||||
"parsedCompletePart": parsedCompletePart is not None,
|
||||
"jsonParsingSuccess": contexts.jsonParsingSuccess
|
||||
}
|
||||
|
||||
# Add complete JSON specific checks
|
||||
if isComplete:
|
||||
result["overlapContextIsEmpty"] = contexts.overlapContext == ""
|
||||
result["isComplete"] = True
|
||||
# For complete JSON, success means overlapContext is empty AND valid JSON
|
||||
result["success"] = isValidJson and (contexts.overlapContext == "")
|
||||
|
||||
# Add error test specific checks
|
||||
if jsonString:
|
||||
result["hasErrors"] = True
|
||||
result["repairSuccess"] = isValidJson
|
||||
|
||||
return result
|
||||
|
||||
async def testAllJsonFiles(self) -> Dict[str, Any]:
|
||||
"""Test splitting and merging all test JSON files."""
|
||||
|
|
@ -723,11 +736,31 @@ class JsonSplitMergeTester12:
|
|||
if result.get("success"):
|
||||
successCount += 1
|
||||
isValidJson = result.get("isValidJson", False)
|
||||
if isValidJson:
|
||||
print(f"✅ {fileName:30s}: Valid JSON - completePart parsed successfully")
|
||||
isComplete = result.get("isComplete", False)
|
||||
hasErrors = result.get("hasErrors", False)
|
||||
|
||||
if isComplete:
|
||||
overlapEmpty = result.get("overlapContextIsEmpty", False)
|
||||
if isValidJson and overlapEmpty:
|
||||
print(f"✅ {fileName:30s}: Complete JSON - overlapContext='' and valid JSON")
|
||||
elif not overlapEmpty:
|
||||
print(f"⚠️ {fileName:30s}: Complete JSON but overlapContext not empty")
|
||||
else:
|
||||
jsonError = result.get("jsonError", "Unknown error")
|
||||
print(f"⚠️ {fileName:30s}: Complete JSON but not valid - {jsonError}")
|
||||
elif hasErrors:
|
||||
repairSuccess = result.get("repairSuccess", False)
|
||||
if repairSuccess:
|
||||
print(f"✅ {fileName:30s}: JSON with errors - repair successful")
|
||||
else:
|
||||
jsonError = result.get("jsonError", "Unknown error")
|
||||
print(f"❌ {fileName:30s}: JSON with errors - repair failed - {jsonError}")
|
||||
else:
|
||||
jsonError = result.get("jsonError", "Unknown error")
|
||||
print(f"⚠️ {fileName:30s}: Contexts generated but completePart is not valid JSON - {jsonError}")
|
||||
if isValidJson:
|
||||
print(f"✅ {fileName:30s}: Valid JSON - completePart parsed successfully")
|
||||
else:
|
||||
jsonError = result.get("jsonError", "Unknown error")
|
||||
print(f"⚠️ {fileName:30s}: Contexts generated but completePart is not valid JSON - {jsonError}")
|
||||
else:
|
||||
error = result.get("error", "Unknown error")
|
||||
print(f"❌ {fileName:30s}: FAILED - {error}")
|
||||
|
|
|
|||
Loading…
Reference in a new issue