From ee59e09b6d7e58d99c743ac88e4b8d897e5ccc1f Mon Sep 17 00:00:00 2001
From: ValueOn AG
Date: Mon, 6 Oct 2025 15:39:25 +0200
Subject: [PATCH] ai loop tested
---
modules/connectors/connectorDbPostgre.py | 34 ++
modules/datamodels/datamodelAi.py | 1 +
modules/datamodels/datamodelChat.py | 2 +
modules/services/serviceAi/mainServiceAi.py | 94 ++-
.../mainServiceGeneration.py | 10 +-
.../serviceGeneration/prompt_builder.py | 72 +++
.../renderers/csv_renderer.py | 48 +-
.../renderers/docx_renderer.py | 60 +-
.../renderers/excel_renderer.py | 74 +--
.../renderers/html_renderer.py | 47 +-
.../renderers/json_renderer.py | 53 +-
.../renderers/markdown_renderer.py | 52 +-
.../renderers/pdf_renderer.py | 52 +-
.../renderers/text_renderer.py | 55 +-
...t1_cont_false_openai_callAiBasic_gpt35.txt | 8 +
...art1_cont_false_perplexity_callAiBasic.txt | 78 +++
.../processing/shared/placeholderFactory.py | 120 +++-
.../shared/promptGenerationActionsReact.py | 50 +-
...t1_cont_false_openai_callAiBasic_gpt35.txt | 6 +
..._part1_cont_true_anthropic_callAiBasic.txt | 46 ++
...part2_cont_false_anthropic_callAiBasic.txt | 39 ++
...t1_cont_false_openai_callAiBasic_gpt35.txt | 8 +
..._part1_cont_true_anthropic_callAiBasic.txt | 36 ++
..._part2_cont_true_anthropic_callAiBasic.txt | 27 +
..._part3_cont_true_anthropic_callAiBasic.txt | 27 +
..._part4_cont_true_anthropic_callAiBasic.txt | 27 +
...part5_cont_false_anthropic_callAiBasic.txt | 27 +
...t1_cont_false_openai_callAiBasic_gpt35.txt | 6 +
..._part1_cont_true_anthropic_callAiBasic.txt | 54 ++
..._part2_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part3_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part4_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part5_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part6_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part7_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part8_cont_true_anthropic_callAiBasic.txt | 55 ++
..._part9_cont_true_anthropic_callAiBasic.txt | 55 ++
...part10_cont_true_anthropic_callAiBasic.txt | 55 ++
...t1_cont_false_openai_callAiBasic_gpt35.txt | 6 +
.../extracted_content.txt | 246 --------
.../rendered_output.txt | 1 -
.../extracted_content.txt | 76 +++
.../meta.txt | 2 +-
.../rendered_output.txt | 1 +
.../extracted_content.txt | 81 +++
.../render_input_20251006-132934/meta.txt | 4 +
.../rendered_output.txt | 1 +
.../extracted_content.txt | 136 +++++
.../render_input_20251006-133209/meta.txt | 4 +
.../rendered_output.txt | 1 +
.../extracted_content.txt | 537 ++++++++++++++++++
.../render_input_20251006-133402/meta.txt | 4 +
.../rendered_output.txt | 1 +
.../obj/m20251006-125112_9_1_0/message.json | 19 -
.../obj/m20251006-125217_9_1_1/message.json | 19 -
.../m20251006-125217_9_1_1/message_text.txt | 4 -
.../document_001_metadata.json | 12 -
.../obj/m20251006-125220_9_0_0/message.json | 19 -
.../obj/m20251006-125220_9_1_0/message.json | 19 -
.../m20251006-125220_9_1_0/message_text.txt | 4 -
.../message.json | 12 +-
.../message_text.txt | 0
.../obj/m20251006-152502_3_1_0/message.json | 19 +
.../m20251006-152502_3_1_0/message_text.txt | 6 +
.../obj/m20251006-152503_3_1_0/message.json | 19 +
.../m20251006-152503_3_1_0/message_text.txt | 3 +
.../obj/m20251006-152543_3_1_1/message.json | 19 +
.../m20251006-152543_3_1_1/message_text.txt | 4 +
.../document_001_metadata.json | 12 +
.../obj/m20251006-152546_3_0_0/message.json | 19 +
.../message_text.txt | 2 +-
.../obj/m20251006-152546_3_1_0/message.json | 19 +
.../m20251006-152546_3_1_0/message_text.txt | 4 +
.../obj/m20251006-152907_4_0_0/message.json | 19 +
.../m20251006-152907_4_0_0/message_text.txt | 1 +
.../obj/m20251006-152914_4_1_0/message.json | 19 +
.../message_text.txt | 2 +-
.../message.json | 10 +-
.../message_text.txt | 0
.../obj/m20251006-152934_4_1_1/message.json | 19 +
.../m20251006-152934_4_1_1/message_text.txt | 4 +
.../document_001_metadata.json | 12 +
.../obj/m20251006-152937_4_0_0/message.json | 19 +
.../m20251006-152937_4_0_0/message_text.txt | 4 +
.../obj/m20251006-152937_4_1_0/message.json | 19 +
.../m20251006-152937_4_1_0/message_text.txt | 4 +
.../obj/m20251006-153117_5_0_0/message.json | 19 +
.../m20251006-153117_5_0_0/message_text.txt | 1 +
.../obj/m20251006-153125_5_1_0/message.json | 19 +
.../m20251006-153125_5_1_0/message_text.txt | 6 +
.../obj/m20251006-153126_5_1_0/message.json | 19 +
.../m20251006-153126_5_1_0/message_text.txt | 3 +
.../obj/m20251006-153209_5_1_1/message.json | 19 +
.../m20251006-153209_5_1_1/message_text.txt | 4 +
.../document_001_metadata.json | 12 +
.../obj/m20251006-153212_5_1_0/message.json | 19 +
.../m20251006-153212_5_1_0/message_text.txt | 4 +
.../obj/m20251006-153213_5_0_0/message.json | 19 +
.../m20251006-153213_5_0_0/message_text.txt | 4 +
.../obj/m20251006-153309_6_0_0/message.json | 19 +
.../m20251006-153309_6_0_0/message_text.txt | 1 +
.../obj/m20251006-153318_6_1_0/message.json | 19 +
.../m20251006-153318_6_1_0/message_text.txt | 6 +
.../obj/m20251006-153319_6_1_0/message.json | 19 +
.../m20251006-153319_6_1_0/message_text.txt | 3 +
.../obj/m20251006-153403_6_1_1/message.json | 19 +
.../m20251006-153403_6_1_1/message_text.txt | 4 +
.../document_001_metadata.json | 12 +
.../obj/m20251006-153406_6_1_0/message.json | 19 +
.../m20251006-153406_6_1_0/message_text.txt | 4 +
.../obj/m20251006-153407_6_0_0/message.json | 19 +
.../m20251006-153407_6_0_0/message_text.txt | 4 +
.../obj/m20251006-153843_7_0_0/message.json | 19 +
.../m20251006-153843_7_0_0/message_text.txt | 1 +
.../obj/m20251006-153851_7_1_0/message.json | 19 +
.../m20251006-153851_7_1_0/message_text.txt | 6 +
.../obj/m20251006-153853_7_1_0/message.json | 19 +
.../m20251006-153853_7_1_0/message_text.txt | 3 +
.../obj/m20251006-153859_7_1_1/message.json | 19 +
.../m20251006-153859_7_1_1/message_text.txt | 6 +
.../obj/m20251006-153907_7_1_2/message.json | 19 +
.../m20251006-153907_7_1_2/message_text.txt | 6 +
.../obj/m20251006-153915_7_1_3/message.json | 19 +
.../m20251006-153915_7_1_3/message_text.txt | 6 +
.../obj/m20251006-153923_7_1_4/message.json | 19 +
.../m20251006-153923_7_1_4/message_text.txt | 6 +
126 files changed, 2867 insertions(+), 771 deletions(-)
create mode 100644 modules/services/serviceGeneration/prompt_builder.py
create mode 100644 modules/test-chat/ai-responses/ai_text_20251006-132457-767_part1_cont_false_openai_callAiBasic_gpt35.txt
create mode 100644 modules/test-chat/ai-responses/ai_text_20251006-132542-878_part1_cont_false_perplexity_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-132909-453_part1_cont_false_openai_callAiBasic_gpt35.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-132928-806_part1_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-132934-067_part2_cont_false_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133118-952_part1_cont_false_openai_callAiBasic_gpt35.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133139-404_part1_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133146-444_part2_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133154-825_part3_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133201-983_part4_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133209-028_part5_cont_false_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133312-266_part1_cont_false_openai_callAiBasic_gpt35.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133327-881_part1_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133330-647_part2_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133333-511_part3_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133336-741_part4_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133340-102_part5_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133343-485_part6_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133350-119_part7_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133356-058_part8_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133359-562_part9_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133402-924_part10_cont_true_anthropic_callAiBasic.txt
create mode 100644 test-chat/ai-responses/ai_text_20251006-133844-773_part1_cont_false_openai_callAiBasic_gpt35.txt
delete mode 100644 test-chat/extraction/render_input_20251006-105217/extracted_content.txt
delete mode 100644 test-chat/extraction/render_input_20251006-105217/rendered_output.txt
create mode 100644 test-chat/extraction/render_input_20251006-132542/extracted_content.txt
rename test-chat/extraction/{render_input_20251006-105217 => render_input_20251006-132542}/meta.txt (82%)
create mode 100644 test-chat/extraction/render_input_20251006-132542/rendered_output.txt
create mode 100644 test-chat/extraction/render_input_20251006-132934/extracted_content.txt
create mode 100644 test-chat/extraction/render_input_20251006-132934/meta.txt
create mode 100644 test-chat/extraction/render_input_20251006-132934/rendered_output.txt
create mode 100644 test-chat/extraction/render_input_20251006-133209/extracted_content.txt
create mode 100644 test-chat/extraction/render_input_20251006-133209/meta.txt
create mode 100644 test-chat/extraction/render_input_20251006-133209/rendered_output.txt
create mode 100644 test-chat/extraction/render_input_20251006-133402/extracted_content.txt
create mode 100644 test-chat/extraction/render_input_20251006-133402/meta.txt
create mode 100644 test-chat/extraction/render_input_20251006-133402/rendered_output.txt
delete mode 100644 test-chat/obj/m20251006-125112_9_1_0/message.json
delete mode 100644 test-chat/obj/m20251006-125217_9_1_1/message.json
delete mode 100644 test-chat/obj/m20251006-125217_9_1_1/message_text.txt
delete mode 100644 test-chat/obj/m20251006-125217_9_1_1/round9_task1_action1_results/document_001_metadata.json
delete mode 100644 test-chat/obj/m20251006-125220_9_0_0/message.json
delete mode 100644 test-chat/obj/m20251006-125220_9_1_0/message.json
delete mode 100644 test-chat/obj/m20251006-125220_9_1_0/message_text.txt
rename test-chat/obj/{m20251006-125105_9_0_0 => m20251006-152454_3_0_0}/message.json (56%)
rename test-chat/obj/{m20251006-125105_9_0_0 => m20251006-152454_3_0_0}/message_text.txt (100%)
create mode 100644 test-chat/obj/m20251006-152502_3_1_0/message.json
create mode 100644 test-chat/obj/m20251006-152502_3_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-152503_3_1_0/message.json
create mode 100644 test-chat/obj/m20251006-152503_3_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-152543_3_1_1/message.json
create mode 100644 test-chat/obj/m20251006-152543_3_1_1/message_text.txt
create mode 100644 test-chat/obj/m20251006-152543_3_1_1/round3_task1_action1_results/document_001_metadata.json
create mode 100644 test-chat/obj/m20251006-152546_3_0_0/message.json
rename test-chat/obj/{m20251006-125220_9_0_0 => m20251006-152546_3_0_0}/message_text.txt (50%)
create mode 100644 test-chat/obj/m20251006-152546_3_1_0/message.json
create mode 100644 test-chat/obj/m20251006-152546_3_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-152907_4_0_0/message.json
create mode 100644 test-chat/obj/m20251006-152907_4_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-152914_4_1_0/message.json
rename test-chat/obj/{m20251006-125112_9_1_0 => m20251006-152914_4_1_0}/message_text.txt (64%)
rename test-chat/obj/{m20251006-125113_9_1_0 => m20251006-152915_4_1_0}/message.json (67%)
rename test-chat/obj/{m20251006-125113_9_1_0 => m20251006-152915_4_1_0}/message_text.txt (100%)
create mode 100644 test-chat/obj/m20251006-152934_4_1_1/message.json
create mode 100644 test-chat/obj/m20251006-152934_4_1_1/message_text.txt
create mode 100644 test-chat/obj/m20251006-152934_4_1_1/round4_task1_action1_results/document_001_metadata.json
create mode 100644 test-chat/obj/m20251006-152937_4_0_0/message.json
create mode 100644 test-chat/obj/m20251006-152937_4_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-152937_4_1_0/message.json
create mode 100644 test-chat/obj/m20251006-152937_4_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153117_5_0_0/message.json
create mode 100644 test-chat/obj/m20251006-153117_5_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153125_5_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153125_5_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153126_5_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153126_5_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153209_5_1_1/message.json
create mode 100644 test-chat/obj/m20251006-153209_5_1_1/message_text.txt
create mode 100644 test-chat/obj/m20251006-153209_5_1_1/round5_task1_action1_results/document_001_metadata.json
create mode 100644 test-chat/obj/m20251006-153212_5_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153212_5_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153213_5_0_0/message.json
create mode 100644 test-chat/obj/m20251006-153213_5_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153309_6_0_0/message.json
create mode 100644 test-chat/obj/m20251006-153309_6_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153318_6_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153318_6_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153319_6_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153319_6_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153403_6_1_1/message.json
create mode 100644 test-chat/obj/m20251006-153403_6_1_1/message_text.txt
create mode 100644 test-chat/obj/m20251006-153403_6_1_1/round6_task1_action1_results/document_001_metadata.json
create mode 100644 test-chat/obj/m20251006-153406_6_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153406_6_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153407_6_0_0/message.json
create mode 100644 test-chat/obj/m20251006-153407_6_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153843_7_0_0/message.json
create mode 100644 test-chat/obj/m20251006-153843_7_0_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153851_7_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153851_7_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153853_7_1_0/message.json
create mode 100644 test-chat/obj/m20251006-153853_7_1_0/message_text.txt
create mode 100644 test-chat/obj/m20251006-153859_7_1_1/message.json
create mode 100644 test-chat/obj/m20251006-153859_7_1_1/message_text.txt
create mode 100644 test-chat/obj/m20251006-153907_7_1_2/message.json
create mode 100644 test-chat/obj/m20251006-153907_7_1_2/message_text.txt
create mode 100644 test-chat/obj/m20251006-153915_7_1_3/message.json
create mode 100644 test-chat/obj/m20251006-153915_7_1_3/message_text.txt
create mode 100644 test-chat/obj/m20251006-153923_7_1_4/message.json
create mode 100644 test-chat/obj/m20251006-153923_7_1_4/message_text.txt
diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py
index c17fa2c3..fc06d645 100644
--- a/modules/connectors/connectorDbPostgre.py
+++ b/modules/connectors/connectorDbPostgre.py
@@ -324,6 +324,40 @@ class DatabaseConnector:
# Create table from Pydantic model
self._create_table_from_model(cursor, table, model_class)
logger.info(f"Created table '{table}' with columns from Pydantic model")
+ else:
+ # Table exists: ensure all columns from model are present (simple additive migration)
+ try:
+ cursor.execute("""
+ SELECT column_name FROM information_schema.columns
+ WHERE LOWER(table_name) = LOWER(%s) AND table_schema = 'public'
+ """, (table,))
+ existing_columns = {row['column_name'] for row in cursor.fetchall()}
+
+ # Desired columns based on model
+ model_fields = _get_model_fields(model_class)
+ desired_columns = set(['id']) | set(model_fields.keys()) | {'_createdAt', '_modifiedAt', '_createdBy', '_modifiedBy'}
+
+ # Add missing columns
+ for col in sorted(desired_columns - existing_columns):
+ # Determine SQL type
+ if col in ['id']:
+ continue # primary key exists already
+ sql_type = model_fields.get(col)
+ if col in ['_createdAt']:
+ sql_type = 'DOUBLE PRECISION'
+ elif col in ['_modifiedAt']:
+ sql_type = 'DOUBLE PRECISION'
+ elif col in ['_createdBy', '_modifiedBy']:
+ sql_type = 'VARCHAR(255)'
+ if not sql_type:
+ sql_type = 'TEXT'
+ try:
+ cursor.execute(f'ALTER TABLE "{table}" ADD COLUMN "{col}" {sql_type}')
+ logger.info(f"Added missing column '{col}' ({sql_type}) to '{table}'")
+ except Exception as add_err:
+ logger.warning(f"Could not add column '{col}' to '{table}': {add_err}")
+ except Exception as ensure_err:
+ logger.warning(f"Could not ensure columns for existing table '{table}': {ensure_err}")
self.connection.commit()
return True
diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py
index 06cf0179..ad06f785 100644
--- a/modules/datamodels/datamodelAi.py
+++ b/modules/datamodels/datamodelAi.py
@@ -115,6 +115,7 @@ class AiCallOptions(BaseModel):
# Model generation parameters
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0, description="Temperature for response generation (0.0-2.0, lower = more consistent)")
maxTokens: Optional[int] = Field(default=None, ge=1, le=32000, description="Maximum tokens in response")
+ maxParts: Optional[int] = Field(default=1000, ge=1, le=1000, description="Maximum number of continuation parts to fetch")
class AiCallRequest(BaseModel):
diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py
index b7b42aad..d423fa80 100644
--- a/modules/datamodels/datamodelChat.py
+++ b/modules/datamodels/datamodelChat.py
@@ -145,6 +145,7 @@ class ChatMessage(BaseModel, ModelMixin):
documents: List[ChatDocument] = Field(default_factory=list, description="Associated documents")
documentsLabel: Optional[str] = Field(None, description="Label for the set of documents")
message: Optional[str] = Field(None, description="Message content")
+ summary: Optional[str] = Field(None, description="Short summary of this message for planning/history")
role: str = Field(description="Role of the message sender")
status: str = Field(description="Status of the message (first, step, last)")
sequenceNr: int = Field(description="Sequence number of the message (set automatically)")
@@ -169,6 +170,7 @@ register_model_labels(
"documents": {"en": "Documents", "fr": "Documents"},
"documentsLabel": {"en": "Documents Label", "fr": "Label des documents"},
"message": {"en": "Message", "fr": "Message"},
+ "summary": {"en": "Summary", "fr": "Résumé"},
"role": {"en": "Role", "fr": "Rôle"},
"status": {"en": "Status", "fr": "Statut"},
"sequenceNr": {"en": "Sequence Number", "fr": "Numéro de séquence"},
diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py
index deb0cce9..95e0c108 100644
--- a/modules/services/serviceAi/mainServiceAi.py
+++ b/modules/services/serviceAi/mainServiceAi.py
@@ -977,16 +977,18 @@ class AiService:
full_prompt += (
"\n\nINSTRUCTIONS (COMPLETENESS):\n"
"- Continue from where the previous content ended. Do NOT repeat earlier content.\n"
- "- If more parts are still needed after this response, append a final line exactly: 'CONTINUATION: true'.\n"
- "- If the content is now complete, append a final line exactly: 'CONTINUATION: false'.\n"
+ "- If more parts are still needed after this response, the LAST LINE of your response MUST be exactly: 'CONTINUATION: true'.\n"
+ "- If the content is now complete, the LAST LINE of your response MUST be exactly: 'CONTINUATION: false'.\n"
+ "- The continuation line MUST be the final line of your output. Do NOT output anything after it (no notes, no explanations).\n"
)
else:
# First call (no prior context): deliver full content or first part
full_prompt += (
"\n\nINSTRUCTIONS (COMPLETENESS):\n"
"- Deliver the complete content. Do NOT truncate.\n"
- "- If platform limits force truncation, provide the first complete section(s) only and append a final line exactly: 'CONTINUATION: true'.\n"
- "- If the entire content is fully included, append a final line exactly: 'CONTINUATION: false'.\n"
+ "- If platform limits force truncation, provide the first complete section(s) only and ensure the LAST LINE of your response is exactly: 'CONTINUATION: true'.\n"
+ "- If the entire content is fully included, ensure the LAST LINE of your response is exactly: 'CONTINUATION: false'.\n"
+ "- The continuation line MUST be the final line of your output. Do NOT output anything after it (no notes, no explanations).\n"
)
except Exception:
# Non-fatal if any issue building guidance
@@ -1025,9 +1027,23 @@ class AiService:
pass
content_first = response.content or ""
merged_content, needs_more = _split_content_and_flag(content_first)
+ try:
+ self._writeAiResponseDebug(
+ label='ai_text',
+ content=content_first,
+ partIndex=1,
+ modelName=getattr(response, 'modelName', None),
+ continuation=needs_more
+ )
+ except Exception:
+ pass
# Iteratively request next parts if flagged
- max_parts = 10
+ # Allow configurable max parts via options; default = 1000
+ try:
+ max_parts = int(getattr(options, 'maxParts', 1000) or 1000)
+ except Exception:
+ max_parts = 1000
part_index = 1
while needs_more and part_index < max_parts:
part_index += 1
@@ -1037,7 +1053,8 @@ class AiService:
+ "\n\nINSTRUCTIONS (CONTINUE NEXT PART ONLY):\n"
"- Continue from where the previous content ended.\n"
"- Do NOT repeat earlier content.\n"
- "- Append a final line exactly: 'CONTINUATION: true' if more parts are needed, otherwise 'CONTINUATION: false'.\n"
+ "- The LAST LINE of your response MUST be exactly one of: 'CONTINUATION: true' (if more parts are needed) or 'CONTINUATION: false' (if complete).\n"
+ "- The continuation line MUST be the final line of your output. Do NOT output anything after it (no notes, no explanations).\n"
)
next_request = AiCallRequest(
prompt=subsequent_prompt,
@@ -1047,6 +1064,16 @@ class AiService:
next_response = await self.aiObjects.call(next_request)
part_text = next_response.content or ""
part_clean, needs_more = _split_content_and_flag(part_text)
+ try:
+ self._writeAiResponseDebug(
+ label='ai_text',
+ content=part_text,
+ partIndex=part_index,
+ modelName=getattr(next_response, 'modelName', None),
+ continuation=needs_more
+ )
+ except Exception:
+ pass
if part_clean:
# Separate parts clearly
merged_content = (merged_content + "\n\n" + part_clean).strip()
@@ -1219,6 +1246,34 @@ class AiService:
# Swallow to avoid recursive logging issues
pass
+ def _writeAiResponseDebug(self, label: str, content: str, partIndex: int = 1, modelName: str = None, continuation: bool = None) -> None:
+ """Persist raw AI response parts for debugging under test-chat/ai-responses."""
+ try:
+ import os
+ from datetime import datetime, UTC
+ # Base dir: gateway/test-chat/ai-responses (go up 4 levels from this file)
+ # .../gateway/modules/services/serviceAi/mainServiceAi.py -> up to gateway root
+ gatewayDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+ outDir = os.path.join(gatewayDir, 'test-chat', 'ai-responses')
+ os.makedirs(outDir, exist_ok=True)
+ ts = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3]
+ suffix = []
+ if partIndex is not None:
+ suffix.append(f"part{partIndex}")
+ if continuation is not None:
+ suffix.append(f"cont_{str(continuation).lower()}")
+ if modelName:
+ safeModel = ''.join(c if c.isalnum() or c in ('-', '_') else '-' for c in modelName)
+ suffix.append(safeModel)
+ suffixStr = ('_' + '_'.join(suffix)) if suffix else ''
+ fname = f"{label}_{ts}{suffixStr}.txt"
+ fpath = os.path.join(outDir, fname)
+ with open(fpath, 'w', encoding='utf-8') as f:
+ f.write(content or '')
+ except Exception:
+ # Do not raise; best-effort debug write
+ pass
+
def _exceedsTokenLimit(self, text: str, model: ModelCapabilities, safety_margin: float) -> bool:
"""
Check if text exceeds model token limit with safety margin.
@@ -1347,6 +1402,25 @@ class AiService:
# Process documents with format-specific prompt
ai_response = await self._callAiText(extraction_prompt, documents, options)
+
+ # Parse filename header from AI response if present
+ parsed_filename = None
+ try:
+ if ai_response:
+ first_newline = ai_response.find('\n')
+ header_line = ai_response if first_newline == -1 else ai_response[:first_newline]
+ if header_line.strip().lower().startswith('filename:'):
+ parsed = header_line.split(':', 1)[1].strip()
+ # basic sanitization
+ import re
+ parsed = re.sub(r"[^a-zA-Z0-9._-]", "-", parsed)
+ parsed = re.sub(r"-+", "-", parsed).strip('-')
+ if parsed:
+ parsed_filename = parsed
+ # remove header line from content for rendering
+ ai_response = ai_response[first_newline+1:].lstrip('\n') if first_newline != -1 else ''
+ except Exception:
+ parsed_filename = None
if not ai_response or ai_response.strip() == "":
raise Exception("AI content generation failed")
@@ -1358,10 +1432,14 @@ class AiService:
title=title
)
- # Generate meaningful filename
+ # Generate meaningful filename (use AI-provided if valid, else fallback)
from datetime import datetime, UTC
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
- filename = f"{title.replace(' ', '_')}_{timestamp}.{outputFormat}"
+ if parsed_filename and parsed_filename.lower().endswith(f".{outputFormat.lower()}"):
+ filename = parsed_filename
+ else:
+ safe_title = ''.join(c if c.isalnum() else '-' for c in (title or 'document')).strip('-')
+ filename = f"{safe_title or 'document'}-{timestamp}.{outputFormat}"
# Return structured result with document information
return {
diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py
index 7a7b6baf..76883be4 100644
--- a/modules/services/serviceGeneration/mainServiceGeneration.py
+++ b/modules/services/serviceGeneration/mainServiceGeneration.py
@@ -363,8 +363,14 @@ class GenerationService:
if not renderer:
raise ValueError(f"Unsupported output format: {output_format}")
- # Get the format-specific extraction prompt
- extraction_prompt = renderer.getExtractionPrompt(user_prompt, title)
+ # Build centralized prompt with generic rules + format-specific guidelines
+ from .prompt_builder import buildExtractionPrompt
+ extraction_prompt = buildExtractionPrompt(
+ output_format=output_format,
+ renderer=renderer,
+ user_prompt=user_prompt,
+ title=title
+ )
logger.info(f"Generated {output_format}-specific extraction prompt: {len(extraction_prompt)} characters")
return extraction_prompt
diff --git a/modules/services/serviceGeneration/prompt_builder.py b/modules/services/serviceGeneration/prompt_builder.py
new file mode 100644
index 00000000..208c4c18
--- /dev/null
+++ b/modules/services/serviceGeneration/prompt_builder.py
@@ -0,0 +1,72 @@
+"""
+Centralized prompt builder for document generation across formats.
+
+Builds a robust prompt that:
+- Accepts any user intent (no fixed structure assumptions)
+- Injects format-specific guidelines from the selected renderer
+- Adds a common policy section to always use real data from source docs
+- Requires the AI to output a filename header that we can parse and use
+"""
+
+from typing import Protocol
+
+
+class _RendererLike(Protocol):
+ def getExtractionPrompt(self, user_prompt: str, title: str) -> str: # returns only format-specific guidelines
+ ...
+
+
+def buildExtractionPrompt(
+ output_format: str,
+ renderer: _RendererLike,
+ user_prompt: str,
+ title: str
+) -> str:
+ """
+ Build the final extraction prompt by combining:
+ - The raw user prompt (verbatim)
+ - Generic cross-format instructions (filename header + real-data policy)
+ - Format-specific guidelines snippet provided by the renderer
+
+ The AI must place a single filename header at the very top:
+ FILENAME:
+ followed by a blank line and then ONLY the document content according to the target format.
+ """
+
+ format_guidelines = renderer.getExtractionPrompt(user_prompt, title)
+
+ # Generic block appears once for every format
+ generic_intro = f"""
+{user_prompt}
+
+You are generating a document in {output_format.upper()} format for the title: "{title}".
+
+Rules:
+- The user's intent fully defines the structure. Do not assume a fixed template or headings.
+- Use only factual information extracted from the supplied source documents.
+- Do not invent, hallucinate, or include placeholders (e.g., "lorem ipsum", "TBD").
+- The output must strictly follow the target format and be ready for saving without extra wrapping.
+- At the VERY TOP output exactly one line with the filename header:
+ FILENAME:
+ - The base name should be short, descriptive, and kebab-case or snake-case without spaces.
+ - Include the correct extension for the requested format (e.g., .html, .pdf, .docx, .md, .txt, .json, .csv, .xlsx).
+ - Avoid special characters beyond [a-zA-Z0-9-_].
+ - After this header, insert a single blank line and then provide ONLY the document content.
+
+Common policy:
+- Use the actual data from the source documents to create the content.
+- Do not generate placeholder text or templates.
+- Extract and use the real data provided in the source documents to create meaningful content.
+""".strip()
+
+ # Final assembly
+ final_prompt = (
+ generic_intro
+ + "\n\nFORMAT-SPECIFIC GUIDELINES:\n"
+ + format_guidelines.strip()
+ + "\n\nGenerate the complete document content now based on the source documents below:"
+ )
+
+ return final_prompt
+
+
diff --git a/modules/services/serviceGeneration/renderers/csv_renderer.py b/modules/services/serviceGeneration/renderers/csv_renderer.py
index 42248922..9ef6882c 100644
--- a/modules/services/serviceGeneration/renderers/csv_renderer.py
+++ b/modules/services/serviceGeneration/renderers/csv_renderer.py
@@ -26,44 +26,16 @@ class CsvRenderer(BaseRenderer):
return 70
def getExtractionPrompt(self, user_prompt: str, title: str) -> str:
- """Get CSV-specific extraction prompt."""
- return f"""
-{user_prompt}
-
-Generate a comprehensive CSV report with the title: "{title}"
-
-CSV FORMAT REQUIREMENTS:
-- Create structured data in CSV format
-- Use proper CSV syntax with commas and quotes
-- Include headers for all columns
-- Structure data in rows and columns
-- Include source document information
-- Add metadata as additional rows
-
-CSV STRUCTURE:
-- First row: Headers (Section, Type, Heading, Content, Source)
-- Data rows: One per section/item
-- Use quotes around content that contains commas
-- Escape quotes properly
-- Include metadata rows at the end
-
-FORMATTING RULES:
-- Headers: Section, Type, Heading, Content, Source
-- Content: Escape commas and quotes, limit length
-- Source: Include document name and page if available
-- Metadata: Add special rows for generation info
-
-OUTPUT POLICY:
-- Return ONLY CSV data
-- No markdown, no code blocks, no additional text
-- Properly formatted CSV
-- Include all necessary information
-- Valid CSV that can be imported
-
-CRITICAL: Use the actual data from the source documents to create the content. Do not generate placeholder text or templates. Extract and use the real data provided in the source documents to create meaningful content.
-
-Generate the complete CSV report using the actual data from the source documents:
-"""
+ """Return only CSV-specific guidelines; global prompt is built centrally."""
+ return (
+ "CSV FORMAT GUIDELINES:\n"
+ "- Emit ONLY CSV text without fences or commentary.\n"
+ "- Include a single header row with clear column names.\n"
+ "- Quote fields containing commas, quotes, or newlines; escape quotes by doubling them.\n"
+ "- Use rows to represent items/records derived from sources.\n"
+ "- Keep cells concise; include units in headers when useful.\n"
+ "OUTPUT: Return ONLY valid CSV content that can be imported."
+ )
async def render(self, extracted_content: str, title: str) -> Tuple[str, str]:
"""Render extracted content to CSV format."""
diff --git a/modules/services/serviceGeneration/renderers/docx_renderer.py b/modules/services/serviceGeneration/renderers/docx_renderer.py
index 75663b4d..134f00cd 100644
--- a/modules/services/serviceGeneration/renderers/docx_renderer.py
+++ b/modules/services/serviceGeneration/renderers/docx_renderer.py
@@ -39,58 +39,14 @@ class DocxRenderer(BaseRenderer):
return 115
def getExtractionPrompt(self, user_prompt: str, title: str) -> str:
- """Get DOCX-specific extraction prompt."""
- return f"""
-{user_prompt}
-
-Generate a comprehensive DOCX report with the title: "{title}"
-
-DOCX FORMAT REQUIREMENTS:
-- Create structured content suitable for Word documents
-- Use clear headings and sections with proper hierarchy
-- Include tables for structured data
-- Use bullet points and numbered lists where appropriate
-- Include source document information
-- Structure content for professional presentation
-- Use consistent formatting throughout
-
-DOCX STRUCTURE:
-- Title page with report title and generation date
-- Table of contents (if multiple sections)
-- Executive summary
-- Main content sections with clear headings
-- Data tables and analysis
-- Conclusions and recommendations
-- Appendices with source information
-
-FORMATTING RULES:
-- Use clear section headings (H1, H2, H3 style)
-- Include consistent paragraph formatting
-- Use tables with proper alignment and borders
-- Use bullet points and numbered lists
-- Add source citations and references
-- Include generation metadata
-- Use professional fonts and spacing
-
-OUTPUT POLICY:
-- Return ONLY plain text content suitable for Word document generation
-- NO markdown formatting (no **bold**, no # headings, no --- separators)
-- NO HTML tags
-- NO code blocks
-- Use plain text with clear structure
-- Use line breaks for separation
-- Use indentation for lists
-- Use ALL CAPS for major headings
-- Use Title Case for subheadings
-- Use bullet points with dashes (-) for lists
-- Use numbers (1., 2., 3.) for numbered lists
-- Professional document format
-- Include all necessary information
-
-CRITICAL: Use the actual data from the source documents to create the content. Do not generate placeholder text or templates. Extract and use the real data provided in the source documents to create meaningful content.
-
-Generate the complete DOCX report content using the actual data from the source documents:
-"""
+ """Return only DOCX-specific guidelines; global prompt is built centrally."""
+ return (
+ "DOCX FORMAT GUIDELINES:\n"
+ "- Provide plain text content suitable for Word generation (no markdown/HTML).\n"
+ "- Use clear section hierarchy; bullet and numbered lists where needed.\n"
+ "- Include tables as simple pipe-delimited lines if tabular data is needed.\n"
+ "OUTPUT: Return ONLY the structured plain text to be converted into DOCX."
+ )
async def render(self, extracted_content: str, title: str) -> Tuple[str, str]:
"""Render extracted content to DOCX format."""
diff --git a/modules/services/serviceGeneration/renderers/excel_renderer.py b/modules/services/serviceGeneration/renderers/excel_renderer.py
index e9ec80cf..1472201b 100644
--- a/modules/services/serviceGeneration/renderers/excel_renderer.py
+++ b/modules/services/serviceGeneration/renderers/excel_renderer.py
@@ -36,71 +36,15 @@ class ExcelRenderer(BaseRenderer):
return 110
def getExtractionPrompt(self, user_prompt: str, title: str) -> str:
- """Get Excel-specific extraction prompt."""
- return f"""
-{user_prompt}
-
-Generate a comprehensive Excel report with the title: "{title}"
-
-EXCEL FORMAT REQUIREMENTS:
-- Create structured data suitable for Excel spreadsheets
-- Use clear column headers and organized rows
-- Include multiple sheets if needed (Summary, Data, Analysis, etc.)
-- Use proper data types (text, numbers, dates)
-- Include formulas where appropriate
-- Structure data in tables with clear headers
-- Include source document information
-- Add metadata and generation information
-
-EXCEL STRUCTURE:
-- Sheet 1: Summary/Overview with key metrics
-- Sheet 2: Detailed data in tabular format
-- Sheet 3: Analysis and insights
-- Use proper column headers (A, B, C, etc.)
-- Include data validation and formatting hints
-- Add comments for complex data
-
-FORMATTING RULES:
-- Headers: Use bold formatting, clear column names
-- Data: Organize in rows and columns, consistent formatting
-- Numbers: Use proper number formatting (currency, percentages, etc.)
-- Dates: Use standard date format (YYYY-MM-DD)
-- Text: Left-aligned, wrap long text
-- Formulas: Use Excel formula syntax (=SUM, =AVERAGE, etc.)
-- Colors: Use conditional formatting for highlights
-
-SHEET STRUCTURE:
-Sheet 1 - Summary:
-- Report Title
-- Key Metrics (counts, totals, averages)
-- Executive Summary
-- Generation Date
-
-Sheet 2 - Data:
-- Column A: Item/Category
-- Column B: Value/Amount
-- Column C: Percentage
-- Column D: Source Document
-- Column E: Notes/Comments
-
-Sheet 3 - Analysis:
-- Trends and patterns
-- Comparisons
-- Recommendations
-- Charts descriptions
-
-OUTPUT POLICY:
-- Return ONLY Excel-compatible data
-- No HTML, no markdown, no code blocks
-- Structured data that can be imported to Excel
-- Include sheet names and structure
-- Professional spreadsheet format
-- Include all necessary information
-
-CRITICAL: Use the actual data from the source documents to create the content. Do not generate placeholder text or templates. Extract and use the real data provided in the source documents to create meaningful content.
-
-Generate the complete Excel report data using the actual data from the source documents:
-"""
+ """Return only Excel-specific guidelines; global prompt is built centrally."""
+ return (
+ "EXCEL FORMAT GUIDELINES:\n"
+ "- Output one or more pipe-delimited tables with a single header row.\n"
+ "- Let user intent define columns; use clear names and ISO dates.\n"
+ "- Separate multiple tables by a single blank line.\n"
+ "- No markdown/HTML/code fences; tables only unless user explicitly asks for notes.\n"
+ "OUTPUT: Return ONLY pipe-delimited tables suitable for import."
+ )
async def render(self, extracted_content: str, title: str) -> Tuple[str, str]:
"""Render extracted content to Excel format."""
diff --git a/modules/services/serviceGeneration/renderers/html_renderer.py b/modules/services/serviceGeneration/renderers/html_renderer.py
index 81fab683..c2b7e586 100644
--- a/modules/services/serviceGeneration/renderers/html_renderer.py
+++ b/modules/services/serviceGeneration/renderers/html_renderer.py
@@ -24,43 +24,16 @@ class HtmlRenderer(BaseRenderer):
return 100
def getExtractionPrompt(self, user_prompt: str, title: str) -> str:
- """Get HTML-specific extraction prompt."""
- return f"""
-{user_prompt}
-
-Generate a comprehensive HTML report with the title: "{title}"
-
-HTML STRUCTURE REQUIREMENTS:
-- Create a complete, self-contained HTML document
-- Start with:
-- Include: , (with and ), and
-- Use proper HTML5 semantic elements: , , , ,