diff --git a/env_dev.env b/env_dev.env index 90d21538..d33c5598 100644 --- a/env_dev.env +++ b/env_dev.env @@ -44,7 +44,7 @@ APP_FRONTEND_URL = http://localhost:5176 # AI configuration Connector_AiOpenai_API_SECRET = DEV_ENC:Z0FBQUFBQnBaSnM4TWFRRmxVQmNQblVIYmc1Y0Q3aW9zZUtDWlNWdGZjbFpncGp2NHN2QjkxMWxibUJnZDBId252MWk5TXN3Yk14ajFIdi1CTkx2ZWx2QzF5OFR6LUx5azQ3dnNLaXJBOHNxc0tlWmtZcTFVelF4eXBSM2JkbHd2eTM0VHNXdHNtVUprZWtPVzctNlJsZHNmM20tU1N6Q1Q2cHFYSi1tNlhZNDNabTVuaEVGWmIydEhadTcyMlBURmw2aUJxOF9GTzR0dTZiNGZfOFlHaVpPZ1A1LXhhOEFtN1J5TEVNNWtMcGpyNkMzSl8xRnZsaTF1WTZrOUZmb0cxVURjSGFLS2dIYTQyZEJtTm90bEYxVWxNNXVPdTVjaVhYbXhxT3JsVDM5VjZMVFZKSE1tZnM9 Connector_AiAnthropic_API_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpENmFBWG16STFQUVZxNzZZRzRLYTA4X3lRanF1VkF4cU45OExNMzlsQmdISGFxTUxud1dXODBKcFhMVG9KNjdWVnlTTFFROVc3NDlsdlNHLUJXeG41NDBHaXhHR0VHVWl5UW9RNkVWbmlhakRKVW5pM0R4VHk0LUw0TV9LdkljNHdBLXJua21NQkl2b3l4UkVkMGN1YjBrMmJEeWtMay1jbmxrYWJNbUV0aktCXzU1djR2d2RSQXZORTNwcG92ZUVvVGMtQzQzTTVncEZTRGRtZUFIZWQ0dz09 -Connector_AiPerplexity_API_SECRET = DEV_ENC:Z0FBQUFBQm82Mzk2Q1MwZ0dNcUVBcUtuRDJIcTZkMXVvYnpjM3JEMzJiT1NKSHljX282ZDIyZTJYc09VSTdVNXAtOWU2UXp5S193NTk5dHJsWlFjRjhWektFOG1DVGY4ZUhHTXMzS0RPN1lNcF9nSlVWbW5BZ1hkZDVTejl6bVZNRFVvX29xamJidWRFMmtjQmkyRUQ2RUh6UTN1aWNPSUJBPT0= +Connector_AiPerplexity_API_SECRET = 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 Connector_AiTavily_API_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEQTdnUHMwd2pIaXNtMmtCTFREd0pyQXRKb1F5eGtHSnkyOGZiUnlBOFc0b3Vzcndrc3ViRm1nMDJIOEZKYWxqdWNkZGh5N0Z4R0JlQmxXSG5pVnJUR2VYckZhMWNMZ1FNeXJ3enJLVlpiblhOZTNleUg3ZzZyUzRZanFSeDlVMkI= Connector_AiPrivateLlm_API_SECRET = DEV_ENC:Z0FBQUFBQnBudkpGRHM5eFdUVmVZU1R1cHBwN1RlMUx4T0NlLTJLUFFVX3J2OElDWFpuZmJHVmp4Z3BNNWMwZUVVZUd2TFhRSjVmVkVlcFlVRWtybXh0ZHloZ01ZcnVvX195YjdlWVdEcjZSWFFTTlNBWUlaTlNoLWhqVFBIb0thVlBiaWhjYjFQOFY= Connector_AiMistral_API_SECRET = DEV_ENC:Z0FBQUFBQnBudkpGeEQxYUIxOHhia0JlQWpWQ2dWQWZzY3l6SWwyUnJoR1hRQWloX2lxb2lGNkc4UnA4U2tWNjJaYzB1d1hvNG9fWUp1N3V4OW9FMGhaWVhjSlVwWEc1X2loVDBSZDEtdHdfcTA5QkcxQTR4OHc4RkRzclJrU2d1RFZpNDJkRDRURlE= diff --git a/env_int.env b/env_int.env index 5fe11cbc..95e28430 100644 --- a/env_int.env +++ b/env_int.env @@ -44,7 +44,7 @@ APP_FRONTEND_URL = https://nyla-int.poweron-center.net # AI configuration Connector_AiOpenai_API_SECRET = INT_ENC:Z0FBQUFBQnBaSnM4MENkQ2xJVmE5WFZKUkh2SHJFby1YVXN3ZmVxRkptS3ZWRmlwdU93ZEJjSjlMV2NGbU5mS3NCdmFfcmFYTEJNZXFIQ3ozTWE4ZC1pemlQNk9wbjU1d3BPS0ZCTTZfOF8yWmVXMWx0TU1DamlJLVFhSTJXclZsY3hMVWlPcXVqQWtMdER4T252NHZUWEhUOTdIN1VGR3ltazEweXFqQ0lvb0hYWmxQQnpxb0JwcFNhRDNGWXdoRTVJWm9FalZpTUF5b1RqZlRaYnVKYkp0NWR5Vko1WWJ0Wmg2VWJzYXZ0Z3Q4UkpsTldDX2dsekhKMmM4YjRoa2RwemMwYVQwM2cyMFlvaU5mOTVTWGlROU8xY2ZVRXlxZzJqWkxURWlGZGI2STZNb0NpdEtWUnM9 Connector_AiAnthropic_API_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjT1ZlRWVJdVZMT3ljSFJDcFdxRFBRVkZhS204NnN5RDBlQ0tpenhTM0FFVktuWW9mWHNwRWx2dHB0eDBSZ0JFQnZKWlp6c01pVGREWHd1eGpERnU0Q2xhaks1clQ1ZXVsdnd2ZzhpNXNQS1BhY3FjSkdkVEhHalNaRGR4emhpakZncnpDQUVxOHVXQzVUWmtQc0FsYmFwTF9TSG5FOUFtWk5Ick1NcHFvY2s1T1c2WXlRUFFJZnh6TWhuaVpMYmppcDR0QUx0a0R6RXlwbGRYb1R4dzJkUT09 -Connector_AiPerplexity_API_SECRET = INT_ENC:Z0FBQUFBQm82Mzk2UWZJdUFhSW8yc3RKc0tKRXphd0xWMkZOVlFpSGZ4SGhFWnk0cTF5VjlKQVZjdS1QSWdkS0pUSWw4OFU5MjUxdTVQel9aeWVIZTZ5TXRuVmFkZG0zWEdTOGdHMHpsTzI0TGlWYURKU1Q0VVpKTlhxUk5FTmN6SUJScDZ3ZldIaUJZcWpaQVRiSEpyQm9tRTNDWk9KTnZBPT0= +Connector_AiPerplexity_API_SECRET = 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 Connector_AiTavily_API_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkdkJMTDY0akhXNzZDWHVYSEt1cDZoOWEzSktneHZEV2JndTNmWlNSMV9KbFNIZmQzeVlrNE5qUEIwcUlBSGM1a0hOZ3J6djIyOVhnZzI3M1dIUkdicl9FVXF3RGktMmlEYmhnaHJfWTdGUkktSXVUSGdQMC1vSEV6VE8zR2F1SVk= Connector_AiPrivateLlm_API_SECRET = INT_ENC:Z0FBQUFBQnBudkpGSjZ1NWh0aWc1R3Z4MHNaeS1HamtUbndhcUZFZDlqUDhjSmg5eHFfdlVkU0RsVkJ2UVRaMWs3aWhraG5jSlc0YkxNWHVmR2JoSW5ENFFCdkJBM0VienlKSnhzNnBKbTJOUTFKczRfWlQ3bWpmUkRTT1I1OGNUSTlQdExacGRpeXg= Connector_AiMistral_API_SECRET = INT_ENC:Z0FBQUFBQnBudkpGZTNtZ1E4TWIxSEU1OUlreUpxZkJIR0Vxcm9xRHRUbnBxbTQ1cXlkbnltWkJVdTdMYWZ4c3Fsam42TERWUTVhNzZFMU9xVjdyRGFCYml6bmZsZFd2YmJzemlrSWN6Q3o3X0NXX2xXNUQteTNONHdKYzJ5YVpLLWdhU2JhSTJQZnI= diff --git a/env_prod.env b/env_prod.env index af202370..5e60702c 100644 --- a/env_prod.env +++ b/env_prod.env @@ -44,7 +44,7 @@ APP_FRONTEND_URL = https://nyla.poweron-center.net # AI configuration Connector_AiOpenai_API_SECRET = PROD_ENC:Z0FBQUFBQnBaSnM4TWJOVm4xVkx6azRlNDdxN3UxLUdwY2hhdGYxRGp4VFJqYXZIcmkxM1ZyOWV2M0Z4MHdFNkVYQ0ROb1d6LUZFUEdvMHhLMEtXYVBCRzM5TlYyY3ROYWtJRk41cDZxd0tYYi00MjVqMTh4QVcyTXl0bmVocEFHbXQwREpwNi1vODdBNmwzazE5bkpNelE2WXpvblIzWlQwbGdEelI2WXFqT1RibXVHcjNWbVhwYzBOM25XTzNmTDAwUjRvYk4yNjIyZHc5c2RSZzREQUFCdUwyb0ZuOXN1dzI2c2FKdXI4NGxEbk92czZWamJXU3ZSbUlLejZjRklRRk4tLV9aVUFZekI2bTU4OHYxNTUybDg3RVo0ZTh6dXNKRW5GNXVackZvcm9laGI0X3R6V3M9 Connector_AiAnthropic_API_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3TnhYdlhSLW5RbXJyMHFXX0V0bHhuTDlTaFJsRDl2dTdIUTFtVFAwTE8tY3hLbzNSMnVTLXd3RUZualN3MGNzc1kwOTIxVUN2WW1rYi1TendFRVVBSVNqRFVjckEzNExyTGNaUkJLMmozazUwemI1cnhrcEtZVXJrWkdaVFFramp3MWZ6RmY2aGlRMXVEYjM2M3ZlbmxMdnNCRDM1QWR0Wmd6MWVnS1I1c01nV3hRLXg3d2NTZXVfTi1Wdm16UnRyNGsyRTZ0bG9TQ1g1OFB5Z002bmQ3QT09 -Connector_AiPerplexity_API_SECRET = PROD_ENC:Z0FBQUFBQm82Mzk2Q1FGRkJEUkI4LXlQbHYzT2RkdVJEcmM4WGdZTWpJTEhoeUF1NW5LUVpJdDBYN3k1WFN4a2FQSWJSQmd0U0xJbzZDTmFFN05FcXl0Z3V1OEpsZjYydV94TXVjVjVXRTRYSWdLMkd5XzZIbFV6emRCZHpuOUpQeThadE5xcDNDVGV1RHJrUEN0c1BBYXctZFNWcFRuVXhRPT0= +Connector_AiPerplexity_API_SECRET = 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 Connector_AiTavily_API_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z3NmItcDh6V0JpcE5Jc0NlUWZqcmllRHB5eDlNZmVnUlNVenhNTm5xWExzbjJqdE1GZ0hTSUYtb2dvdWNhTnlQNmVWQ2NGVDgwZ0MwMWZBMlNKWEhzdlF3TlZzTXhCZWM4Z1Uwb18tSTRoU1JBVTVkSkJHOTJwX291b3dPaVphVFg= Connector_AiPrivateLlm_API_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGanZ6U3pzZWkwXzVPWGtIQ040XzFrTXc5QWRnazdEeEktaUJ0akJmNnEzbWUzNHczLTJfc2dIdzBDY0FTaXZYcDhxNFdNbTNtbEJTb2VRZ0ZYd05hdlNLR1h6SUFzVml2Z1FLY1BjTl90UWozUGxtak1URnhhZmNDRWFTb0dKVUo= Connector_AiMistral_API_SECRET = PROD_ENC:Z0FBQUFBQnBudkpGc2tQc2lvMk1YZk01Q1dob1U5cnR0dG03WWE3WkpoOWo0SEpvLU9Rc2lCNDExdy1wZExaN3lpT2FEQkxnaHRmWmZUUUZUUUJmblZreGlpaFpOdnFhbzlEd1RsVVJtX216cmhxTm5BcTN2eUZ2T054cDE5bmlEamJ3NGR6MVpFQnA= diff --git a/modules/aicore/aicoreBase.py b/modules/aicore/aicoreBase.py index e39c0a54..2f998f1d 100644 --- a/modules/aicore/aicoreBase.py +++ b/modules/aicore/aicoreBase.py @@ -12,8 +12,8 @@ IMPORTANT: Model Registration Requirements """ from abc import ABC, abstractmethod -from typing import List, Dict, Any, Optional -from modules.datamodels.datamodelAi import AiModel +from typing import List, Dict, Any, Optional, AsyncGenerator, Union +from modules.datamodels.datamodelAi import AiModel, AiModelCall, AiModelResponse class BaseConnectorAi(ABC): @@ -102,3 +102,24 @@ class BaseConnectorAi(ABC): """Get only available models.""" models = self.getCachedModels() return [model for model in models if model.isAvailable] + + async def callAiBasicStream(self, modelCall: AiModelCall) -> AsyncGenerator[Union[str, AiModelResponse], None]: + """Stream AI response. Yields str deltas during generation, then final AiModelResponse. + + Default implementation: falls back to non-streaming callAiBasic. + Override in connectors that support streaming. + """ + response = await self.callAiBasic(modelCall) + if response.content: + yield response.content + yield response + + async def callEmbedding(self, modelCall: AiModelCall) -> AiModelResponse: + """Generate embeddings for input texts. Override in connectors that support embeddings. + + Reads texts from modelCall.embeddingInput. + Returns AiModelResponse with metadata["embeddings"] containing the vectors. + """ + raise NotImplementedError( + f"{self.__class__.__name__} does not support embeddings" + ) diff --git a/modules/aicore/aicorePluginAnthropic.py b/modules/aicore/aicorePluginAnthropic.py index 5809a203..85bbfa75 100644 --- a/modules/aicore/aicorePluginAnthropic.py +++ b/modules/aicore/aicorePluginAnthropic.py @@ -1,9 +1,10 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. +import json import logging import httpx import os -from typing import Dict, Any, List +from typing import Dict, Any, List, AsyncGenerator, Union from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from .aicoreBase import BaseConnectorAi @@ -61,13 +62,15 @@ class AiAnthropic(BaseConnectorAi): speedRating=6, # Slower due to high-quality processing qualityRating=10, # Best quality available functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.QUALITY, processingMode=ProcessingModeEnum.DETAILED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 9), (OperationTypeEnum.DATA_ANALYSE, 9), (OperationTypeEnum.DATA_GENERATE, 9), - (OperationTypeEnum.DATA_EXTRACT, 8) + (OperationTypeEnum.DATA_EXTRACT, 8), + (OperationTypeEnum.AGENT, 9), ), version="claude-sonnet-4-5-20250929", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.003 + (bytesReceived / 4 / 1000) * 0.015 @@ -85,13 +88,15 @@ class AiAnthropic(BaseConnectorAi): speedRating=9, # Very fast, lightweight model qualityRating=8, # Good quality, cost-efficient functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.SPEED, processingMode=ProcessingModeEnum.BASIC, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 8), (OperationTypeEnum.DATA_ANALYSE, 8), (OperationTypeEnum.DATA_GENERATE, 8), - (OperationTypeEnum.DATA_EXTRACT, 7) + (OperationTypeEnum.DATA_EXTRACT, 7), + (OperationTypeEnum.AGENT, 7), ), version="claude-haiku-4-5-20251001", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.001 + (bytesReceived / 4 / 1000) * 0.005 @@ -109,13 +114,15 @@ class AiAnthropic(BaseConnectorAi): speedRating=5, # Moderate latency, most capable qualityRating=10, # Top-tier intelligence functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.QUALITY, processingMode=ProcessingModeEnum.DETAILED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 10), - (OperationTypeEnum.DATA_ANALYSE, 10), + (OperationTypeEnum.DATA_ANALYSE, 8), (OperationTypeEnum.DATA_GENERATE, 10), - (OperationTypeEnum.DATA_EXTRACT, 9) + (OperationTypeEnum.DATA_EXTRACT, 9), + (OperationTypeEnum.AGENT, 10), ), version="claude-opus-4-6", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.005 + (bytesReceived / 4 / 1000) * 0.025 @@ -158,53 +165,15 @@ class AiAnthropic(BaseConnectorAi): HTTPException: For errors in API communication """ try: - # Extract parameters from modelCall - messages = modelCall.messages model = modelCall.model options = modelCall.options temperature = getattr(options, "temperature", None) if temperature is None: temperature = model.temperature maxTokens = model.maxTokens - - # Transform OpenAI-style messages to Anthropic format: - # - Move any 'system' role content to top-level 'system' - # - Keep only 'user'/'assistant' messages in the list - system_contents: List[str] = [] - converted_messages: List[Dict[str, Any]] = [] - for m in messages: - role = m.get("role") - content = m.get("content", "") - if role == "system": - # Collect system content; Anthropic expects top-level 'system' - if isinstance(content, list): - # Join text parts if provided as blocks - joined = "\n\n".join( - [ - (part.get("text") if isinstance(part, dict) else str(part)) - for part in content - ] - ) - system_contents.append(joined) - else: - system_contents.append(str(content)) - continue - # For Anthropic, content can be a string; pass through strings, collapse blocks - if isinstance(content, list): - # Collapse to text if blocks are provided - collapsed = "\n\n".join( - [ - (part.get("text") if isinstance(part, dict) else str(part)) - for part in content - ] - ) - converted_messages.append({"role": role, "content": collapsed}) - else: - converted_messages.append({"role": role, "content": content}) - system_prompt = "\n\n".join([s for s in system_contents if s]) if system_contents else None + converted_messages, system_prompt = _convertMessagesForAnthropic(modelCall.messages) - # Create Anthropic API payload payload: Dict[str, Any] = { "model": model.name, "messages": converted_messages, @@ -217,6 +186,13 @@ class AiAnthropic(BaseConnectorAi): payload["max_tokens"] = maxTokens if system_prompt: payload["system"] = system_prompt + + if modelCall.tools: + payload["tools"] = _convertToolsToAnthropicFormat(modelCall.tools) + if modelCall.toolChoice: + payload["tool_choice"] = modelCall.toolChoice + else: + payload["tool_choice"] = {"type": "auto"} response = await self.httpClient.post( model.apiUrl, @@ -244,29 +220,39 @@ class AiAnthropic(BaseConnectorAi): # Parse response anthropicResponse = response.json() - # Extract content from response + # Extract content and tool_use blocks from response content = "" + toolCalls = [] if "content" in anthropicResponse: if isinstance(anthropicResponse["content"], list): - # Content is a list of parts (in newer API versions) for part in anthropicResponse["content"]: if part.get("type") == "text": content += part.get("text", "") + elif part.get("type") == "tool_use": + toolCalls.append({ + "id": part.get("id", ""), + "type": "function", + "function": { + "name": part.get("name", ""), + "arguments": json.dumps(part.get("input", {})) if isinstance(part.get("input"), dict) else str(part.get("input", "{}")) + } + }) else: - # Direct content as string (in older API versions) content = anthropicResponse["content"] - # Debug logging for empty responses - if not content or content.strip() == "": + if not content and not toolCalls: logger.warning(f"Anthropic API returned empty content. Full response: {anthropicResponse}") content = "[Anthropic API returned empty response]" - # Return standardized response + metadata = {"response_id": anthropicResponse.get("id", "")} + if toolCalls: + metadata["toolCalls"] = toolCalls + return AiModelResponse( content=content, success=True, modelId=model.name, - metadata={"response_id": anthropicResponse.get("id", "")} + metadata=metadata ) except Exception as e: @@ -278,7 +264,102 @@ class AiAnthropic(BaseConnectorAi): error_detail += f" | Status: {e.status_code}" logger.error(error_detail, exc_info=True) raise HTTPException(status_code=500, detail=error_detail) - + + async def callAiBasicStream(self, modelCall: AiModelCall) -> AsyncGenerator[Union[str, AiModelResponse], None]: + """Stream Anthropic response. Yields str deltas, then final AiModelResponse.""" + try: + model = modelCall.model + options = modelCall.options + temperature = getattr(options, "temperature", None) + if temperature is None: + temperature = model.temperature + + converted, system_prompt = _convertMessagesForAnthropic(modelCall.messages) + + payload: Dict[str, Any] = { + "model": model.name, + "messages": converted, + "temperature": temperature, + "max_tokens": model.maxTokens, + "stream": True, + } + if system_prompt: + payload["system"] = system_prompt + if modelCall.tools: + payload["tools"] = _convertToolsToAnthropicFormat(modelCall.tools) + payload["tool_choice"] = modelCall.toolChoice or {"type": "auto"} + + fullContent = "" + toolUseBlocks: Dict[int, Dict[str, Any]] = {} + currentToolIdx = -1 + + async with self.httpClient.stream("POST", model.apiUrl, json=payload) as response: + if response.status_code != 200: + body = await response.aread() + raise HTTPException(status_code=500, detail=f"Anthropic stream error: {response.status_code} - {body.decode()}") + + async for line in response.aiter_lines(): + if not line.startswith("data: "): + continue + try: + event = json.loads(line[6:]) + except json.JSONDecodeError: + continue + + eventType = event.get("type", "") + + if eventType == "content_block_start": + block = event.get("content_block", {}) + idx = event.get("index", 0) + if block.get("type") == "tool_use": + currentToolIdx = idx + toolUseBlocks[idx] = { + "id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": "", + } + + elif eventType == "content_block_delta": + delta = event.get("delta", {}) + if delta.get("type") == "text_delta": + text = delta.get("text", "") + fullContent += text + yield text + elif delta.get("type") == "input_json_delta": + idx = event.get("index", currentToolIdx) + if idx in toolUseBlocks: + toolUseBlocks[idx]["arguments"] += delta.get("partial_json", "") + + elif eventType == "message_stop": + break + + metadata: Dict[str, Any] = {} + if toolUseBlocks: + metadata["toolCalls"] = [ + { + "id": tb["id"], + "type": "function", + "function": { + "name": tb["name"], + "arguments": tb["arguments"], + }, + } + for tb in toolUseBlocks.values() + ] + + yield AiModelResponse( + content=fullContent, + success=True, + modelId=model.name, + metadata=metadata, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error streaming Anthropic API: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Error streaming Anthropic API: {e}") + async def callAiImage(self, modelCall: AiModelCall) -> AiModelResponse: """ Analyzes an image using Anthropic's vision capabilities using standardized pattern. @@ -330,6 +411,20 @@ class AiAnthropic(BaseConnectorAi): mimeType = parts[0].replace("data:", "") base64Data = parts[1] + + import base64 as _b64 + try: + rawHead = _b64.b64decode(base64Data[:32]) + if rawHead[:3] == b"\xff\xd8\xff": + mimeType = "image/jpeg" + elif rawHead[:8] == b"\x89PNG\r\n\x1a\n": + mimeType = "image/png" + elif rawHead[:4] == b"GIF8": + mimeType = "image/gif" + elif rawHead[:4] == b"RIFF" and rawHead[8:12] == b"WEBP": + mimeType = "image/webp" + except Exception: + pass # Convert to Anthropic's vision format anthropicMessages = [{ @@ -424,4 +519,101 @@ class AiAnthropic(BaseConnectorAi): content="", success=False, error=f"Error during image analysis: {str(e)}" - ) \ No newline at end of file + ) + + +def _convertMessagesForAnthropic(messages: List[Dict[str, Any]]): + """Convert OpenAI-style messages to Anthropic format. Returns (messages, system_prompt).""" + system_contents: List[str] = [] + converted_messages: List[Dict[str, Any]] = [] + pendingToolResults: List[Dict[str, Any]] = [] + + def _flush(): + if not pendingToolResults: + return + converted_messages.append({"role": "user", "content": list(pendingToolResults)}) + pendingToolResults.clear() + + def _collapse(content): + if isinstance(content, list): + return "\n\n".join( + (part.get("text") if isinstance(part, dict) else str(part)) + for part in content + ) + return str(content) if content else "" + + for m in messages: + role = m.get("role") + content = m.get("content", "") + + if role == "system": + system_contents.append(_collapse(content)) + continue + if role == "tool": + pendingToolResults.append({ + "type": "tool_result", + "tool_use_id": m.get("tool_call_id", ""), + "content": str(content) if content else "", + }) + continue + + _flush() + + if role == "assistant" and m.get("tool_calls"): + contentBlocks = [] + textPart = _collapse(content) + if textPart: + contentBlocks.append({"type": "text", "text": textPart}) + for tc in m["tool_calls"]: + fn = tc.get("function", {}) + inputData = fn.get("arguments", "{}") + if isinstance(inputData, str): + try: + inputData = json.loads(inputData) + except (json.JSONDecodeError, ValueError): + inputData = {} + contentBlocks.append({ + "type": "tool_use", + "id": tc.get("id", ""), + "name": fn.get("name", ""), + "input": inputData, + }) + converted_messages.append({"role": "assistant", "content": contentBlocks}) + continue + + converted_messages.append({"role": role, "content": _collapse(content)}) + + _flush() + + merged: List[Dict[str, Any]] = [] + for msg in converted_messages: + if merged and merged[-1]["role"] == msg["role"]: + prev = merged[-1] + pc, nc = prev["content"], msg["content"] + if isinstance(pc, str) and isinstance(nc, str): + prev["content"] = pc + "\n\n" + nc + elif isinstance(pc, list) and isinstance(nc, list): + prev["content"] = pc + nc + elif isinstance(pc, str) and isinstance(nc, list): + prev["content"] = [{"type": "text", "text": pc}] + nc + elif isinstance(pc, list) and isinstance(nc, str): + prev["content"] = pc + [{"type": "text", "text": nc}] + else: + merged.append(msg) + + system_prompt = "\n\n".join([s for s in system_contents if s]) if system_contents else None + return merged, system_prompt + + +def _convertToolsToAnthropicFormat(openaiTools: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert OpenAI-style tool definitions to Anthropic format.""" + anthropicTools = [] + for tool in openaiTools: + if tool.get("type") == "function": + fn = tool["function"] + anthropicTools.append({ + "name": fn["name"], + "description": fn.get("description", ""), + "input_schema": fn.get("parameters", {"type": "object", "properties": {}}) + }) + return anthropicTools \ No newline at end of file diff --git a/modules/aicore/aicorePluginMistral.py b/modules/aicore/aicorePluginMistral.py index 92e0f924..a4f0e476 100644 --- a/modules/aicore/aicorePluginMistral.py +++ b/modules/aicore/aicorePluginMistral.py @@ -1,8 +1,9 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. import logging +import json as _json import httpx -from typing import List +from typing import List, Dict, Any, AsyncGenerator, Union from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from .aicoreBase import BaseConnectorAi @@ -66,13 +67,15 @@ class AiMistral(BaseConnectorAi): speedRating=8, # Good speed for complex tasks qualityRating=9, # High quality functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.BALANCED, processingMode=ProcessingModeEnum.ADVANCED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 9), (OperationTypeEnum.DATA_ANALYSE, 9), (OperationTypeEnum.DATA_GENERATE, 9), - (OperationTypeEnum.DATA_EXTRACT, 8) + (OperationTypeEnum.DATA_EXTRACT, 8), + (OperationTypeEnum.AGENT, 8), ), version="mistral-large-latest", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.0005 + (bytesReceived / 4 / 1000) * 0.0015 @@ -90,17 +93,40 @@ class AiMistral(BaseConnectorAi): speedRating=9, # Very fast, lightweight model qualityRating=7, # Good quality, cost-efficient functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.SPEED, processingMode=ProcessingModeEnum.BASIC, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 7), (OperationTypeEnum.DATA_ANALYSE, 7), (OperationTypeEnum.DATA_GENERATE, 8), - (OperationTypeEnum.DATA_EXTRACT, 7) + (OperationTypeEnum.DATA_EXTRACT, 7), + (OperationTypeEnum.AGENT, 6), ), version="mistral-small-latest", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00006 + (bytesReceived / 4 / 1000) * 0.00018 ), + AiModel( + name="mistral-embed", + displayName="Mistral Embed", + connectorType="mistral", + apiUrl="https://api.mistral.ai/v1/embeddings", + temperature=0.0, + maxTokens=0, + contextLength=8192, + costPer1kTokensInput=0.0001, # $0.10/M tokens + costPer1kTokensOutput=0.0, + speedRating=10, + qualityRating=7, + functionCall=self.callEmbedding, + priority=PriorityEnum.COST, + processingMode=ProcessingModeEnum.BASIC, + operationTypes=createOperationTypeRatings( + (OperationTypeEnum.EMBEDDING, 8) + ), + version="mistral-embed", + calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.0001 + ), AiModel( name="mistral-large-latest", displayName="Mistral Large 3 Vision", @@ -215,7 +241,105 @@ class AiMistral(BaseConnectorAi): except Exception as e: logger.error(f"Error calling Mistral API: {str(e)}") raise HTTPException(status_code=500, detail=f"Error calling Mistral API: {str(e)}") - + + async def callAiBasicStream(self, modelCall: AiModelCall) -> AsyncGenerator[Union[str, AiModelResponse], None]: + """Stream Mistral response. Yields str deltas, then final AiModelResponse.""" + try: + model = modelCall.model + options = modelCall.options + temperature = getattr(options, "temperature", None) + if temperature is None: + temperature = model.temperature + + payload: Dict[str, Any] = { + "model": model.name, + "messages": modelCall.messages, + "temperature": temperature, + "max_tokens": model.maxTokens, + "stream": True, + } + + fullContent = "" + + async with self.httpClient.stream("POST", model.apiUrl, json=payload) as response: + if response.status_code != 200: + body = await response.aread() + raise HTTPException(status_code=500, detail=f"Mistral stream error: {response.status_code} - {body.decode()}") + + async for line in response.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:] + if data.strip() == "[DONE]": + break + try: + chunk = _json.loads(data) + except _json.JSONDecodeError: + continue + + delta = chunk.get("choices", [{}])[0].get("delta", {}) + if "content" in delta and delta["content"]: + fullContent += delta["content"] + yield delta["content"] + + yield AiModelResponse( + content=fullContent, + success=True, + modelId=model.name, + metadata={}, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error streaming Mistral API: {e}") + raise HTTPException(status_code=500, detail=f"Error streaming Mistral API: {e}") + + async def callEmbedding(self, modelCall: AiModelCall) -> AiModelResponse: + """Generate embeddings via the Mistral Embeddings API. + + Reads texts from modelCall.embeddingInput. + Returns vectors in metadata["embeddings"]. + """ + try: + model = modelCall.model + texts = modelCall.embeddingInput or [] + if not texts: + return AiModelResponse( + content="", success=False, error="No embeddingInput provided" + ) + + payload = {"model": model.name, "input": texts} + response = await self.httpClient.post(model.apiUrl, json=payload) + + if response.status_code != 200: + errorMessage = f"Mistral Embedding API error: {response.status_code} - {response.text}" + logger.error(errorMessage) + if response.status_code == 429: + raise RateLimitExceededException(f"Rate limit exceeded for {model.name}") + raise HTTPException(status_code=500, detail=errorMessage) + + responseJson = response.json() + embeddings = [item["embedding"] for item in responseJson["data"]] + usage = responseJson.get("usage", {}) + + return AiModelResponse( + content="", + success=True, + modelId=model.name, + tokensUsed={ + "input": usage.get("prompt_tokens", 0), + "output": 0, + "total": usage.get("total_tokens", 0), + }, + metadata={"embeddings": embeddings}, + ) + except RateLimitExceededException: + raise + except Exception as e: + logger.error(f"Error calling Mistral Embedding API: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error calling Mistral Embedding API: {str(e)}") + async def callAiImage(self, modelCall: AiModelCall) -> AiModelResponse: """ Analyzes an image with the Mistral Vision API using standardized pattern. diff --git a/modules/aicore/aicorePluginOpenai.py b/modules/aicore/aicorePluginOpenai.py index 5465858c..366f7dde 100644 --- a/modules/aicore/aicorePluginOpenai.py +++ b/modules/aicore/aicorePluginOpenai.py @@ -1,8 +1,9 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. import logging +import json as _json import httpx -from typing import List +from typing import List, Dict, Any, AsyncGenerator, Union from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from .aicoreBase import BaseConnectorAi @@ -67,13 +68,15 @@ class AiOpenai(BaseConnectorAi): speedRating=8, # Good speed for complex tasks qualityRating=10, # High quality functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.BALANCED, processingMode=ProcessingModeEnum.ADVANCED, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 9), (OperationTypeEnum.DATA_ANALYSE, 10), (OperationTypeEnum.DATA_GENERATE, 10), - (OperationTypeEnum.DATA_EXTRACT, 7) + (OperationTypeEnum.DATA_EXTRACT, 7), + (OperationTypeEnum.AGENT, 9), ), version="gpt-4o", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.0025 + (bytesReceived / 4 / 1000) * 0.01 @@ -92,13 +95,15 @@ class AiOpenai(BaseConnectorAi): speedRating=9, # Very fast qualityRating=8, # Good quality, replaces gpt-3.5-turbo functionCall=self.callAiBasic, + functionCallStream=self.callAiBasicStream, priority=PriorityEnum.SPEED, processingMode=ProcessingModeEnum.BASIC, operationTypes=createOperationTypeRatings( (OperationTypeEnum.PLAN, 8), (OperationTypeEnum.DATA_ANALYSE, 8), (OperationTypeEnum.DATA_GENERATE, 9), - (OperationTypeEnum.DATA_EXTRACT, 7) + (OperationTypeEnum.DATA_EXTRACT, 7), + (OperationTypeEnum.AGENT, 8), ), version="gpt-4o-mini", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00015 + (bytesReceived / 4 / 1000) * 0.0006 @@ -125,6 +130,48 @@ class AiOpenai(BaseConnectorAi): version="gpt-4o", calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.0025 + (bytesReceived / 4 / 1000) * 0.01 ), + AiModel( + name="text-embedding-3-small", + displayName="OpenAI Embedding Small", + connectorType="openai", + apiUrl="https://api.openai.com/v1/embeddings", + temperature=0.0, + maxTokens=0, + contextLength=8191, + costPer1kTokensInput=0.00002, # $0.02/M tokens + costPer1kTokensOutput=0.0, + speedRating=10, + qualityRating=8, + functionCall=self.callEmbedding, + priority=PriorityEnum.COST, + processingMode=ProcessingModeEnum.BASIC, + operationTypes=createOperationTypeRatings( + (OperationTypeEnum.EMBEDDING, 10) + ), + version="text-embedding-3-small", + calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00002 + ), + AiModel( + name="text-embedding-3-large", + displayName="OpenAI Embedding Large", + connectorType="openai", + apiUrl="https://api.openai.com/v1/embeddings", + temperature=0.0, + maxTokens=0, + contextLength=8191, + costPer1kTokensInput=0.00013, # $0.13/M tokens + costPer1kTokensOutput=0.0, + speedRating=9, + qualityRating=10, + functionCall=self.callEmbedding, + priority=PriorityEnum.QUALITY, + processingMode=ProcessingModeEnum.ADVANCED, + operationTypes=createOperationTypeRatings( + (OperationTypeEnum.EMBEDDING, 10) + ), + version="text-embedding-3-large", + calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: (bytesSent / 4 / 1000) * 0.00013 + ), AiModel( name="dall-e-3", displayName="OpenAI DALL-E 3", @@ -179,6 +226,10 @@ class AiOpenai(BaseConnectorAi): "max_tokens": maxTokens } + if modelCall.tools: + payload["tools"] = modelCall.tools + payload["tool_choice"] = modelCall.toolChoice or "auto" + response = await self.httpClient.post( model.apiUrl, json=payload @@ -218,22 +269,150 @@ class AiOpenai(BaseConnectorAi): raise HTTPException(status_code=500, detail=error_message) responseJson = response.json() - content = responseJson["choices"][0]["message"]["content"] + choiceMessage = responseJson["choices"][0]["message"] + content = choiceMessage.get("content") or "" + + metadata = {"response_id": responseJson.get("id", "")} + if choiceMessage.get("tool_calls"): + metadata["toolCalls"] = choiceMessage["tool_calls"] return AiModelResponse( content=content, success=True, modelId=model.name, - metadata={"response_id": responseJson.get("id", "")} + metadata=metadata ) except ContextLengthExceededException: - # Re-raise context length exceptions without wrapping raise except Exception as e: logger.error(f"Error calling OpenAI API: {str(e)}") raise HTTPException(status_code=500, detail=f"Error calling OpenAI API: {str(e)}") - + + async def callAiBasicStream(self, modelCall: AiModelCall) -> AsyncGenerator[Union[str, AiModelResponse], None]: + """Stream OpenAI response. Yields str deltas, then final AiModelResponse.""" + try: + messages = modelCall.messages + model = modelCall.model + options = modelCall.options + temperature = getattr(options, "temperature", None) + if temperature is None: + temperature = model.temperature + + payload: Dict[str, Any] = { + "model": model.name, + "messages": messages, + "temperature": temperature, + "max_tokens": model.maxTokens, + "stream": True, + } + if modelCall.tools: + payload["tools"] = modelCall.tools + payload["tool_choice"] = modelCall.toolChoice or "auto" + + fullContent = "" + toolCallsAccum: Dict[int, Dict[str, Any]] = {} + + async with self.httpClient.stream("POST", model.apiUrl, json=payload) as response: + if response.status_code != 200: + body = await response.aread() + raise HTTPException(status_code=500, detail=f"OpenAI stream error: {response.status_code} - {body.decode()}") + + async for line in response.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:] + if data.strip() == "[DONE]": + break + try: + chunk = _json.loads(data) + except _json.JSONDecodeError: + continue + + delta = chunk.get("choices", [{}])[0].get("delta", {}) + + if "content" in delta and delta["content"]: + fullContent += delta["content"] + yield delta["content"] + + for tcDelta in delta.get("tool_calls", []): + idx = tcDelta.get("index", 0) + if idx not in toolCallsAccum: + toolCallsAccum[idx] = { + "id": tcDelta.get("id", ""), + "type": "function", + "function": {"name": "", "arguments": ""}, + } + if tcDelta.get("id"): + toolCallsAccum[idx]["id"] = tcDelta["id"] + fn = tcDelta.get("function", {}) + if fn.get("name"): + toolCallsAccum[idx]["function"]["name"] = fn["name"] + if fn.get("arguments"): + toolCallsAccum[idx]["function"]["arguments"] += fn["arguments"] + + metadata: Dict[str, Any] = {} + if toolCallsAccum: + metadata["toolCalls"] = [toolCallsAccum[i] for i in sorted(toolCallsAccum)] + + yield AiModelResponse( + content=fullContent, + success=True, + modelId=model.name, + metadata=metadata, + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error streaming OpenAI API: {e}") + raise HTTPException(status_code=500, detail=f"Error streaming OpenAI API: {e}") + + async def callEmbedding(self, modelCall: AiModelCall) -> AiModelResponse: + """Generate embeddings via the OpenAI Embeddings API. + + Reads texts from modelCall.embeddingInput. + Returns vectors in metadata["embeddings"]. + """ + try: + model = modelCall.model + texts = modelCall.embeddingInput or [] + if not texts: + return AiModelResponse( + content="", success=False, error="No embeddingInput provided" + ) + + payload = {"model": model.name, "input": texts} + response = await self.httpClient.post(model.apiUrl, json=payload) + + if response.status_code != 200: + errorMessage = f"OpenAI Embedding API error: {response.status_code} - {response.text}" + logger.error(errorMessage) + if response.status_code == 429: + raise RateLimitExceededException(f"Rate limit exceeded for {model.name}") + raise HTTPException(status_code=500, detail=errorMessage) + + responseJson = response.json() + embeddings = [item["embedding"] for item in responseJson["data"]] + usage = responseJson.get("usage", {}) + + return AiModelResponse( + content="", + success=True, + modelId=model.name, + tokensUsed={ + "input": usage.get("prompt_tokens", 0), + "output": 0, + "total": usage.get("total_tokens", 0), + }, + metadata={"embeddings": embeddings}, + ) + except (RateLimitExceededException, ContextLengthExceededException): + raise + except Exception as e: + logger.error(f"Error calling OpenAI Embedding API: {str(e)}") + raise HTTPException(status_code=500, detail=f"Error calling OpenAI Embedding API: {str(e)}") + async def callAiImage(self, modelCall: AiModelCall) -> AiModelResponse: """ Analyzes an image with the OpenAI Vision API using standardized pattern. diff --git a/modules/aicore/aicorePluginTavily.py b/modules/aicore/aicorePluginTavily.py index 635cd4eb..80f9d5b4 100644 --- a/modules/aicore/aicorePluginTavily.py +++ b/modules/aicore/aicorePluginTavily.py @@ -288,7 +288,16 @@ class AiTavily(BaseConnectorAi): if maxResults < minResults or maxResults > maxAllowedResults: raise ValueError(f"maxResults must be between {minResults} and {maxAllowedResults}") - # Perform actual API call + # Tavily enforces a 400-character query limit + TAVILY_MAX_QUERY_LENGTH = 400 + if len(query) > TAVILY_MAX_QUERY_LENGTH: + truncated = query[:TAVILY_MAX_QUERY_LENGTH] + lastSpace = truncated.rfind(' ') + if lastSpace > TAVILY_MAX_QUERY_LENGTH // 2: + truncated = truncated[:lastSpace] + logger.warning(f"Tavily query truncated from {len(query)} to {len(truncated)} chars") + query = truncated + # Build kwargs only for provided options to avoid API rejections kwargs: dict = {"query": query, "max_results": maxResults} if searchDepth is not None: diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index c4457117..77689ae5 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -41,6 +41,11 @@ class SystemTable(BaseModel): ) +def _isVectorType(sqlType: str) -> bool: + """Check if a SQL type string represents a pgvector column.""" + return sqlType.upper().startswith("VECTOR") + + def _isJsonbType(fieldType) -> bool: """Check if a type should be stored as JSONB in PostgreSQL.""" # Direct dict or list @@ -70,20 +75,26 @@ def _isJsonbType(fieldType) -> bool: def _get_model_fields(model_class) -> Dict[str, str]: - """Get all fields from Pydantic model and map to SQL types.""" - # Pydantic v2 + """Get all fields from Pydantic model and map to SQL types. + + Supports explicit db_type override via json_schema_extra={"db_type": "vector(1536)"}. + This enables pgvector columns without special-casing field names. + """ model_fields = model_class.model_fields fields = {} for field_name, field_info in model_fields.items(): - # Pydantic v2 field_type = field_info.annotation + # Explicit db_type override (e.g. vector columns) + extra = field_info.json_schema_extra + if extra and isinstance(extra, dict) and "db_type" in extra: + fields[field_name] = extra["db_type"] + continue + # Check for JSONB fields (Dict, List, or complex types) - # Purely type-based detection - no hardcoded field names if _isJsonbType(field_type): fields[field_name] = "JSONB" - # Simple type mapping elif field_type in (str, type(None)) or ( get_origin(field_type) is Union and type(None) in get_args(field_type) ): @@ -95,11 +106,45 @@ def _get_model_fields(model_class) -> Dict[str, str]: elif field_type == bool: fields[field_name] = "BOOLEAN" else: - fields[field_name] = "TEXT" # Default to TEXT + fields[field_name] = "TEXT" return fields +def _parseRecordFields(record: Dict[str, Any], fields: Dict[str, str], context: str = "") -> None: + """Parse record fields in-place: numeric typing, vector parsing, JSONB deserialization.""" + import json as _json + + for fieldName, fieldType in fields.items(): + if fieldName not in record: + continue + value = record[fieldName] + + if fieldType in ("DOUBLE PRECISION", "INTEGER") and value is not None: + try: + record[fieldName] = float(value) if fieldType == "DOUBLE PRECISION" else int(value) + except (ValueError, TypeError): + logger.warning(f"Could not convert {fieldName} to {fieldType} ({context}): {value}") + + elif _isVectorType(fieldType) and value is not None: + if isinstance(value, str): + try: + record[fieldName] = [float(v) for v in value.strip("[]").split(",")] + except (ValueError, TypeError): + logger.warning(f"Could not parse vector field {fieldName} ({context})") + elif isinstance(value, list): + pass # already a list + + elif fieldType == "JSONB" and value is not None: + try: + if isinstance(value, str): + record[fieldName] = _json.loads(value) + elif not isinstance(value, (dict, list)): + record[fieldName] = _json.loads(str(value)) + except (_json.JSONDecodeError, TypeError, ValueError): + logger.warning(f"Could not parse JSONB field {fieldName}, keeping as string ({context})") + + # Cache connectors by (host, database, port) to avoid duplicate inits for same database. # Thread safety: _connector_cache_lock protects cache access. userId is request-scoped via # contextvars to avoid races when concurrent requests share the same connector. @@ -187,6 +232,9 @@ class DatabaseConnector: # Thread safety self._lock = threading.Lock() + # pgvector extension state + self._vectorExtensionEnabled = False + # Initialize system table self._systemTableName = "_system" self._initializeSystemTable() @@ -500,10 +548,32 @@ class DatabaseConnector: self.connection.rollback() return False + def _ensureVectorExtension(self) -> bool: + """Enable pgvector extension if not already enabled. Called lazily on first vector table.""" + if self._vectorExtensionEnabled: + return True + try: + self._ensure_connection() + with self.connection.cursor() as cursor: + cursor.execute("CREATE EXTENSION IF NOT EXISTS vector") + self.connection.commit() + self._vectorExtensionEnabled = True + logger.info("pgvector extension enabled") + return True + except Exception as e: + logger.error(f"Failed to enable pgvector extension: {e}") + if hasattr(self, "connection") and self.connection: + self.connection.rollback() + return False + def _create_table_from_model(self, cursor, table: str, model_class: type) -> None: """Create table with columns matching Pydantic model fields.""" fields = _get_model_fields(model_class) + # Enable pgvector if any field uses vector type + if any(_isVectorType(sqlType) for sqlType in fields.values()): + self._ensureVectorExtension() + # Build column definitions with quoted identifiers to preserve exact case columns = ['"id" VARCHAR(255) PRIMARY KEY'] for field_name, sql_type in fields.items(): @@ -576,28 +646,25 @@ class DatabaseConnector: elif hasattr(value, "value"): value = value.value + # Handle vector fields (pgvector) - convert List[float] to string + elif col in fields and _isVectorType(fields[col]) and value is not None: + if isinstance(value, list): + value = f"[{','.join(str(v) for v in value)}]" + # Handle JSONB fields - ensure proper JSON format for PostgreSQL elif col in fields and fields[col] == "JSONB" and value is not None: import json if isinstance(value, (dict, list)): - # Convert Python objects to JSON string for PostgreSQL JSONB value = json.dumps(value) elif isinstance(value, str): - # Validate that it's valid JSON, if not, try to parse and re-serialize try: - # Test if it's already valid JSON json.loads(value) - # If successful, keep as is - pass except (json.JSONDecodeError, TypeError): - # If not valid JSON, convert to JSON string value = json.dumps(value) elif hasattr(value, 'model_dump'): - # Handle Pydantic models value = json.dumps(value.model_dump()) else: - # Convert other types to JSON value = json.dumps(value) values.append(value) @@ -635,46 +702,7 @@ class DatabaseConnector: record = dict(row) fields = _get_model_fields(model_class) - # Ensure numeric fields are properly typed and parse JSONB fields - for field_name, field_type in fields.items(): - # Ensure numeric fields (float/int) are properly typed - # psycopg2 may return them as strings in some environments (e.g., Azure PostgreSQL) - if field_type in ("DOUBLE PRECISION", "INTEGER") and field_name in record: - value = record[field_name] - if value is not None: - try: - if field_type == "DOUBLE PRECISION": - record[field_name] = float(value) - elif field_type == "INTEGER": - record[field_name] = int(value) - except (ValueError, TypeError): - # If conversion fails, log warning but keep original value - logger.warning( - f"Could not convert {field_name} to {field_type} for record {recordId}: {value}" - ) - elif ( - field_type == "JSONB" - and field_name in record - and record[field_name] is not None - ): - import json - - try: - if isinstance(record[field_name], str): - # Parse JSON string back to Python object - record[field_name] = json.loads(record[field_name]) - elif isinstance(record[field_name], (dict, list)): - # Already a Python object, keep as is - pass - else: - # Try to parse as JSON - record[field_name] = json.loads(str(record[field_name])) - except (json.JSONDecodeError, TypeError, ValueError): - # If parsing fails, keep as string - logger.warning( - f"Could not parse JSONB field {field_name}, keeping as string: {record[field_name]}" - ) - pass + _parseRecordFields(record, fields, f"record {recordId}") return record except Exception as e: @@ -737,55 +765,24 @@ class DatabaseConnector: cursor.execute(f'SELECT * FROM "{table}" ORDER BY "id"') records = [dict(row) for row in cursor.fetchall()] - # Handle JSONB fields for all records fields = _get_model_fields(model_class) - model_fields = model_class.model_fields # Get Pydantic model fields + modelFields = model_class.model_fields for record in records: - for field_name, field_type in fields.items(): - if field_type == "JSONB" and field_name in record: - if record[field_name] is None: - # Generic type-based default: List types -> [], Dict types -> {} - # Interfaces handle domain-specific defaults - field_info = model_fields.get(field_name) - if field_info: - field_annotation = field_info.annotation - # Check if it's a List type - if (field_annotation == list or - (hasattr(field_annotation, "__origin__") and - field_annotation.__origin__ is list)): - record[field_name] = [] - # Check if it's a Dict type - elif (field_annotation == dict or - (hasattr(field_annotation, "__origin__") and - field_annotation.__origin__ is dict)): - record[field_name] = {} - else: - record[field_name] = None - else: - record[field_name] = None - else: - import json - - try: - if isinstance(record[field_name], str): - # Parse JSON string back to Python object - record[field_name] = json.loads( - record[field_name] - ) - elif isinstance(record[field_name], (dict, list)): - # Already a Python object, keep as is - pass - else: - # Try to parse as JSON - record[field_name] = json.loads( - str(record[field_name]) - ) - except (json.JSONDecodeError, TypeError, ValueError): - # If parsing fails, keep as string - logger.warning( - f"Could not parse JSONB field {field_name}, keeping as string: {record[field_name]}" - ) - pass + _parseRecordFields(record, fields, f"table {table}") + # Set type-aware defaults for NULL JSONB fields + for fieldName, fieldType in fields.items(): + if fieldType == "JSONB" and fieldName in record and record[fieldName] is None: + fieldInfo = modelFields.get(fieldName) + if fieldInfo: + fieldAnnotation = fieldInfo.annotation + if (fieldAnnotation == list or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is list)): + record[fieldName] = [] + elif (fieldAnnotation == dict or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is dict)): + record[fieldName] = {} return records except Exception as e: @@ -936,70 +933,23 @@ class DatabaseConnector: cursor.execute(query, where_values) records = [dict(row) for row in cursor.fetchall()] - # Handle JSONB fields and ensure numeric types are correct fields = _get_model_fields(model_class) - model_fields = model_class.model_fields # Get Pydantic model fields + modelFields = model_class.model_fields for record in records: - for field_name, field_type in fields.items(): - # Ensure numeric fields (float/int) are properly typed - # psycopg2 may return them as strings in some environments (e.g., Azure PostgreSQL) - if field_type in ("DOUBLE PRECISION", "INTEGER") and field_name in record: - value = record[field_name] - if value is not None: - try: - if field_type == "DOUBLE PRECISION": - record[field_name] = float(value) - elif field_type == "INTEGER": - record[field_name] = int(value) - except (ValueError, TypeError): - # If conversion fails, log warning but keep original value - logger.warning( - f"Could not convert {field_name} to {field_type} for record {record.get('id', 'unknown')}: {value}" - ) - elif field_type == "JSONB" and field_name in record: - if record[field_name] is None: - # Generic type-based default: List types -> [], Dict types -> {} - # Interfaces handle domain-specific defaults - field_info = model_fields.get(field_name) - if field_info: - field_annotation = field_info.annotation - # Check if it's a List type - if (field_annotation == list or - (hasattr(field_annotation, "__origin__") and - field_annotation.__origin__ is list)): - record[field_name] = [] - # Check if it's a Dict type - elif (field_annotation == dict or - (hasattr(field_annotation, "__origin__") and - field_annotation.__origin__ is dict)): - record[field_name] = {} - else: - record[field_name] = None - else: - record[field_name] = None - else: - import json - - try: - if isinstance(record[field_name], str): - # Parse JSON string back to Python object - record[field_name] = json.loads( - record[field_name] - ) - elif isinstance(record[field_name], (dict, list)): - # Already a Python object, keep as is - pass - else: - # Try to parse as JSON - record[field_name] = json.loads( - str(record[field_name]) - ) - except (json.JSONDecodeError, TypeError, ValueError): - # If parsing fails, keep as string - logger.warning( - f"Could not parse JSONB field {field_name}, keeping as string: {record[field_name]}" - ) - pass + _parseRecordFields(record, fields, f"table {table}") + for fieldName, fieldType in fields.items(): + if fieldType == "JSONB" and fieldName in record and record[fieldName] is None: + fieldInfo = modelFields.get(fieldName) + if fieldInfo: + fieldAnnotation = fieldInfo.annotation + if (fieldAnnotation == list or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is list)): + record[fieldName] = [] + elif (fieldAnnotation == dict or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is dict)): + record[fieldName] = {} # If fieldFilter is available, reduce the fields if fieldFilter and isinstance(fieldFilter, list): @@ -1080,7 +1030,10 @@ class DatabaseConnector: existingRecord.update(record) # Save updated record - self._saveRecord(model_class, recordId, existingRecord) + saved = self._saveRecord(model_class, recordId, existingRecord) + if not saved: + table = model_class.__name__ + raise ValueError(f"Failed to save record {recordId} to table {table}") return existingRecord def recordDelete(self, model_class: type, recordId: str) -> bool: @@ -1127,6 +1080,85 @@ class DatabaseConnector: initialId = systemData.get(table) return initialId + def semanticSearch( + self, + modelClass: type, + vectorColumn: str, + queryVector: List[float], + limit: int = 10, + recordFilter: Dict[str, Any] = None, + minScore: float = None, + ) -> List[Dict[str, Any]]: + """Semantic search using pgvector cosine distance. + + Args: + modelClass: Pydantic model class for the table. + vectorColumn: Name of the vector column to search. + queryVector: Query vector as List[float]. + limit: Maximum number of results. + recordFilter: Additional WHERE filters (field: value). + minScore: Minimum cosine similarity (0.0 - 1.0). + + Returns: + List of records with an added '_score' field (cosine similarity), + sorted by similarity descending. + """ + table = modelClass.__name__ + + try: + if not self._ensureTableExists(modelClass): + return [] + + vectorStr = f"[{','.join(str(v) for v in queryVector)}]" + + whereConditions = [] + whereValues = [] + + if recordFilter: + for field, value in recordFilter.items(): + if value is None: + whereConditions.append(f'"{field}" IS NULL') + elif isinstance(value, (list, tuple)): + if not value: + whereConditions.append("1 = 0") + else: + whereConditions.append(f'"{field}" = ANY(%s)') + whereValues.append(list(value)) + else: + whereConditions.append(f'"{field}" = %s') + whereValues.append(value) + + if minScore is not None: + whereConditions.append( + f'1 - ("{vectorColumn}" <=> %s::vector) >= %s' + ) + whereValues.extend([vectorStr, minScore]) + + whereClause = "" + if whereConditions: + whereClause = " WHERE " + " AND ".join(whereConditions) + + query = ( + f'SELECT *, 1 - ("{vectorColumn}" <=> %s::vector) AS "_score" ' + f'FROM "{table}"{whereClause} ' + f'ORDER BY "{vectorColumn}" <=> %s::vector ' + f'LIMIT %s' + ) + params = [vectorStr] + whereValues + [vectorStr, limit] + + with self.connection.cursor() as cursor: + cursor.execute(query, params) + records = [dict(row) for row in cursor.fetchall()] + + fields = _get_model_fields(modelClass) + for record in records: + _parseRecordFields(record, fields, f"semanticSearch {table}") + + return records + except Exception as e: + logger.error(f"Error in semantic search on {table}: {e}") + return [] + def close(self): """Close the database connection.""" if ( @@ -1141,5 +1173,4 @@ class DatabaseConnector: try: self.close() except Exception: - # Ignore errors during cleanup pass diff --git a/modules/connectors/connectorProviderBase.py b/modules/connectors/connectorProviderBase.py new file mode 100644 index 00000000..71ad0ecf --- /dev/null +++ b/modules/connectors/connectorProviderBase.py @@ -0,0 +1,54 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Abstract base classes for the Provider-Connector architecture (1:n). + +One ProviderConnector per vendor (e.g. MsftConnector, GoogleConnector). +Each ProviderConnector exposes n ServiceAdapters (e.g. SharepointAdapter, OutlookAdapter). +All ServiceAdapters share the same access token from the UserConnection. +""" + +from abc import ABC, abstractmethod +from typing import List, Optional + + +class ServiceAdapter(ABC): + """Standardized operations for a single service of a provider.""" + + @abstractmethod + async def browse(self, path: str, filter: Optional[str] = None) -> list: + """List items (files/folders) at the given path.""" + ... + + @abstractmethod + async def download(self, path: str) -> bytes: + """Download a file and return its content bytes.""" + ... + + @abstractmethod + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + """Upload a file to the given path. Returns metadata of the created entry.""" + ... + + @abstractmethod + async def search(self, query: str, path: Optional[str] = None) -> list: + """Search for items matching the query.""" + ... + + +class ProviderConnector(ABC): + """One connector per provider. Manages a UserConnection + token. + Provides access to n services of the provider.""" + + def __init__(self, connection, accessToken: str): + self.connection = connection + self.accessToken = accessToken + + @abstractmethod + def getAvailableServices(self) -> List[str]: + """Which services does this provider offer?""" + ... + + @abstractmethod + def getServiceAdapter(self, service: str) -> ServiceAdapter: + """Return the ServiceAdapter for a specific service.""" + ... diff --git a/modules/connectors/connectorResolver.py b/modules/connectors/connectorResolver.py new file mode 100644 index 00000000..4304378e --- /dev/null +++ b/modules/connectors/connectorResolver.py @@ -0,0 +1,94 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""ConnectorResolver -- resolves a connectionId to the correct ProviderConnector and ServiceAdapter. + +Registry maps authority values to ProviderConnector classes. +The resolver loads the UserConnection, obtains a fresh token via SecurityService, +and instantiates the appropriate connector. +""" + +import logging +from typing import Dict, Any, Type, Optional + +from modules.connectors.connectorProviderBase import ProviderConnector, ServiceAdapter + +logger = logging.getLogger(__name__) + + +class ConnectorResolver: + """Resolves connectionId → ProviderConnector (with fresh token) → ServiceAdapter.""" + + _providerRegistry: Dict[str, Type[ProviderConnector]] = {} + + def __init__(self, securityService, dbInterface): + """ + Args: + securityService: SecurityService instance (for getFreshToken) + dbInterface: DB interface with getUserConnection(connectionId) + """ + self._security = securityService + self._db = dbInterface + self._ensureRegistered() + + def _ensureRegistered(self): + """Lazy-register known providers on first instantiation.""" + if ConnectorResolver._providerRegistry: + return + try: + from modules.connectors.providerMsft.connectorMsft import MsftConnector + ConnectorResolver._providerRegistry["msft"] = MsftConnector + except ImportError: + logger.warning("MsftConnector not available") + + try: + from modules.connectors.providerGoogle.connectorGoogle import GoogleConnector + ConnectorResolver._providerRegistry["google"] = GoogleConnector + except ImportError: + logger.debug("GoogleConnector not available (stub)") + + try: + from modules.connectors.providerFtp.connectorFtp import FtpConnector + ConnectorResolver._providerRegistry["local:ftp"] = FtpConnector + except ImportError: + logger.debug("FtpConnector not available (stub)") + + async def resolve(self, connectionId: str) -> ProviderConnector: + """Resolve connectionId to a ProviderConnector with a fresh access token.""" + connection = await self._loadConnection(connectionId) + if not connection: + raise ValueError(f"UserConnection not found: {connectionId}") + + authority = getattr(connection, "authority", None) + if not authority: + raise ValueError(f"Connection {connectionId} has no authority") + + authorityStr = authority.value if hasattr(authority, "value") else str(authority) + providerClass = self._providerRegistry.get(authorityStr) + if not providerClass: + raise ValueError(f"No ProviderConnector registered for authority: {authorityStr}") + + token = self._security.getFreshToken(connectionId) + if not token or not token.tokenAccess: + raise ValueError(f"No valid token for connection {connectionId}") + + return providerClass(connection, token.tokenAccess) + + async def resolveService(self, connectionId: str, service: str) -> ServiceAdapter: + """Resolve connectionId + service name to a concrete ServiceAdapter.""" + provider = await self.resolve(connectionId) + available = provider.getAvailableServices() + if service not in available: + raise ValueError(f"Service '{service}' not available. Options: {available}") + return provider.getServiceAdapter(service) + + async def _loadConnection(self, connectionId: str) -> Optional[Any]: + """Load UserConnection from DB.""" + try: + if hasattr(self._db, "getUserConnection"): + return self._db.getUserConnection(connectionId) + if hasattr(self._db, "loadRecord"): + from modules.datamodels.datamodelUam import UserConnection + return self._db.loadRecord(UserConnection, connectionId) + except Exception as e: + logger.error(f"Failed to load connection {connectionId}: {e}") + return None diff --git a/modules/services/serviceExtraction/extractors/__init__.py b/modules/connectors/providerFtp/__init__.py similarity index 59% rename from modules/services/serviceExtraction/extractors/__init__.py rename to modules/connectors/providerFtp/__init__.py index 085d67cf..ee198298 100644 --- a/modules/services/serviceExtraction/extractors/__init__.py +++ b/modules/connectors/providerFtp/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. - - +"""FTP/SFTP Provider Connector stub.""" diff --git a/modules/connectors/providerFtp/connectorFtp.py b/modules/connectors/providerFtp/connectorFtp.py new file mode 100644 index 00000000..3b04c0b7 --- /dev/null +++ b/modules/connectors/providerFtp/connectorFtp.py @@ -0,0 +1,48 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""FTP/SFTP ProviderConnector stub. + +Implements the ProviderConnector interface for FTP/SFTP file access. +Full implementation follows when FTP integration is prioritized. +""" + +import logging +from typing import List, Optional + +from modules.connectors.connectorProviderBase import ProviderConnector, ServiceAdapter +from modules.datamodels.datamodelDataSource import ExternalEntry + +logger = logging.getLogger(__name__) + + +class FtpFilesAdapter(ServiceAdapter): + """FTP files ServiceAdapter (stub).""" + + def __init__(self, accessToken: str): + self._accessToken = accessToken + + async def browse(self, path: str, filter: Optional[str] = None) -> List[ExternalEntry]: + logger.info(f"FTP browse stub: {path}") + return [] + + async def download(self, path: str) -> bytes: + logger.info(f"FTP download stub: {path}") + return b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + return {"error": "FTP upload not yet implemented"} + + async def search(self, query: str, path: Optional[str] = None) -> List[ExternalEntry]: + return [] + + +class FtpConnector(ProviderConnector): + """FTP ProviderConnector -- 1 connection -> files.""" + + def getAvailableServices(self) -> List[str]: + return ["files"] + + def getServiceAdapter(self, service: str) -> ServiceAdapter: + if service != "files": + raise ValueError(f"FTP only supports 'files' service, got '{service}'") + return FtpFilesAdapter(self.accessToken) diff --git a/modules/connectors/providerGoogle/__init__.py b/modules/connectors/providerGoogle/__init__.py new file mode 100644 index 00000000..0e09a79e --- /dev/null +++ b/modules/connectors/providerGoogle/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Google Provider Connector -- 1 Connection : n Services (Drive, Gmail).""" diff --git a/modules/connectors/providerGoogle/connectorGoogle.py b/modules/connectors/providerGoogle/connectorGoogle.py new file mode 100644 index 00000000..216b9019 --- /dev/null +++ b/modules/connectors/providerGoogle/connectorGoogle.py @@ -0,0 +1,232 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Google ProviderConnector -- Drive and Gmail via Google OAuth.""" + +import logging +from typing import Any, Dict, List, Optional + +import aiohttp + +from modules.connectors.connectorProviderBase import ProviderConnector, ServiceAdapter +from modules.datamodels.datamodelDataSource import ExternalEntry + +logger = logging.getLogger(__name__) + +_DRIVE_BASE = "https://www.googleapis.com/drive/v3" +_GMAIL_BASE = "https://gmail.googleapis.com/gmail/v1" + + +async def _googleGet(token: str, url: str) -> Dict[str, Any]: + headers = {"Authorization": f"Bearer {token}"} + timeout = aiohttp.ClientTimeout(total=20) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(url, headers=headers) as resp: + if resp.status in (200, 201): + return await resp.json() + errorText = await resp.text() + logger.warning(f"Google API {resp.status}: {errorText[:300]}") + return {"error": f"{resp.status}: {errorText[:200]}"} + except Exception as e: + return {"error": str(e)} + + +class DriveAdapter(ServiceAdapter): + """Google Drive ServiceAdapter -- browse files and folders.""" + + def __init__(self, accessToken: str): + self._token = accessToken + + async def browse(self, path: str, filter: Optional[str] = None) -> List[ExternalEntry]: + folderId = (path or "").strip("/") or "root" + query = f"'{folderId}' in parents and trashed=false" + fields = "files(id,name,mimeType,size,modifiedTime,parents)" + url = f"{_DRIVE_BASE}/files?q={query}&fields={fields}&pageSize=100&orderBy=folder,name" + + result = await _googleGet(self._token, url) + if "error" in result: + logger.warning(f"Google Drive browse failed: {result['error']}") + return [] + + entries = [] + for f in result.get("files", []): + isFolder = f.get("mimeType") == "application/vnd.google-apps.folder" + entries.append(ExternalEntry( + name=f.get("name", ""), + path=f"/{f.get('id', '')}", + isFolder=isFolder, + size=int(f.get("size", 0)) if f.get("size") else None, + mimeType=f.get("mimeType") if not isFolder else None, + metadata={"id": f.get("id"), "modifiedTime": f.get("modifiedTime")}, + )) + return entries + + _EXPORT_MIME_MAP = { + "application/vnd.google-apps.document": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.google-apps.spreadsheet": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.google-apps.presentation": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.google-apps.drawing": "application/pdf", + } + + async def download(self, path: str) -> bytes: + fileId = (path or "").strip("/") + if not fileId: + return b"" + headers = {"Authorization": f"Bearer {self._token}"} + timeout = aiohttp.ClientTimeout(total=60) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + # Try direct download first + url = f"{_DRIVE_BASE}/files/{fileId}?alt=media" + async with session.get(url, headers=headers) as resp: + if resp.status == 200: + return await resp.read() + logger.debug(f"Google Drive direct download returned {resp.status} for {fileId}") + + # If 403/404, check if it's a native Google file that needs export + metaUrl = f"{_DRIVE_BASE}/files/{fileId}?fields=mimeType,name" + async with session.get(metaUrl, headers=headers) as metaResp: + if metaResp.status != 200: + logger.warning(f"Google Drive metadata fetch failed ({metaResp.status}) for {fileId}") + return b"" + meta = await metaResp.json() + fileMime = meta.get("mimeType", "") + fileName = meta.get("name", fileId) + + exportMime = self._EXPORT_MIME_MAP.get(fileMime) + if not exportMime: + logger.warning(f"Google Drive: unsupported mimeType '{fileMime}' for file '{fileName}' ({fileId})") + return b"" + + exportUrl = f"{_DRIVE_BASE}/files/{fileId}/export?mimeType={exportMime}" + logger.info(f"Google Drive: exporting '{fileName}' as {exportMime}") + async with session.get(exportUrl, headers=headers) as exportResp: + if exportResp.status == 200: + return await exportResp.read() + logger.warning(f"Google Drive export failed ({exportResp.status}) for '{fileName}'") + except Exception as e: + logger.error(f"Google Drive download failed for {fileId}: {e}") + return b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + return {"error": "Google Drive upload not yet implemented"} + + async def search(self, query: str, path: Optional[str] = None) -> List[ExternalEntry]: + safeQuery = query.replace("'", "\\'") + folderId = (path or "").strip("/") + qParts = [f"name contains '{safeQuery}'", "trashed=false"] + if folderId: + qParts.append(f"'{folderId}' in parents") + qStr = " and ".join(qParts) + url = f"{_DRIVE_BASE}/files?q={qStr}&fields=files(id,name,mimeType,size)&pageSize=25" + logger.debug(f"Google Drive search: q={qStr}") + result = await _googleGet(self._token, url) + if "error" in result: + return [] + return [ + ExternalEntry( + name=f.get("name", ""), + path=f"/{f.get('id', '')}", + isFolder=f.get("mimeType") == "application/vnd.google-apps.folder", + size=int(f.get("size", 0)) if f.get("size") else None, + ) + for f in result.get("files", []) + ] + + +class GmailAdapter(ServiceAdapter): + """Gmail ServiceAdapter -- browse labels and messages.""" + + def __init__(self, accessToken: str): + self._token = accessToken + + async def browse(self, path: str, filter: Optional[str] = None) -> list: + cleanPath = (path or "").strip("/") + + if not cleanPath: + url = f"{_GMAIL_BASE}/users/me/labels" + result = await _googleGet(self._token, url) + if "error" in result: + logger.warning(f"Gmail labels failed: {result['error']}") + return [] + _SYSTEM_LABELS = {"INBOX", "SENT", "DRAFT", "TRASH", "SPAM", "STARRED", "IMPORTANT"} + labels = [] + for lbl in result.get("labels", []): + labelId = lbl.get("id", "") + labelName = lbl.get("name", labelId) + if lbl.get("type") == "system" and labelId not in _SYSTEM_LABELS: + continue + labels.append(ExternalEntry( + name=labelName, + path=f"/{labelId}", + isFolder=True, + metadata={"id": labelId, "type": lbl.get("type", "")}, + )) + labels.sort(key=lambda e: (0 if e.metadata.get("type") == "system" else 1, e.name)) + return labels + + url = f"{_GMAIL_BASE}/users/me/messages?labelIds={cleanPath}&maxResults=25" + result = await _googleGet(self._token, url) + if "error" in result: + return [] + + entries = [] + for msg in result.get("messages", [])[:25]: + msgId = msg.get("id", "") + detailUrl = f"{_GMAIL_BASE}/users/me/messages/{msgId}?format=metadata&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=Date" + detail = await _googleGet(self._token, detailUrl) + if "error" in detail: + entries.append(ExternalEntry(name=f"Message {msgId}", path=f"/{cleanPath}/{msgId}", isFolder=False)) + continue + headers = {h.get("name", ""): h.get("value", "") for h in detail.get("payload", {}).get("headers", [])} + entries.append(ExternalEntry( + name=headers.get("Subject", "(no subject)"), + path=f"/{cleanPath}/{msgId}", + isFolder=False, + metadata={ + "id": msgId, + "from": headers.get("From", ""), + "date": headers.get("Date", ""), + "snippet": detail.get("snippet", ""), + }, + )) + return entries + + async def download(self, path: str) -> bytes: + return b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + return {"error": "Gmail upload not applicable"} + + async def search(self, query: str, path: Optional[str] = None) -> list: + url = f"{_GMAIL_BASE}/users/me/messages?q={query}&maxResults=10" + result = await _googleGet(self._token, url) + if "error" in result: + return [] + return [ + ExternalEntry( + name=f"Message {m.get('id', '')}", + path=f"/{m.get('id', '')}", + isFolder=False, + metadata={"id": m.get("id")}, + ) + for m in result.get("messages", []) + ] + + +class GoogleConnector(ProviderConnector): + """Google ProviderConnector -- 1 connection -> Drive + Gmail.""" + + _SERVICE_MAP = { + "drive": DriveAdapter, + "gmail": GmailAdapter, + } + + def getAvailableServices(self) -> List[str]: + return list(self._SERVICE_MAP.keys()) + + def getServiceAdapter(self, service: str) -> ServiceAdapter: + adapterClass = self._SERVICE_MAP.get(service) + if not adapterClass: + raise ValueError(f"Unknown Google service: {service}. Available: {list(self._SERVICE_MAP.keys())}") + return adapterClass(self.accessToken) diff --git a/modules/connectors/providerMsft/__init__.py b/modules/connectors/providerMsft/__init__.py new file mode 100644 index 00000000..2229ecb3 --- /dev/null +++ b/modules/connectors/providerMsft/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Microsoft Provider Connector -- 1 Connection : n Services (SharePoint, Outlook, Teams, OneDrive).""" diff --git a/modules/connectors/providerMsft/connectorMsft.py b/modules/connectors/providerMsft/connectorMsft.py new file mode 100644 index 00000000..26aa3790 --- /dev/null +++ b/modules/connectors/providerMsft/connectorMsft.py @@ -0,0 +1,459 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Microsoft ProviderConnector -- one MSFT connection serves SharePoint, Outlook, Teams, OneDrive. + +All ServiceAdapters share the same OAuth access token obtained from the +UserConnection (authority=msft). +""" + +import logging +import aiohttp +import asyncio +from typing import Dict, Any, List, Optional + +from modules.connectors.connectorProviderBase import ProviderConnector, ServiceAdapter +from modules.datamodels.datamodelDataSource import ExternalEntry + +logger = logging.getLogger(__name__) + +_GRAPH_BASE = "https://graph.microsoft.com/v1.0" + + +class _GraphApiMixin: + """Shared Graph API call logic for all MSFT service adapters.""" + + def __init__(self, accessToken: str): + self._accessToken = accessToken + + async def _graphGet(self, endpoint: str) -> Dict[str, Any]: + return await _makeGraphCall(self._accessToken, endpoint, "GET") + + async def _graphPost(self, endpoint: str, data: Any = None) -> Dict[str, Any]: + return await _makeGraphCall(self._accessToken, endpoint, "POST", data) + + async def _graphPut(self, endpoint: str, data: bytes = None) -> Dict[str, Any]: + return await _makeGraphCall(self._accessToken, endpoint, "PUT", data) + + async def _graphDelete(self, endpoint: str) -> Dict[str, Any]: + return await _makeGraphCall(self._accessToken, endpoint, "DELETE") + + async def _graphDownload(self, endpoint: str) -> Optional[bytes]: + """Download binary content from Graph API.""" + headers = {"Authorization": f"Bearer {self._accessToken}"} + timeout = aiohttp.ClientTimeout(total=60) + url = f"{_GRAPH_BASE}/{endpoint.lstrip('/')}" + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(url, headers=headers) as resp: + if resp.status == 200: + return await resp.read() + logger.error(f"Download failed {resp.status}: {await resp.text()}") + return None + except Exception as e: + logger.error(f"Graph download error: {e}") + return None + + +async def _makeGraphCall( + token: str, endpoint: str, method: str = "GET", data: Any = None +) -> Dict[str, Any]: + """Execute a single Microsoft Graph API call.""" + url = f"{_GRAPH_BASE}/{endpoint.lstrip('/')}" + contentType = "application/json" + if method == "PUT" and isinstance(data, bytes): + contentType = "application/octet-stream" + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": contentType, + } + timeout = aiohttp.ClientTimeout(total=30) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + kwargs: Dict[str, Any] = {"headers": headers} + if data is not None: + kwargs["data"] = data + + if method == "GET": + async with session.get(url, **kwargs) as resp: + return await _handleResponse(resp) + elif method == "POST": + async with session.post(url, **kwargs) as resp: + return await _handleResponse(resp) + elif method == "PUT": + async with session.put(url, **kwargs) as resp: + return await _handleResponse(resp) + elif method == "DELETE": + async with session.delete(url, **kwargs) as resp: + if resp.status in (200, 204): + return {} + return await _handleResponse(resp) + + except asyncio.TimeoutError: + return {"error": f"Graph API timeout: {endpoint}"} + except Exception as e: + return {"error": f"Graph API error: {e}"} + + return {"error": f"Unsupported method: {method}"} + + +async def _handleResponse(resp: aiohttp.ClientResponse) -> Dict[str, Any]: + if resp.status in (200, 201): + return await resp.json() + errorText = await resp.text() + logger.error(f"Graph API {resp.status}: {errorText}") + return {"error": f"{resp.status}: {errorText}"} + + +def _graphItemToExternalEntry(item: Dict[str, Any], basePath: str = "") -> ExternalEntry: + isFolder = "folder" in item + return ExternalEntry( + name=item.get("name", ""), + path=f"{basePath}/{item.get('name', '')}" if basePath else item.get("name", ""), + isFolder=isFolder, + size=item.get("size"), + mimeType=item.get("file", {}).get("mimeType") if not isFolder else None, + lastModified=None, + metadata={ + "id": item.get("id"), + "webUrl": item.get("webUrl"), + "childCount": item.get("folder", {}).get("childCount") if isFolder else None, + }, + ) + + +# --------------------------------------------------------------------------- +# SharePoint Adapter +# --------------------------------------------------------------------------- + +class SharepointAdapter(_GraphApiMixin, ServiceAdapter): + """ServiceAdapter for SharePoint (files, sites) via Microsoft Graph.""" + + async def browse(self, path: str, filter: Optional[str] = None) -> List[ExternalEntry]: + """List items in a SharePoint folder. + + Path format: /sites// + Root "/" lists available sites via discovery. + """ + if not path or path == "/": + return await self._discoverSites() + + siteId, folderPath = _parseSharepointPath(path) + if not siteId: + return await self._discoverSites() + + if not folderPath or folderPath == "/": + endpoint = f"sites/{siteId}/drive/root/children" + else: + cleanPath = folderPath.lstrip("/") + endpoint = f"sites/{siteId}/drive/root:/{cleanPath}:/children" + + result = await self._graphGet(endpoint) + if "error" in result: + logger.warning(f"SharePoint browse failed: {result['error']}") + return [] + + entries = [_graphItemToExternalEntry(item, path) for item in result.get("value", [])] + if filter: + entries = [e for e in entries if _matchFilter(e, filter)] + return entries + + async def _discoverSites(self) -> List[ExternalEntry]: + """Discover accessible SharePoint sites.""" + result = await self._graphGet("sites?search=*&$top=50") + if "error" in result: + logger.warning(f"SharePoint site discovery failed: {result['error']}") + return [] + return [ + ExternalEntry( + name=s.get("displayName") or s.get("name", ""), + path=f"/sites/{s.get('id', '')}", + isFolder=True, + metadata={ + "id": s.get("id"), + "webUrl": s.get("webUrl"), + "description": s.get("description", ""), + }, + ) + for s in result.get("value", []) + if s.get("displayName") + ] + + async def download(self, path: str) -> bytes: + siteId, filePath = _parseSharepointPath(path) + if not siteId or not filePath: + return b"" + cleanPath = filePath.strip("/") + endpoint = f"sites/{siteId}/drive/root:/{cleanPath}:/content" + data = await self._graphDownload(endpoint) + return data or b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + siteId, folderPath = _parseSharepointPath(path) + if not siteId: + return {"error": "Invalid SharePoint path"} + cleanFolder = (folderPath or "").strip("/") + uploadPath = f"{cleanFolder}/{fileName}" if cleanFolder else fileName + endpoint = f"sites/{siteId}/drive/root:/{uploadPath}:/content" + result = await self._graphPut(endpoint, data) + return result + + async def search(self, query: str, path: Optional[str] = None) -> List[ExternalEntry]: + siteId, _ = _parseSharepointPath(path or "") + if not siteId: + return [] + safeQuery = query.replace("'", "''") + endpoint = f"sites/{siteId}/drive/root/search(q='{safeQuery}')" + result = await self._graphGet(endpoint) + if "error" in result: + return [] + return [_graphItemToExternalEntry(item) for item in result.get("value", [])] + + +# --------------------------------------------------------------------------- +# Outlook Adapter +# --------------------------------------------------------------------------- + +class OutlookAdapter(_GraphApiMixin, ServiceAdapter): + """ServiceAdapter for Outlook (mail, calendar) via Microsoft Graph.""" + + async def browse(self, path: str, filter: Optional[str] = None) -> List[ExternalEntry]: + """List mail folders or messages. + + path = "" or "/" → list mail folders + path = "/Inbox" → list messages in Inbox + """ + if not path or path == "/": + result = await self._graphGet("me/mailFolders") + if "error" in result: + return [] + return [ + ExternalEntry( + name=f.get("displayName", ""), + path=f"/{f.get('id', '')}", + isFolder=True, + metadata={"id": f.get("id"), "totalItemCount": f.get("totalItemCount")}, + ) + for f in result.get("value", []) + ] + + folderId = path.strip("/") + endpoint = f"me/mailFolders/{folderId}/messages?$top=25&$orderby=receivedDateTime desc" + result = await self._graphGet(endpoint) + if "error" in result: + return [] + return [ + ExternalEntry( + name=m.get("subject", "(no subject)"), + path=f"{path}/{m.get('id', '')}", + isFolder=False, + metadata={ + "id": m.get("id"), + "from": m.get("from", {}).get("emailAddress", {}).get("address"), + "receivedDateTime": m.get("receivedDateTime"), + "hasAttachments": m.get("hasAttachments", False), + }, + ) + for m in result.get("value", []) + ] + + async def download(self, path: str) -> bytes: + """Download a mail message as JSON bytes.""" + import json + messageId = path.strip("/").split("/")[-1] + result = await self._graphGet(f"me/messages/{messageId}") + if "error" in result: + return b"" + return json.dumps(result, ensure_ascii=False).encode("utf-8") + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + """Not applicable for Outlook in the file sense.""" + return {"error": "Upload not supported for Outlook"} + + async def search(self, query: str, path: Optional[str] = None) -> List[ExternalEntry]: + safeQuery = query.replace("'", "''") + endpoint = f"me/messages?$search=\"{safeQuery}\"&$top=25" + result = await self._graphGet(endpoint) + if "error" in result: + return [] + return [ + ExternalEntry( + name=m.get("subject", "(no subject)"), + path=f"/search/{m.get('id', '')}", + isFolder=False, + metadata={ + "id": m.get("id"), + "from": m.get("from", {}).get("emailAddress", {}).get("address"), + "receivedDateTime": m.get("receivedDateTime"), + }, + ) + for m in result.get("value", []) + ] + + async def sendMail( + self, to: List[str], subject: str, body: str, + cc: Optional[List[str]] = None, attachments: Optional[List[Dict]] = None + ) -> Dict[str, Any]: + """Send an email via Microsoft Graph.""" + import json + message: Dict[str, Any] = { + "subject": subject, + "body": {"contentType": "Text", "content": body}, + "toRecipients": [{"emailAddress": {"address": addr}} for addr in to], + } + if cc: + message["ccRecipients"] = [{"emailAddress": {"address": addr}} for addr in cc] + + payload = json.dumps({"message": message, "saveToSentItems": True}).encode("utf-8") + result = await self._graphPost("me/sendMail", payload) + if "error" in result: + return result + return {"success": True} + + +# --------------------------------------------------------------------------- +# Teams Adapter (Stub) +# --------------------------------------------------------------------------- + +class TeamsAdapter(_GraphApiMixin, ServiceAdapter): + """ServiceAdapter for Microsoft Teams -- browse joined teams and channels.""" + + async def browse(self, path: str, filter: Optional[str] = None) -> list: + cleanPath = (path or "").strip("/") + + if not cleanPath: + result = await self._graphGet("me/joinedTeams") + if "error" in result: + logger.warning(f"Teams browse failed: {result['error']}") + return [] + return [ + ExternalEntry( + name=t.get("displayName", ""), + path=f"/{t.get('id', '')}", + isFolder=True, + metadata={"id": t.get("id"), "description": t.get("description", "")}, + ) + for t in result.get("value", []) + ] + + parts = cleanPath.split("/", 1) + teamId = parts[0] + if len(parts) == 1: + result = await self._graphGet(f"teams/{teamId}/channels") + if "error" in result: + return [] + return [ + ExternalEntry( + name=ch.get("displayName", ""), + path=f"/{teamId}/{ch.get('id', '')}", + isFolder=True, + metadata={"id": ch.get("id"), "membershipType": ch.get("membershipType", "")}, + ) + for ch in result.get("value", []) + ] + + return [] + + async def download(self, path: str) -> bytes: + return b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + return {"error": "Teams upload not implemented"} + + async def search(self, query: str, path: Optional[str] = None) -> list: + return [] + + +# --------------------------------------------------------------------------- +# OneDrive Adapter (Stub -- similar to SharePoint but personal drive) +# --------------------------------------------------------------------------- + +class OneDriveAdapter(_GraphApiMixin, ServiceAdapter): + """ServiceAdapter stub for OneDrive (personal drive).""" + + async def browse(self, path: str, filter: Optional[str] = None) -> List[ExternalEntry]: + cleanPath = (path or "").strip("/") + if not cleanPath: + endpoint = "me/drive/root/children" + else: + endpoint = f"me/drive/root:/{cleanPath}:/children" + + result = await self._graphGet(endpoint) + if "error" in result: + return [] + entries = [_graphItemToExternalEntry(item, path) for item in result.get("value", [])] + if filter: + entries = [e for e in entries if _matchFilter(e, filter)] + return entries + + async def download(self, path: str) -> bytes: + cleanPath = (path or "").strip("/") + if not cleanPath: + return b"" + data = await self._graphDownload(f"me/drive/root:/{cleanPath}:/content") + return data or b"" + + async def upload(self, path: str, data: bytes, fileName: str) -> dict: + cleanPath = (path or "").strip("/") + uploadPath = f"{cleanPath}/{fileName}" if cleanPath else fileName + endpoint = f"me/drive/root:/{uploadPath}:/content" + return await self._graphPut(endpoint, data) + + async def search(self, query: str, path: Optional[str] = None) -> List[ExternalEntry]: + safeQuery = query.replace("'", "''") + endpoint = f"me/drive/root/search(q='{safeQuery}')" + result = await self._graphGet(endpoint) + if "error" in result: + return [] + return [_graphItemToExternalEntry(item) for item in result.get("value", [])] + + +# --------------------------------------------------------------------------- +# MsftConnector (1:n) +# --------------------------------------------------------------------------- + +class MsftConnector(ProviderConnector): + """Microsoft ProviderConnector -- 1 connection → n services.""" + + _SERVICE_MAP = { + "sharepoint": SharepointAdapter, + "outlook": OutlookAdapter, + "teams": TeamsAdapter, + "onedrive": OneDriveAdapter, + } + + def getAvailableServices(self) -> List[str]: + return list(self._SERVICE_MAP.keys()) + + def getServiceAdapter(self, service: str) -> ServiceAdapter: + adapterClass = self._SERVICE_MAP.get(service) + if not adapterClass: + raise ValueError(f"Unknown MSFT service: {service}. Available: {list(self._SERVICE_MAP.keys())}") + return adapterClass(self.accessToken) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _parseSharepointPath(path: str) -> tuple: + """Parse a SharePoint path into (siteId, innerPath). + + Expected format: /sites// + Also accepts bare siteId if no /sites/ prefix. + """ + if not path: + return ("", "") + clean = path.strip("/") + if clean.startswith("sites/"): + parts = clean.split("/", 2) + siteId = parts[1] if len(parts) > 1 else "" + innerPath = parts[2] if len(parts) > 2 else "" + return (siteId, innerPath) + parts = clean.split("/", 1) + return (parts[0], parts[1] if len(parts) > 1 else "") + + +def _matchFilter(entry: ExternalEntry, pattern: str) -> bool: + """Simple glob-like filter (supports * wildcard).""" + import fnmatch + return fnmatch.fnmatch(entry.name.lower(), pattern.lower()) diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py index b94422a7..296500aa 100644 --- a/modules/datamodels/datamodelAi.py +++ b/modules/datamodels/datamodelAi.py @@ -26,6 +26,12 @@ class OperationTypeEnum(str, Enum): WEB_SEARCH_DATA = "webSearch" # Returns list of URLs only WEB_CRAWL = "webCrawl" # Web crawl for a given URL + # Agent Operations + AGENT = "agent" # Agent loop: reasoning + tool use + + # Embedding Operations + EMBEDDING = "embedding" # Text → vector conversion for semantic search + # Speech Operations (dedicated pipeline, bypasses standard model selection) SPEECH_TEAMS = "speechTeams" # Teams Meeting AI analysis: decide if/how to respond @@ -102,6 +108,7 @@ class AiModel(BaseModel): # Function reference (not serialized) functionCall: Optional[Callable] = Field(default=None, exclude=True, description="Function to call for this model") + functionCallStream: Optional[Callable] = Field(default=None, exclude=True, description="Streaming function: yields str deltas, then final AiModelResponse") calculatepriceCHF: Optional[Callable] = Field(default=None, exclude=True, description="Function to calculate price in USD") # Selection criteria - capabilities with ratings @@ -155,10 +162,12 @@ class AiCallOptions(BaseModel): class AiCallRequest(BaseModel): """Centralized AI call request payload for interface use.""" - prompt: str = Field(description="The user prompt") + prompt: str = Field(default="", description="The user prompt") context: Optional[str] = Field(default=None, description="Optional external context (e.g., extracted docs)") options: AiCallOptions = Field(default_factory=AiCallOptions) - contentParts: Optional[List['ContentPart']] = None # NEW: Content parts for model-aware chunking + contentParts: Optional[List['ContentPart']] = None # Content parts for model-aware chunking + messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="OpenAI-style messages for multi-turn agent conversations") + tools: Optional[List[Dict[str, Any]]] = Field(default=None, description="Tool definitions for native function calling") class AiCallResponse(BaseModel): @@ -172,14 +181,19 @@ class AiCallResponse(BaseModel): bytesSent: int = Field(default=0, description="Input data size in bytes") bytesReceived: int = Field(default=0, description="Output data size in bytes") errorCount: int = Field(default=0, description="0 for success, 1+ for errors") + toolCalls: Optional[List[Dict[str, Any]]] = Field(default=None, description="Tool calls from native function calling") + metadata: Optional[Dict[str, Any]] = Field(default=None, description="Additional response metadata (e.g. embeddings vectors)") class AiModelCall(BaseModel): """Standardized input for AI model calls.""" - messages: List[Dict[str, Any]] = Field(description="Messages in OpenAI format (role, content)") + messages: List[Dict[str, Any]] = Field(default_factory=list, description="Messages in OpenAI format (role, content)") model: Optional[AiModel] = Field(default=None, description="The AI model being called") options: AiCallOptions = Field(default_factory=AiCallOptions, description="Additional model-specific options") + tools: Optional[List[Dict[str, Any]]] = Field(default=None, description="Tool definitions for native function calling") + toolChoice: Optional[Any] = Field(default=None, description="Tool choice: 'auto', 'none', or specific tool") + embeddingInput: Optional[List[str]] = Field(default=None, description="Input texts for embedding models (used instead of messages)") model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/modules/datamodels/datamodelBilling.py b/modules/datamodels/datamodelBilling.py index 2d78d95b..f33c84e3 100644 --- a/modules/datamodels/datamodelBilling.py +++ b/modules/datamodels/datamodelBilling.py @@ -123,6 +123,12 @@ class BillingTransaction(BaseModel): aicoreProvider: Optional[str] = Field(None, description="AICore provider (anthropic, openai, etc.)") aicoreModel: Optional[str] = Field(None, description="AICore model name (e.g., claude-4-sonnet, gpt-4o)") createdByUserId: Optional[str] = Field(None, description="User who created/caused this transaction") + + # AI call metadata (for per-call analytics) + processingTime: Optional[float] = Field(None, description="Processing time in seconds") + bytesSent: Optional[int] = Field(None, description="Bytes sent to AI model") + bytesReceived: Optional[int] = Field(None, description="Bytes received from AI model") + errorCount: Optional[int] = Field(None, description="Number of errors in this call") registerModelLabels( diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 8400ce2e..33a8ca7b 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -1,6 +1,6 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""Chat models: ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument.""" +"""Chat models: ChatWorkflow, ChatMessage, ChatLog, ChatDocument.""" from typing import List, Dict, Any, Optional from enum import Enum @@ -10,44 +10,6 @@ from modules.shared.timeUtils import getUtcTimestamp import uuid -class ChatStat(BaseModel): - """Statistics for chat operations. User-owned, no mandate context.""" - model_config = {"populate_by_name": True, "extra": "allow"} # Allow DB system fields - - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), description="Primary key" - ) - workflowId: Optional[str] = Field( - None, description="Foreign key to workflow (for workflow stats)" - ) - processingTime: Optional[float] = Field( - None, description="Processing time in seconds" - ) - bytesSent: Optional[int] = Field(None, description="Number of bytes sent") - bytesReceived: Optional[int] = Field(None, description="Number of bytes received") - errorCount: Optional[int] = Field(None, description="Number of errors encountered") - process: Optional[str] = Field(None, description="The process that delivers the stats data (e.g. 'action.outlook.readMails', 'ai.process.document.name')") - engine: Optional[str] = Field(None, description="The engine used (e.g. 'ai.anthropic.35', 'ai.tavily.basic', 'renderer.docx')") - priceCHF: Optional[float] = Field(None, description="Calculated price in USD for the operation") - - -registerModelLabels( - "ChatStat", - {"en": "Chat Statistics", "fr": "Statistiques de chat"}, - { - "id": {"en": "ID", "fr": "ID"}, - "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, - "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, - "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, - "bytesReceived": {"en": "Bytes Received", "fr": "Octets reçus"}, - "errorCount": {"en": "Error Count", "fr": "Nombre d'erreurs"}, - "process": {"en": "Process", "fr": "Processus"}, - "engine": {"en": "Engine", "fr": "Moteur"}, - "priceCHF": {"en": "Price CHF", "fr": "Prix CHF"}, - }, -) - - class ChatLog(BaseModel): """Log entries for chat workflows. User-owned, no mandate context.""" id: str = Field( @@ -322,7 +284,6 @@ class ChatWorkflow(BaseModel): startedAt: float = Field(default_factory=getUtcTimestamp, description="When the workflow started (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) logs: List[ChatLog] = Field(default_factory=list, description="Workflow logs", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) messages: List[ChatMessage] = Field(default_factory=list, description="Messages in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) - stats: List[ChatStat] = Field(default_factory=list, description="Workflow statistics list", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) tasks: list = Field(default_factory=list, description="List of tasks in the workflow", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) workflowMode: WorkflowModeEnum = Field(default=WorkflowModeEnum.WORKFLOW_DYNAMIC, description="Workflow mode selector", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ { diff --git a/modules/datamodels/datamodelContent.py b/modules/datamodels/datamodelContent.py new file mode 100644 index 00000000..b2c87ed8 --- /dev/null +++ b/modules/datamodels/datamodelContent.py @@ -0,0 +1,58 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Content Object data models for the container and content extraction pipeline. + +Physical layer: Container hierarchy (ZIP, Folder, File) +Logical layer: Scalar content objects (text, image, videostream, audiostream, other) + +The entire extraction pipeline up to ContentObjects runs without AI. +""" + +from typing import Dict, Any, List, Optional +from pydantic import BaseModel, Field +import uuid + + +class ContainerLimitError(Exception): + """Raised when container extraction exceeds safety limits (size, depth, file count).""" + pass + + +class ContentContextRef(BaseModel): + """Reference to the origin context within a container/file.""" + containerPath: str = Field(description="e.g. 'archiv.zip/folder-a/report.pdf'") + location: str = Field(default="", description="e.g. 'page:5/region:bottomLeft'") + label: Optional[str] = Field(default=None, description="e.g. 'Abbildung 3: Uebersicht'") + pageIndex: Optional[int] = Field(default=None, description="Page number (PDF, DOCX)") + sectionId: Optional[str] = Field(default=None, description="Section/Heading ID") + sheetName: Optional[str] = Field(default=None, description="Sheet name (XLSX)") + slideIndex: Optional[int] = Field(default=None, description="Slide number (PPTX)") + + +class ContentObject(BaseModel): + """Scalar content object extracted from a file. No AI involved.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + fileId: str = Field(description="FK to the physical file") + contentType: str = Field(description="text, image, videostream, audiostream, other") + data: str = Field(default="", description="Content data (text, base64, URL)") + contextRef: ContentContextRef = Field(default_factory=ContentContextRef) + metadata: Dict[str, Any] = Field(default_factory=dict) + sequence: int = Field(default=0, description="Order within the context") + + +class ContentObjectSummary(BaseModel): + """Compact description of a content object for the FileContentIndex.""" + id: str = Field(description="Content object ID") + contentType: str = Field(description="text, image, videostream, audiostream, other") + contextRef: ContentContextRef = Field(default_factory=ContentContextRef) + charCount: Optional[int] = Field(default=None, description="Only for text") + dimensions: Optional[str] = Field(default=None, description="Only for image/video (e.g. '1920x1080')") + duration: Optional[float] = Field(default=None, description="Only for audio/video (seconds)") + + +class FileEntry(BaseModel): + """A file extracted from a container (ZIP, TAR, Folder).""" + path: str = Field(description="Relative path within the container") + data: bytes = Field(description="File content bytes") + mimeType: str = Field(description="Detected MIME type") + size: int = Field(description="File size in bytes") diff --git a/modules/datamodels/datamodelDataSource.py b/modules/datamodels/datamodelDataSource.py new file mode 100644 index 00000000..86e0c7ec --- /dev/null +++ b/modules/datamodels/datamodelDataSource.py @@ -0,0 +1,58 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""DataSource and ExternalEntry models for external data integration. + +DataSource links a UserConnection to an external path (SharePoint folder, +Google Drive folder, FTP directory, etc.) for agent-accessible data containers. +""" + +from typing import Dict, Any, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class DataSource(BaseModel): + """Configured external data source linked to a UserConnection.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + connectionId: str = Field(description="FK to UserConnection") + sourceType: str = Field(description="sharepointFolder, googleDriveFolder, outlookFolder, ftpFolder") + path: str = Field(description="External path (e.g. '/sites/MySite/Documents/Reports')") + label: str = Field(description="User-visible label") + featureInstanceId: Optional[str] = Field(default=None, description="Scoped to feature instance") + mandateId: Optional[str] = Field(default=None, description="Mandate scope") + userId: str = Field(default="", description="Owner user ID") + autoSync: bool = Field(default=False, description="Automatically sync on schedule") + lastSynced: Optional[float] = Field(default=None, description="Last sync timestamp") + createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") + + +registerModelLabels( + "DataSource", + {"en": "Data Source", "de": "Datenquelle", "fr": "Source de données"}, + { + "id": {"en": "ID", "de": "ID", "fr": "ID"}, + "connectionId": {"en": "Connection ID", "de": "Verbindungs-ID", "fr": "ID de connexion"}, + "sourceType": {"en": "Source Type", "de": "Quellentyp", "fr": "Type de source"}, + "path": {"en": "Path", "de": "Pfad", "fr": "Chemin"}, + "label": {"en": "Label", "de": "Bezeichnung", "fr": "Libellé"}, + "featureInstanceId": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de fonctionnalité"}, + "mandateId": {"en": "Mandate ID", "de": "Mandanten-ID", "fr": "ID du mandat"}, + "userId": {"en": "User ID", "de": "Benutzer-ID", "fr": "ID utilisateur"}, + "autoSync": {"en": "Auto Sync", "de": "Auto-Sync", "fr": "Synchro auto"}, + "lastSynced": {"en": "Last Synced", "de": "Letzter Sync", "fr": "Dernier sync"}, + "createdAt": {"en": "Created At", "de": "Erstellt am", "fr": "Créé le"}, + }, +) + + +class ExternalEntry(BaseModel): + """An item (file or folder) from an external data source.""" + name: str = Field(description="Item name") + path: str = Field(description="Full path within the source") + isFolder: bool = Field(default=False, description="True if directory/folder") + size: Optional[int] = Field(default=None, description="File size in bytes") + mimeType: Optional[str] = Field(default=None, description="MIME type (files only)") + lastModified: Optional[float] = Field(default=None, description="Last modification timestamp") + metadata: Dict[str, Any] = Field(default_factory=dict, description="Provider-specific metadata") diff --git a/modules/datamodels/datamodelExtraction.py b/modules/datamodels/datamodelExtraction.py index 65f84de0..71918092 100644 --- a/modules/datamodels/datamodelExtraction.py +++ b/modules/datamodels/datamodelExtraction.py @@ -73,7 +73,7 @@ class ExtractionOptions(BaseModel): """Options for document extraction and processing with clear data structures.""" # Core extraction parameters - prompt: str = Field(description="Extraction prompt for AI processing") + prompt: str = Field(default="", description="Extraction prompt for AI processing") processDocumentsIndividually: bool = Field(default=True, description="Process each document separately") # Image processing parameters @@ -81,7 +81,7 @@ class ExtractionOptions(BaseModel): imageQuality: int = Field(default=85, ge=1, le=100, description="Image quality (1-100)") # Merging strategy - mergeStrategy: MergeStrategy = Field(description="Strategy for merging extraction results") + mergeStrategy: MergeStrategy = Field(default_factory=MergeStrategy, description="Strategy for merging extraction results") # Optional chunking parameters (for backward compatibility) chunkAllowed: Optional[bool] = Field(default=None, description="Whether chunking is allowed") diff --git a/modules/datamodels/datamodelFileFolder.py b/modules/datamodels/datamodelFileFolder.py new file mode 100644 index 00000000..b7a19915 --- /dev/null +++ b/modules/datamodels/datamodelFileFolder.py @@ -0,0 +1,32 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""FileFolder: hierarchical folder structure for file organization.""" + +from typing import Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class FileFolder(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + name: str = Field(description="Folder name", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": True}) + parentId: Optional[str] = Field(default=None, description="Parent folder ID (null = root)", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) + mandateId: Optional[str] = Field(default=None, description="Mandate context", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + featureInstanceId: Optional[str] = Field(default=None, description="Feature instance context", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) + createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + + +registerModelLabels( + "FileFolder", + {"en": "File Folder", "fr": "Dossier de fichiers"}, + { + "id": {"en": "ID", "fr": "ID"}, + "name": {"en": "Name", "fr": "Nom"}, + "parentId": {"en": "Parent Folder", "fr": "Dossier parent"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, + "createdAt": {"en": "Created At", "fr": "Créé le"}, + }, +) diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index 588097e4..e14879a0 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -2,7 +2,7 @@ # All rights reserved. """File-related datamodels: FileItem, FilePreview, FileData.""" -from typing import Dict, Any, Optional, Union +from typing import Dict, Any, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp @@ -20,6 +20,10 @@ class FileItem(BaseModel): fileHash: str = Field(description="Hash of the file", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) fileSize: int = Field(description="Size of the file in bytes", json_schema_extra={"frontend_type": "integer", "frontend_readonly": True, "frontend_required": False}) creationDate: float = Field(default_factory=getUtcTimestamp, description="Date when the file was created (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) + tags: Optional[List[str]] = Field(default=None, description="Tags for categorization and search", json_schema_extra={"frontend_type": "tags", "frontend_readonly": False, "frontend_required": False}) + folderId: Optional[str] = Field(default=None, description="ID of the parent folder", json_schema_extra={"frontend_type": "text", "frontend_readonly": False, "frontend_required": False}) + description: Optional[str] = Field(default=None, description="User-provided description of the file", json_schema_extra={"frontend_type": "textarea", "frontend_readonly": False, "frontend_required": False}) + status: Optional[str] = Field(default=None, description="Processing status: pending, extracted, embedding, indexed, failed", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) registerModelLabels( "FileItem", @@ -33,6 +37,10 @@ registerModelLabels( "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, "creationDate": {"en": "Creation Date", "fr": "Date de création"}, + "tags": {"en": "Tags", "fr": "Tags"}, + "folderId": {"en": "Folder ID", "fr": "ID du dossier"}, + "description": {"en": "Description", "fr": "Description"}, + "status": {"en": "Status", "fr": "Statut"}, }, ) diff --git a/modules/datamodels/datamodelKnowledge.py b/modules/datamodels/datamodelKnowledge.py new file mode 100644 index 00000000..4bc43500 --- /dev/null +++ b/modules/datamodels/datamodelKnowledge.py @@ -0,0 +1,130 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Knowledge Store data models: FileContentIndex, ContentChunk, WorkflowMemory. + +These models support the 3-tier RAG architecture: +- Shared Layer: mandateId-scoped, isShared=True +- Instance Layer: userId + featureInstanceId-scoped +- Workflow Layer: workflowId-scoped (WorkflowMemory) + +Vector fields use json_schema_extra={"db_type": "vector(1536)"} for pgvector. +""" + +from typing import Dict, Any, List, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class FileContentIndex(BaseModel): + """Structural index of a file's content objects. Created without AI. + Lives in the Instance Layer; optionally promoted to Shared Layer via isShared.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key (typically = fileId)") + userId: str = Field(description="Owner user ID") + featureInstanceId: str = Field(default="", description="Feature instance scope") + mandateId: str = Field(default="", description="Mandate scope") + isShared: bool = Field(default=False, description="Visible in Shared Layer for all mandate users") + fileName: str = Field(description="Original file name") + mimeType: str = Field(description="MIME type of the file") + containerPath: Optional[str] = Field(default=None, description="Path within a container (e.g. 'archive.zip/folder/report.pdf')") + totalObjects: int = Field(default=0, description="Total number of content objects extracted") + totalSize: int = Field(default=0, description="Total size of all content objects in bytes") + structure: Dict[str, Any] = Field(default_factory=dict, description="Structural overview (pages, sections, hierarchy)") + objectSummary: List[Dict[str, Any]] = Field(default_factory=list, description="Compact summary per content object") + extractedAt: float = Field(default_factory=getUtcTimestamp, description="Extraction timestamp") + status: str = Field(default="pending", description="Processing status: pending, extracted, embedding, indexed, failed") + + +registerModelLabels( + "FileContentIndex", + {"en": "File Content Index", "fr": "Index du contenu de fichier"}, + { + "id": {"en": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "isShared": {"en": "Shared", "fr": "Partagé"}, + "fileName": {"en": "File Name", "fr": "Nom de fichier"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "containerPath": {"en": "Container Path", "fr": "Chemin du conteneur"}, + "totalObjects": {"en": "Total Objects", "fr": "Nombre total d'objets"}, + "totalSize": {"en": "Total Size", "fr": "Taille totale"}, + "structure": {"en": "Structure", "fr": "Structure"}, + "objectSummary": {"en": "Object Summary", "fr": "Résumé des objets"}, + "extractedAt": {"en": "Extracted At", "fr": "Extrait le"}, + "status": {"en": "Status", "fr": "Statut"}, + }, +) + + +class ContentChunk(BaseModel): + """Persisted content chunk with embedding vector. Reusable across workflows. + Scalar content object (or chunk thereof) with pgvector embedding.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + contentObjectId: str = Field(description="Reference to the content object within FileContentIndex") + fileId: str = Field(description="FK to the source file") + userId: str = Field(description="Owner user ID") + featureInstanceId: str = Field(default="", description="Feature instance scope") + contentType: str = Field(description="Content type: text, image, videostream, audiostream, other") + data: str = Field(description="Content data (text, base64, URL)") + contextRef: Dict[str, Any] = Field(default_factory=dict, description="Context reference (page, position, label)") + summary: Optional[str] = Field(default=None, description="AI-generated summary (on demand)") + chunkMetadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata") + embedding: Optional[List[float]] = Field( + default=None, description="pgvector embedding (NOT NULL for text chunks)", + json_schema_extra={"db_type": "vector(1536)"} + ) + + +registerModelLabels( + "ContentChunk", + {"en": "Content Chunk", "fr": "Fragment de contenu"}, + { + "id": {"en": "ID", "fr": "ID"}, + "contentObjectId": {"en": "Content Object ID", "fr": "ID de l'objet de contenu"}, + "fileId": {"en": "File ID", "fr": "ID du fichier"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, + "contentType": {"en": "Content Type", "fr": "Type de contenu"}, + "data": {"en": "Data", "fr": "Données"}, + "contextRef": {"en": "Context Reference", "fr": "Référence contextuelle"}, + "summary": {"en": "Summary", "fr": "Résumé"}, + "chunkMetadata": {"en": "Metadata", "fr": "Métadonnées"}, + "embedding": {"en": "Embedding", "fr": "Vecteur d'embedding"}, + }, +) + + +class WorkflowMemory(BaseModel): + """Workflow-scoped key-value cache for entities and facts. + Extracted during agent rounds, persisted for cross-round and cross-workflow reuse.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + workflowId: str = Field(description="FK to the workflow") + userId: str = Field(description="Owner user ID") + featureInstanceId: str = Field(default="", description="Feature instance scope") + key: str = Field(description="Key identifier (e.g. 'entity:companyName')") + value: str = Field(description="Extracted value") + source: str = Field(default="extraction", description="Origin: extraction, tool, conversation, summary") + createdAt: float = Field(default_factory=getUtcTimestamp, description="Creation timestamp") + embedding: Optional[List[float]] = Field( + default=None, description="Optional embedding for semantic lookup", + json_schema_extra={"db_type": "vector(1536)"} + ) + + +registerModelLabels( + "WorkflowMemory", + {"en": "Workflow Memory", "fr": "Mémoire de workflow"}, + { + "id": {"en": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "featureInstanceId": {"en": "Feature Instance ID", "fr": "ID de l'instance"}, + "key": {"en": "Key", "fr": "Clé"}, + "value": {"en": "Value", "fr": "Valeur"}, + "source": {"en": "Source", "fr": "Source"}, + "createdAt": {"en": "Created At", "fr": "Créé le"}, + "embedding": {"en": "Embedding", "fr": "Vecteur d'embedding"}, + }, +) diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py index 86f4bb1d..2223a3e6 100644 --- a/modules/datamodels/datamodelVoice.py +++ b/modules/datamodels/datamodelVoice.py @@ -2,6 +2,7 @@ # All rights reserved. """Voice settings datamodel.""" +from typing import Dict, Any, Optional from pydantic import BaseModel, Field from modules.shared.attributeUtils import registerModelLabels from modules.shared.timeUtils import getUtcTimestamp @@ -16,6 +17,7 @@ class VoiceSettings(BaseModel): sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": True}) + ttsVoiceMap: Dict[str, Any] = Field(default_factory=dict, description="Per-language voice mapping, e.g. {'de-DE': {'voiceName': 'de-DE-Wavenet-A'}, 'en-US': {'voiceName': 'en-US-Wavenet-C'}}", json_schema_extra={"frontend_type": "json", "frontend_readonly": False, "frontend_required": False}) translationEnabled: bool = Field(default=True, description="Whether translation is enabled", json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}) targetLanguage: str = Field(default="en-US", description="Target language for translation", json_schema_extra={"frontend_type": "select", "frontend_readonly": False, "frontend_required": False}) creationDate: float = Field(default_factory=getUtcTimestamp, description="Date when the settings were created (UTC timestamp in seconds)", json_schema_extra={"frontend_type": "timestamp", "frontend_readonly": True, "frontend_required": False}) @@ -33,6 +35,7 @@ registerModelLabels( "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, + "ttsVoiceMap": {"en": "TTS Voice Map", "fr": "Carte des voix TTS"}, "translationEnabled": {"en": "Translation Enabled", "fr": "Traduction activée"}, "targetLanguage": {"en": "Target Language", "fr": "Langue cible"}, "creationDate": {"en": "Creation Date", "fr": "Date de création"}, diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py index aead6767..35a61512 100644 --- a/modules/features/automation/mainAutomation.py +++ b/modules/features/automation/mainAutomation.py @@ -180,7 +180,7 @@ def getAutomationServices( for spec in REQUIRED_SERVICES: key = spec["serviceKey"] try: - svc = getService(key, ctx, legacy_hub=None) + svc = getService(key, ctx) setattr(hub, key, svc) except Exception as e: logger.warning(f"Could not resolve service '{key}' for automation: {e}") diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 1b7d703e..eacb90f4 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -21,6 +21,7 @@ from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.interfaces import interfaceDbChat +from modules.interfaces.interfaceDbBilling import getInterface as _getBillingInterface # Configure logger logger = logging.getLogger(__name__) @@ -682,7 +683,9 @@ def get_automation_workflow_chat_data( workflow = chatInterface.getWorkflow(workflowId) if not workflow: raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") - return chatInterface.getUnifiedChatData(workflowId, afterTimestamp) + billingInterface = _getBillingInterface(context.user, context.mandateId) + workflowCost = billingInterface.getWorkflowCost(workflowId) + return chatInterface.getUnifiedChatData(workflowId, afterTimestamp, workflowCost=workflowCost) except HTTPException: raise except Exception as e: diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 741bf05f..559a9187 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -1291,17 +1291,6 @@ class ChatObjects: logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) raise ValueError(f"Error updating message {messageId}: {str(e)}") - def createStat(self, statData: Dict[str, Any]): - """Create stat record. Compatibility with ChatService; stats may not be persisted in chatbot schema.""" - from modules.datamodels.datamodelChat import ChatStat - stat = ChatStat(**statData) - try: - created = self.db.recordCreate(ChatStat, statData) - return ChatStat(**created) - except Exception as e: - logger.debug(f"createStat: not persisting (chatbot schema): {e}") - return stat - def deleteMessage(self, conversationId: str, messageId: str) -> bool: """Deletes a conversation message and related data if user has access.""" try: diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 84947bdd..33f8ae2f 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -179,7 +179,7 @@ def getChatbotServices( for spec in REQUIRED_SERVICES: key = spec["serviceKey"] try: - svc = getService(key, ctx, legacy_hub=None) + svc = getService(key, ctx) setattr(hub, key, svc) except Exception as e: logger.warning(f"Could not resolve service '{key}' for chatbot: {e}") @@ -197,7 +197,7 @@ def getChatStreamingHelper(): from modules.serviceCenter.context import ServiceCenterContext # Minimal context - streaming service only needs it for resolver ctx = ServiceCenterContext(user=__get_placeholder_user(), mandate_id=None, feature_instance_id=None) - streaming = getService("streaming", ctx, legacy_hub=None) + streaming = getService("streaming", ctx) return streaming.getChatStreamingHelper() if streaming else None @@ -219,7 +219,7 @@ def getEventManager(user, mandateId: Optional[str] = None, featureInstanceId: Op mandate_id=mandateId, feature_instance_id=featureInstanceId, ) - streaming = getService("streaming", ctx, legacy_hub=None) + streaming = getService("streaming", ctx) return streaming.getEventManager() @@ -306,12 +306,12 @@ def getChatbotServices( Uses interfaceFeatureChatbot (ChatObjects) for interfaceDbChat to avoid duplicate DB init - chatProcess reuses hub.interfaceDbChat. """ - from modules.services import PublicService + from modules.serviceHub import PublicService from modules.interfaces.interfaceDbApp import getInterface as getAppInterface from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface - from modules.services.serviceChat.mainServiceChat import ChatService - from modules.services.serviceAi.mainServiceAi import AiService - from modules.services.serviceStreaming.mainServiceStreaming import StreamingService + from modules.serviceCenter.services.serviceChat.mainServiceChat import ChatService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.core.serviceStreaming.mainServiceStreaming import StreamingService hub = _ChatbotServiceHub() hub.user = user @@ -344,7 +344,7 @@ def getChatbotServices( feature_instance_id=featureInstanceId, workflow=_workflow, ) - hub.billing = getService("billing", ctx, legacy_hub=None) + hub.billing = getService("billing", ctx) except Exception as e: logger.warning(f"Could not resolve billing service for chatbot: {e}") hub.billing = None diff --git a/modules/features/chatplayground/interfaceFeatureChatplayground.py b/modules/features/chatplayground/interfaceFeatureChatplayground.py index 5a2548ba..3cddbc85 100644 --- a/modules/features/chatplayground/interfaceFeatureChatplayground.py +++ b/modules/features/chatplayground/interfaceFeatureChatplayground.py @@ -135,11 +135,3 @@ class ChatPlaygroundObjects: def createLog(self, log) -> Dict[str, Any]: """Create a new log entry.""" return self._chatInterface.createLog(log) - - def getStats(self, workflowId: str) -> List[Dict[str, Any]]: - """Get stats for a workflow.""" - return self._chatInterface.getStats(workflowId) - - def createStat(self, stat) -> Dict[str, Any]: - """Create a new stat entry.""" - return self._chatInterface.createStat(stat) diff --git a/modules/features/chatplayground/mainChatplayground.py b/modules/features/chatplayground/mainChatplayground.py index d5275a5f..ba1cd094 100644 --- a/modules/features/chatplayground/mainChatplayground.py +++ b/modules/features/chatplayground/mainChatplayground.py @@ -158,7 +158,7 @@ def getChatplaygroundServices( for spec in REQUIRED_SERVICES: key = spec["serviceKey"] try: - svc = getService(key, ctx, legacy_hub=None) + svc = getService(key, ctx) setattr(hub, key, svc) except Exception as e: logger.warning(f"Could not resolve service '{key}' for chatplayground: {e}") diff --git a/modules/features/chatplayground/routeFeatureChatplayground.py b/modules/features/chatplayground/routeFeatureChatplayground.py index 8a87084a..1566c07b 100644 --- a/modules/features/chatplayground/routeFeatureChatplayground.py +++ b/modules/features/chatplayground/routeFeatureChatplayground.py @@ -15,6 +15,7 @@ from modules.auth import limiter, getRequestContext, RequestContext # Import interfaces from modules.interfaces import interfaceDbChat +from modules.interfaces.interfaceDbBilling import getInterface as _getBillingInterface # Import models from modules.datamodels.datamodelChat import ( @@ -220,9 +221,11 @@ def get_workflow_chat_data( detail=f"Workflow with ID {workflowId} not found" ) - # Get unified chat data - chatData = chatInterface.getUnifiedChatData(workflowId, afterTimestamp) + # Get workflow cost from billing transactions (single source of truth) + billingInterface = _getBillingInterface(context.user, context.mandateId) + workflowCost = billingInterface.getWorkflowCost(workflowId) + chatData = chatInterface.getUnifiedChatData(workflowId, afterTimestamp, workflowCost=workflowCost) return chatData except HTTPException: diff --git a/modules/features/codeeditor/routeFeatureCodeeditor.py b/modules/features/codeeditor/routeFeatureCodeeditor.py index 1feacd53..ef239b35 100644 --- a/modules/features/codeeditor/routeFeatureCodeeditor.py +++ b/modules/features/codeeditor/routeFeatureCodeeditor.py @@ -17,7 +17,7 @@ from modules.auth import limiter, getRequestContext, RequestContext from modules.interfaces import interfaceDbChat, interfaceDbManagement from modules.interfaces.interfaceAiObjects import AiObjects from modules.datamodels.datamodelChat import UserInputRequest -from modules.services.serviceStreaming import get_event_manager +from modules.serviceCenter.core.serviceStreaming import get_event_manager from modules.features.codeeditor import codeEditorProcessor, fileContextManager from modules.features.codeeditor.datamodelCodeeditor import FileEditProposal, EditStatusEnum diff --git a/modules/features/commcoach/serviceCommcoach.py b/modules/features/commcoach/serviceCommcoach.py index 38a1893e..ac9ad085 100644 --- a/modules/features/commcoach/serviceCommcoach.py +++ b/modules/features/commcoach/serviceCommcoach.py @@ -1011,7 +1011,7 @@ class CommcoachService: async def _callAi(self, systemPrompt: str, userPrompt: str): """Call the AI service with the given prompts.""" - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService serviceContext = type('Ctx', (), { 'user': self.currentUser, diff --git a/modules/features/neutralization/neutralizePlayground.py b/modules/features/neutralization/neutralizePlayground.py index c49e7cc5..c92d241b 100644 --- a/modules/features/neutralization/neutralizePlayground.py +++ b/modules/features/neutralization/neutralizePlayground.py @@ -7,7 +7,7 @@ from urllib.parse import urlparse, unquote from modules.datamodels.datamodelUam import User from .datamodelFeatureNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig -from modules.services import getInterface as getServices +from modules.serviceHub import getInterface as getServices logger = logging.getLogger(__name__) @@ -205,7 +205,7 @@ class NeutralizationPlayground: async def processSharepointFiles(self, sourcePath: str, targetPath: str) -> Dict[str, Any]: """Process files from SharePoint source path and store neutralized files in target path""" - from modules.services.serviceSharepoint.mainServiceSharepoint import SharepointService + from modules.serviceCenter.services.serviceSharepoint.mainServiceSharepoint import SharepointService processor = SharepointProcessor(self.currentUser, self.services) return await processor.processSharepointFiles(sourcePath, targetPath) diff --git a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py index b3d5040e..cf8f0f53 100644 --- a/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/features/neutralization/serviceNeutralization/mainServiceNeutralization.py @@ -262,8 +262,8 @@ class NeutralizationService: fileId: Optional[str] ) -> Dict[str, Any]: """Extract -> neutralize -> adapt -> generate for PDF/DOCX/XLSX/PPTX.""" - from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService - from modules.services.serviceExtraction.subPipeline import runExtraction + from modules.serviceCenter.services.serviceExtraction.mainServiceExtraction import ExtractionService + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy # Ensure registries exist @@ -405,10 +405,10 @@ class NeutralizationService: def _getRendererForMime(self, mimeType: str): """Get renderer instance and output mime for the given input MIME type.""" - from modules.services.serviceGeneration.renderers.rendererPdf import RendererPdf - from modules.services.serviceGeneration.renderers.rendererDocx import RendererDocx - from modules.services.serviceGeneration.renderers.rendererXlsx import RendererXlsx - from modules.services.serviceGeneration.renderers.rendererPptx import RendererPptx + from modules.serviceCenter.services.serviceGeneration.renderers.rendererPdf import RendererPdf + from modules.serviceCenter.services.serviceGeneration.renderers.rendererDocx import RendererDocx + from modules.serviceCenter.services.serviceGeneration.renderers.rendererXlsx import RendererXlsx + from modules.serviceCenter.services.serviceGeneration.renderers.rendererPptx import RendererPptx mime_map = { "application/pdf": (RendererPdf, "application/pdf"), diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 0425c94c..2ae2378b 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -284,7 +284,7 @@ from .datamodelFeatureRealEstate import ( Land, DokumentTyp, ) -from modules.services import getInterface as getServices +from modules.serviceHub import getInterface as getServices from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector diff --git a/modules/features/teamsbot/routeFeatureTeamsbot.py b/modules/features/teamsbot/routeFeatureTeamsbot.py index fef66eb9..c498a790 100644 --- a/modules/features/teamsbot/routeFeatureTeamsbot.py +++ b/modules/features/teamsbot/routeFeatureTeamsbot.py @@ -843,7 +843,7 @@ async def testVoice( ): """Test TTS voice with AI-generated sample text in the correct language.""" from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum mandateId = _validateInstanceAccess(instanceId, context) diff --git a/modules/features/teamsbot/service.py b/modules/features/teamsbot/service.py index 361577d9..773cc1c9 100644 --- a/modules/features/teamsbot/service.py +++ b/modules/features/teamsbot/service.py @@ -1062,7 +1062,7 @@ class TeamsbotService: # Call SPEECH_TEAMS try: - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService # Create minimal service context for AI billing serviceContext = _ServiceContext(self.currentUser, self.mandateId, self.instanceId) @@ -1684,7 +1684,7 @@ class TeamsbotService: """Summarize a long user-provided session context to its essential points. This reduces token usage in every subsequent AI call.""" try: - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum serviceContext = _ServiceContext(self.currentUser, self.mandateId, self.instanceId) @@ -1738,7 +1738,7 @@ class TeamsbotService: lines.append(f"[{speaker}]: {text}") textToSummarize = "\n".join(lines) - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum serviceContext = _ServiceContext(self.currentUser, self.mandateId, self.instanceId) @@ -1783,7 +1783,7 @@ class TeamsbotService: for t in transcripts ) - from modules.services.serviceAi.mainServiceAi import AiService + from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService serviceContext = _ServiceContext(self.currentUser, self.mandateId, self.instanceId) aiService = AiService(serviceCenter=serviceContext) diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index 82feb22f..9ad41b9d 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -188,7 +188,7 @@ def get_mime_type_options( """Get supported MIME types from the document extraction service. Returns: [{ value: "mime/type", label: "Description" }] """ - from modules.services.serviceExtraction.subRegistry import ExtractorRegistry + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry registry = ExtractorRegistry() formats = registry.getSupportedFormats() diff --git a/modules/features/workspace/__init__.py b/modules/features/workspace/__init__.py new file mode 100644 index 00000000..e4d7dac9 --- /dev/null +++ b/modules/features/workspace/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Unified AI Workspace feature -- merges Codeeditor, Chatbot, and Playground.""" diff --git a/modules/features/workspace/mainWorkspace.py b/modules/features/workspace/mainWorkspace.py new file mode 100644 index 00000000..0737bf13 --- /dev/null +++ b/modules/features/workspace/mainWorkspace.py @@ -0,0 +1,255 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Workspace Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. +Unified AI Workspace combining Codeeditor, Chatbot, and Playground capabilities. +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +FEATURE_CODE = "workspace" +FEATURE_LABEL = {"en": "AI Workspace", "de": "AI Workspace", "fr": "AI Workspace"} +FEATURE_ICON = "mdi-brain" + +UI_OBJECTS = [ + { + "objectKey": "ui.feature.workspace.dashboard", + "label": {"en": "Dashboard", "de": "Dashboard", "fr": "Tableau de bord"}, + "meta": {"area": "dashboard"} + }, + { + "objectKey": "ui.feature.workspace.settings", + "label": {"en": "Settings", "de": "Einstellungen", "fr": "Parametres"}, + "meta": {"area": "settings"} + }, +] + +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.workspace.start", + "label": {"en": "Start Agent", "de": "Agent starten", "fr": "Demarrer agent"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/start/stream", "method": "POST"} + }, + { + "objectKey": "resource.feature.workspace.stop", + "label": {"en": "Stop Agent", "de": "Agent stoppen", "fr": "Arreter agent"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/{workflowId}/stop", "method": "POST"} + }, + { + "objectKey": "resource.feature.workspace.files", + "label": {"en": "Manage Files", "de": "Dateien verwalten", "fr": "Gerer fichiers"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/files", "method": "GET"} + }, + { + "objectKey": "resource.feature.workspace.folders", + "label": {"en": "Manage Folders", "de": "Ordner verwalten", "fr": "Gerer dossiers"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/folders", "method": "GET"} + }, + { + "objectKey": "resource.feature.workspace.datasources", + "label": {"en": "Data Sources", "de": "Datenquellen", "fr": "Sources de donnees"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/datasources", "method": "GET"} + }, + { + "objectKey": "resource.feature.workspace.voice", + "label": {"en": "Voice Input/Output", "de": "Spracheingabe/-ausgabe", "fr": "Entree/sortie vocale"}, + "meta": {"endpoint": "/api/workspace/{instanceId}/voice/*", "method": "POST"} + }, +] + +TEMPLATE_ROLES = [ + { + "roleLabel": "workspace-viewer", + "description": { + "en": "Workspace Viewer - View workspace (read-only)", + "de": "Workspace Betrachter - Workspace ansehen (nur lesen)", + "fr": "Visualiseur Workspace - Consulter le workspace (lecture seule)" + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.workspace.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.workspace.settings", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ] + }, + { + "roleLabel": "workspace-user", + "description": { + "en": "Workspace User - Use AI workspace and tools", + "de": "Workspace Benutzer - AI Workspace und Tools nutzen", + "fr": "Utilisateur Workspace - Utiliser l'espace de travail AI et les outils" + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.workspace.dashboard", "view": True}, + {"context": "UI", "item": "ui.feature.workspace.settings", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.start", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.stop", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.files", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.folders", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.datasources", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.workspace.voice", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, + ] + }, + { + "roleLabel": "workspace-admin", + "description": { + "en": "Workspace Admin - Full access to AI workspace", + "de": "Workspace Admin - Vollzugriff auf AI Workspace", + "fr": "Administrateur Workspace - Acces complet au workspace AI" + }, + "accessRules": [ + {"context": "UI", "item": None, "view": True}, + {"context": "RESOURCE", "item": None, "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + ] + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON, + "autoCreateInstance": True, + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """Register this feature's RBAC objects in the catalog.""" + try: + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + _syncTemplateRolesToDb() + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False + + +def _syncTemplateRolesToDb() -> int: + """Sync template roles and their AccessRules to the database.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + rootInterface = getRootInterface() + + existingRoles = rootInterface.getRolesByFeatureCode(FEATURE_CODE) + templateRoles = [r for r in existingRoles if r.mandateId is None] + existingRoleLabels = {r.roleLabel: str(r.id) for r in templateRoles} + + createdCount = 0 + for roleTemplate in TEMPLATE_ROLES: + roleLabel = roleTemplate["roleLabel"] + + if roleLabel in existingRoleLabels: + roleId = existingRoleLabels[roleLabel] + _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) + else: + newRole = Role( + roleLabel=roleLabel, + description=roleTemplate.get("description", {}), + featureCode=FEATURE_CODE, + mandateId=None, + featureInstanceId=None, + isSystemRole=False + ) + createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) + roleId = createdRole.get("id") + _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) + logger.info(f"Created template role '{roleLabel}' with ID {roleId}") + createdCount += 1 + + if createdCount > 0: + logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") + + return createdCount + + except Exception as e: + logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") + return 0 + + +def _ensureAccessRulesForRole(rootInterface, roleId: str, ruleTemplates: List[Dict[str, Any]]) -> int: + """Ensure AccessRules exist for a role based on templates.""" + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext + + existingRules = rootInterface.getAccessRulesByRole(roleId) + existingSignatures = set() + for rule in existingRules: + sig = (rule.context.value if rule.context else None, rule.item) + existingSignatures.add(sig) + + createdCount = 0 + for template in ruleTemplates: + context = template.get("context", "UI") + item = template.get("item") + sig = (context, item) + + if sig in existingSignatures: + continue + + if context == "UI": + contextEnum = AccessRuleContext.UI + elif context == "DATA": + contextEnum = AccessRuleContext.DATA + elif context == "RESOURCE": + contextEnum = AccessRuleContext.RESOURCE + else: + contextEnum = context + + newRule = AccessRule( + roleId=roleId, + context=contextEnum, + item=item, + view=template.get("view", False), + read=template.get("read"), + create=template.get("create"), + update=template.get("update"), + delete=template.get("delete"), + ) + rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) + createdCount += 1 + + if createdCount > 0: + logger.debug(f"Created {createdCount} AccessRules for role {roleId}") + + return createdCount diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py new file mode 100644 index 00000000..23c0db73 --- /dev/null +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -0,0 +1,1037 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Unified AI Workspace routes. + +SSE-based endpoints that combine the capabilities of Codeeditor, Chatbot, +and Playground into a single agent-driven workspace. +""" + +import logging +import json +import asyncio +from typing import Optional, List + +from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Request, UploadFile, File +from fastapi.responses import StreamingResponse, JSONResponse +from pydantic import BaseModel, Field + +from modules.auth import limiter, getRequestContext, RequestContext +from modules.interfaces import interfaceDbChat, interfaceDbManagement +from modules.interfaces.interfaceAiObjects import AiObjects +from modules.serviceCenter.core.serviceStreaming import get_event_manager +from modules.serviceCenter.services.serviceAgent.datamodelAgent import AgentEventTypeEnum + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/workspace", + tags=["Unified Workspace"], + responses={404: {"description": "Not found"}}, +) + +_aiObjects: Optional[AiObjects] = None + + +class WorkspaceInputRequest(BaseModel): + """Prompt input for the unified workspace.""" + prompt: str = Field(description="User prompt text") + fileIds: List[str] = Field(default_factory=list, description="Referenced file IDs") + uploadedFiles: List[str] = Field(default_factory=list, description="Newly uploaded file IDs") + dataSourceIds: List[str] = Field(default_factory=list, description="Active DataSource IDs") + voiceMode: bool = Field(default=False, description="Enable voice response") + workflowId: Optional[str] = Field(default=None, description="Continue existing workflow") + userLanguage: str = Field(default="en", description="User language code") + + +async def _getAiObjects() -> AiObjects: + global _aiObjects + if _aiObjects is None: + _aiObjects = await AiObjects.create() + return _aiObjects + + +def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: + from modules.interfaces.interfaceDbApp import getRootInterface + rootInterface = getRootInterface() + instance = rootInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException(status_code=404, detail=f"Feature instance {instanceId} not found") + featureAccess = rootInterface.getFeatureAccess(str(context.user.id), instanceId) + if not featureAccess or not featureAccess.enabled: + raise HTTPException(status_code=403, detail="Access denied to this feature instance") + return str(instance.mandateId) if instance.mandateId else None + + +def _getChatInterface(context: RequestContext, featureInstanceId: str = None): + return interfaceDbChat.getInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=featureInstanceId, + ) + + +def _buildResolverDbInterface(chatService): + """Build a DB adapter that ConnectorResolver can use to load UserConnections. + + ConnectorResolver calls db.getUserConnection(connectionId). + interfaceDbApp provides getUserConnectionById(connectionId). + This adapter bridges the method name difference. + """ + class _ResolverDbAdapter: + def __init__(self, appInterface): + self._app = appInterface + def getUserConnection(self, connectionId: str): + if hasattr(self._app, "getUserConnectionById"): + return self._app.getUserConnectionById(connectionId) + return None + appIf = getattr(chatService, "interfaceDbApp", None) + if appIf: + return _ResolverDbAdapter(appIf) + return getattr(chatService, "interfaceDbComponent", None) + + +def _getDbManagement(context: RequestContext, featureInstanceId: str = None): + return interfaceDbManagement.getInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=featureInstanceId, + ) + + +_SOURCE_TYPE_TO_SERVICE = { + "sharepointFolder": "sharepoint", + "onedriveFolder": "onedrive", + "outlookFolder": "outlook", + "googleDriveFolder": "drive", + "gmailFolder": "gmail", + "ftpFolder": "files", +} + + +def _buildDataSourceContext(chatService, dataSourceIds: List[str]) -> str: + """Build a description of active data sources for the agent prompt.""" + parts = [ + "The user has attached the following external data sources to this prompt.", + "IMPORTANT RULES for attached data sources:", + "- Use ONLY browseDataSource, searchDataSource, and downloadFromDataSource to access these sources.", + "- Use the dataSourceId (UUID) exactly as shown below.", + "- Do NOT use listFiles, externalBrowse, or externalSearch for attached data sources -- those tools are for other purposes.", + "- browseDataSource returns BOTH files and folders at the given path.", + "- When downloading files, ALWAYS provide the human-readable fileName (with extension) from the browse results.", + "", + ] + found = False + for dsId in dataSourceIds: + try: + ds = chatService.getDataSource(dsId) if hasattr(chatService, "getDataSource") else None + if ds: + found = True + label = ds.get("label", "") + sourceType = ds.get("sourceType", "") + connectionId = ds.get("connectionId", "") + path = ds.get("path", "/") + service = _SOURCE_TYPE_TO_SERVICE.get(sourceType, sourceType) + logger.info(f"DataSource context: id={dsId}, label={label}, sourceType={sourceType}, service={service}, connectionId={connectionId}, path={path[:80]}") + parts.append( + f"- dataSourceId: {dsId}\n" + f" label: \"{label}\"\n" + f" type: {sourceType} (service: {service})\n" + f" connectionId: {connectionId}\n" + f" path: {path}" + ) + else: + logger.warning(f"DataSource {dsId} not found in DB") + except Exception as e: + logger.warning(f"Error loading DataSource {dsId}: {e}") + return "\n".join(parts) if found else "" + + +async def _deriveWorkflowName(prompt: str, aiService) -> str: + """Use AI to generate a concise workflow title from the user prompt.""" + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum + try: + cleanPrompt = prompt.split("\n[Active Data Sources]")[0].strip()[:300] + req = AiCallRequest( + prompt=( + "Generate a short title (3-6 words) for a chat conversation that starts with this user message. " + "Reply with ONLY the title, nothing else. Same language as the user message.\n\n" + f"User message: {cleanPrompt}" + ), + options=AiCallOptions( + operationType=OperationTypeEnum.DATA_EXTRACT, + priority=PriorityEnum.SPEED, + compressPrompt=False, + temperature=0.3, + ), + ) + resp = await aiService.callAi(req) + title = (resp.content or "").strip().strip('"\'').strip() + if title and len(title) <= 60: + return title + except Exception as e: + logger.warning(f"AI naming failed, using fallback: {e}") + text = prompt.split("\n[Active Data Sources]")[0].split("\n")[0].strip()[:50] + return text or "Chat" + + +# --------------------------------------------------------------------------- +# SSE Stream endpoint +# --------------------------------------------------------------------------- + +@router.post("/{instanceId}/start/stream") +@limiter.limit("60/minute") +async def streamWorkspaceStart( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + userInput: WorkspaceInputRequest = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Start or continue a Workspace session with SSE streaming via serviceAgent.""" + mandateId = _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + aiObjects = await _getAiObjects() + eventManager = get_event_manager() + + if userInput.workflowId: + workflow = chatInterface.getWorkflow(userInput.workflowId) + if not workflow: + raise HTTPException(status_code=404, detail=f"Workflow {userInput.workflowId} not found") + else: + workflow = chatInterface.createWorkflow({ + "featureInstanceId": instanceId, + "status": "active", + "name": "", + "workflowMode": "Dynamic", + }) + + workflowId = workflow.get("id") if isinstance(workflow, dict) else getattr(workflow, "id", str(workflow)) + queueId = f"workspace-{workflowId}" + eventManager.create_queue(queueId) + + chatInterface.createMessage({ + "workflowId": workflowId, + "role": "user", + "message": userInput.prompt, + }) + + agentTask = asyncio.ensure_future( + _runWorkspaceAgent( + workflowId=workflowId, + queueId=queueId, + prompt=userInput.prompt, + fileIds=userInput.fileIds, + dataSourceIds=userInput.dataSourceIds, + voiceMode=userInput.voiceMode, + instanceId=instanceId, + user=context.user, + mandateId=mandateId or "", + aiObjects=aiObjects, + chatInterface=chatInterface, + eventManager=eventManager, + userLanguage=userInput.userLanguage, + ) + ) + eventManager.register_agent_task(queueId, agentTask) + + async def _sseGenerator(): + queue = eventManager.get_queue(queueId) + if not queue: + return + while True: + try: + event = await asyncio.wait_for(queue.get(), timeout=120) + except asyncio.TimeoutError: + yield "data: {\"type\": \"keepalive\"}\n\n" + continue + + if event is None: + break + + ssePayload = event.get("data", event) if isinstance(event, dict) else event + yield f"data: {json.dumps(ssePayload, default=str)}\n\n" + + eventType = ssePayload.get("type", "") if isinstance(ssePayload, dict) else "" + if eventType in ("complete", "error", "stopped"): + break + + await eventManager.cleanup(queueId, delay=30) + + return StreamingResponse( + _sseGenerator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + +async def _runWorkspaceAgent( + workflowId: str, + queueId: str, + prompt: str, + fileIds: List[str], + dataSourceIds: List[str], + voiceMode: bool, + instanceId: str, + user, + mandateId: str, + aiObjects, + chatInterface, + eventManager, + userLanguage: str = "en", +): + """Run the serviceAgent loop and forward events to the SSE queue.""" + try: + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=user, + mandate_id=mandateId, + feature_instance_id=instanceId, + workflow_id=workflowId, + ) + agentService = getService("agent", ctx) + chatService = getService("chat", ctx) + aiService = getService("ai", ctx) + + wfRecord = chatInterface.getWorkflow(workflowId) if workflowId else None + wfName = "" + if wfRecord: + wfName = wfRecord.get("name", "") if isinstance(wfRecord, dict) else getattr(wfRecord, "name", "") + if not wfName.strip() or wfName.startswith("Neuer Chat"): + async def _nameInBackground(): + try: + autoName = await _deriveWorkflowName(prompt, aiService) + chatInterface.updateWorkflow(workflowId, {"name": autoName}) + await eventManager.emit_event(queueId, "workflowUpdated", { + "type": "workflowUpdated", + "workflowId": workflowId, + "name": autoName, + }) + except Exception as nameErr: + logger.warning(f"AI workflow naming failed: {nameErr}") + asyncio.ensure_future(_nameInBackground()) + + enrichedPrompt = prompt + if dataSourceIds: + dsInfo = _buildDataSourceContext(chatService, dataSourceIds) + if dsInfo: + enrichedPrompt = f"{prompt}\n\n[Active Data Sources]\n{dsInfo}" + + async for event in agentService.runAgent( + prompt=enrichedPrompt, + fileIds=fileIds, + workflowId=workflowId, + userLanguage=userLanguage, + ): + if eventManager.is_cancelled(queueId): + logger.info(f"Agent cancelled by user for workflow {workflowId}") + break + + sseEvent = { + "type": event.type.value if hasattr(event.type, "value") else event.type, + "workflowId": workflowId, + } + if event.content: + sseEvent["content"] = event.content + if event.type == AgentEventTypeEnum.MESSAGE: + sseEvent["item"] = { + "id": f"msg-{workflowId}-{id(event)}", + "role": "assistant", + "content": event.content, + "workflowId": workflowId, + } + if event.data: + sseEvent["item"] = event.data + + await eventManager.emit_event(queueId, sseEvent["type"], sseEvent) + + if event.type in (AgentEventTypeEnum.FINAL, AgentEventTypeEnum.ERROR): + if event.content: + try: + chatInterface.createMessage({ + "workflowId": workflowId, + "role": "assistant", + "message": event.content, + }) + except Exception as msgErr: + logger.error(f"Failed to persist assistant message: {msgErr}") + + logger.info(f"Agent loop completed for workflow {workflowId}, sending 'complete' event") + await eventManager.emit_event(queueId, "complete", { + "type": "complete", + "workflowId": workflowId, + }) + + except asyncio.CancelledError: + logger.info(f"Agent task cancelled for workflow {workflowId}") + await eventManager.emit_event(queueId, "stopped", { + "type": "stopped", + "workflowId": workflowId, + }) + + except Exception as e: + logger.error(f"Workspace agent error: {e}", exc_info=True) + await eventManager.emit_event(queueId, "error", { + "type": "error", + "content": str(e), + "workflowId": workflowId, + }) + finally: + eventManager._unregister_agent_task(queueId) + + +# --------------------------------------------------------------------------- +# Stop endpoint +# --------------------------------------------------------------------------- + +@router.post("/{instanceId}/{workflowId}/stop") +@limiter.limit("30/minute") +async def stopWorkspace( + request: Request, + instanceId: str = Path(...), + workflowId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + _validateInstanceAccess(instanceId, context) + queueId = f"workspace-{workflowId}" + eventManager = get_event_manager() + cancelled = await eventManager.cancel_agent(queueId) + await eventManager.emit_event(queueId, "stopped", { + "type": "stopped", + "workflowId": workflowId, + }) + logger.info(f"Stop requested for workflow {workflowId}, agent task cancelled: {cancelled}") + return JSONResponse({"status": "stopped", "workflowId": workflowId}) + + +# --------------------------------------------------------------------------- +# Workflow / Conversation endpoints +# --------------------------------------------------------------------------- + +@router.get("/{instanceId}/workflows") +@limiter.limit("60/minute") +async def listWorkspaceWorkflows( + request: Request, + instanceId: str = Path(...), + includeArchived: bool = Query(default=False, description="Include archived workflows"), + context: RequestContext = Depends(getRequestContext), +): + """List workspace workflows/conversations for this instance.""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + workflows = chatInterface.getWorkflows() or [] + items = [] + for wf in workflows: + if isinstance(wf, dict): + item = wf + else: + item = { + "id": getattr(wf, "id", None), + "name": getattr(wf, "name", ""), + "status": getattr(wf, "status", ""), + "startedAt": getattr(wf, "startedAt", None), + "lastActivity": getattr(wf, "lastActivity", None), + } + if not includeArchived and item.get("status") == "archived": + continue + items.append(item) + return JSONResponse({"workflows": items}) + + +class UpdateWorkflowRequest(BaseModel): + """Request body for updating a workflow (PATCH).""" + name: Optional[str] = Field(default=None, description="New workflow name") + status: Optional[str] = Field(default=None, description="New status (active, archived)") + + +@router.patch("/{instanceId}/workflows/{workflowId}") +@limiter.limit("60/minute") +async def patchWorkspaceWorkflow( + request: Request, + instanceId: str = Path(..., description="Feature instance ID"), + workflowId: str = Path(..., description="Workflow ID to update"), + body: UpdateWorkflowRequest = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Update a workspace workflow (e.g. rename).""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + workflow = chatInterface.getWorkflow(workflowId) + if not workflow: + raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") + updateData = {} + if body.name is not None: + updateData["name"] = body.name + if body.status is not None: + updateData["status"] = body.status + if not updateData: + updated = workflow + else: + updated = chatInterface.updateWorkflow(workflowId, updateData) + if isinstance(updated, dict): + return JSONResponse(updated) + return JSONResponse({ + "id": getattr(updated, "id", None), + "name": getattr(updated, "name", ""), + "status": getattr(updated, "status", ""), + "startedAt": getattr(updated, "startedAt", None), + "lastActivity": getattr(updated, "lastActivity", None), + }) + + +@router.delete("/{instanceId}/workflows/{workflowId}") +@limiter.limit("30/minute") +async def deleteWorkspaceWorkflow( + request: Request, + instanceId: str = Path(...), + workflowId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Delete a workspace workflow and its messages.""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + workflow = chatInterface.getWorkflow(workflowId) + if not workflow: + raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found") + chatInterface.deleteWorkflow(workflowId) + return JSONResponse({"status": "deleted", "workflowId": workflowId}) + + +@router.post("/{instanceId}/workflows") +@limiter.limit("30/minute") +async def createWorkspaceWorkflow( + request: Request, + instanceId: str = Path(...), + body: dict = Body(default={}), + context: RequestContext = Depends(getRequestContext), +): + """Create a new empty workspace workflow.""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + name = body.get("name", "Neuer Chat") + workflow = chatInterface.createWorkflow({ + "featureInstanceId": instanceId, + "status": "active", + "name": name, + "workflowMode": "Dynamic", + }) + wfId = workflow.get("id") if isinstance(workflow, dict) else getattr(workflow, "id", None) + wfName = workflow.get("name") if isinstance(workflow, dict) else getattr(workflow, "name", name) + return JSONResponse({"id": wfId, "name": wfName, "status": "active"}) + + +@router.get("/{instanceId}/workflows/{workflowId}/messages") +@limiter.limit("60/minute") +async def getWorkspaceMessages( + request: Request, + instanceId: str = Path(...), + workflowId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Get all messages for a workspace workflow/conversation.""" + _validateInstanceAccess(instanceId, context) + chatInterface = _getChatInterface(context, featureInstanceId=instanceId) + messages = chatInterface.getMessages(workflowId) or [] + items = [] + for msg in messages: + if isinstance(msg, dict): + items.append(msg) + else: + items.append({ + "id": getattr(msg, "id", None), + "role": getattr(msg, "role", ""), + "content": getattr(msg, "message", "") or getattr(msg, "content", ""), + "createdAt": getattr(msg, "publishedAt", None) or getattr(msg, "createdAt", None), + }) + return JSONResponse({"messages": items}) + + +# --------------------------------------------------------------------------- +# File and folder list endpoints +# --------------------------------------------------------------------------- + +@router.get("/{instanceId}/files") +@limiter.limit("60/minute") +async def listWorkspaceFiles( + request: Request, + instanceId: str = Path(...), + folderId: Optional[str] = Query(None), + tags: Optional[str] = Query(None), + search: Optional[str] = Query(None), + context: RequestContext = Depends(getRequestContext), +): + _validateInstanceAccess(instanceId, context) + dbMgmt = _getDbManagement(context, featureInstanceId=instanceId) + files = dbMgmt.getAllFiles() + + from modules.interfaces.interfaceDbApp import getRootInterface + rootInterface = getRootInterface() + instanceLabelCache: dict = {} + + result = [] + for f in (files or []): + item = f if isinstance(f, dict) else f.model_dump() + fiId = item.get("featureInstanceId") or "" + if fiId and fiId not in instanceLabelCache: + fi = rootInterface.getFeatureInstance(fiId) + instanceLabelCache[fiId] = fi.label if fi else fiId + item["featureInstanceId"] = fiId + item["featureInstanceLabel"] = instanceLabelCache.get(fiId, "(Global)") + result.append(item) + + return JSONResponse({"files": result}) + + +@router.get("/{instanceId}/files/{fileId}/content") +@limiter.limit("60/minute") +async def getFileContent( + request: Request, + instanceId: str = Path(...), + fileId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Return the raw content of a file for preview.""" + from fastapi.responses import Response + _validateInstanceAccess(instanceId, context) + dbMgmt = _getDbManagement(context, featureInstanceId=instanceId) + fileRecord = dbMgmt.getFile(fileId) + if not fileRecord: + raise HTTPException(status_code=404, detail=f"File {fileId} not found") + fileData = fileRecord if isinstance(fileRecord, dict) else fileRecord.model_dump() + filePath = fileData.get("filePath") + if not filePath: + raise HTTPException(status_code=404, detail="File has no stored path") + import os + if not os.path.isfile(filePath): + raise HTTPException(status_code=404, detail="File not found on disk") + mimeType = fileData.get("mimeType", "application/octet-stream") + with open(filePath, "rb") as fh: + content = fh.read() + return Response(content=content, media_type=mimeType) + + +@router.get("/{instanceId}/folders") +@limiter.limit("60/minute") +async def listWorkspaceFolders( + request: Request, + instanceId: str = Path(...), + parentId: Optional[str] = Query(None), + context: RequestContext = Depends(getRequestContext), +): + _validateInstanceAccess(instanceId, context) + try: + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getService("chat", ctx) + folders = chatService.listFolders(parentId=parentId) + return JSONResponse({"folders": folders or []}) + except Exception: + return JSONResponse({"folders": []}) + + +@router.get("/{instanceId}/datasources") +@limiter.limit("60/minute") +async def listWorkspaceDataSources( + request: Request, + instanceId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + _validateInstanceAccess(instanceId, context) + try: + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getService("chat", ctx) + dataSources = chatService.listDataSources(featureInstanceId=instanceId) + return JSONResponse({"dataSources": dataSources or []}) + except Exception: + return JSONResponse({"dataSources": []}) + + +@router.get("/{instanceId}/connections") +@limiter.limit("60/minute") +async def listWorkspaceConnections( + request: Request, + instanceId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Return the user's active connections (UserConnections).""" + _validateInstanceAccess(instanceId, context) + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getService("chat", ctx) + connections = chatService.getUserConnections() + items = [] + for c in connections or []: + conn = c if isinstance(c, dict) else (c.model_dump() if hasattr(c, "model_dump") else {}) + authority = conn.get("authority") + if hasattr(authority, "value"): + authority = authority.value + status = conn.get("status") + if hasattr(status, "value"): + status = status.value + items.append({ + "id": conn.get("id"), + "authority": authority, + "externalUsername": conn.get("externalUsername"), + "externalEmail": conn.get("externalEmail"), + "status": status, + }) + return JSONResponse({"connections": items}) + + +class CreateDataSourceRequest(BaseModel): + """Request body for creating a DataSource.""" + connectionId: str = Field(description="Connection ID") + sourceType: str = Field(description="Source type") + path: str = Field(description="Path") + label: str = Field(description="Label") + + +@router.post("/{instanceId}/datasources") +@limiter.limit("60/minute") +async def createWorkspaceDataSource( + request: Request, + instanceId: str = Path(...), + body: CreateDataSourceRequest = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Create a new DataSource for this workspace instance.""" + _validateInstanceAccess(instanceId, context) + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getService("chat", ctx) + dataSource = chatService.createDataSource( + connectionId=body.connectionId, + sourceType=body.sourceType, + path=body.path, + label=body.label, + featureInstanceId=instanceId, + ) + return JSONResponse(dataSource if isinstance(dataSource, dict) else dataSource.model_dump()) + + +@router.delete("/{instanceId}/datasources/{dataSourceId}") +@limiter.limit("60/minute") +async def deleteWorkspaceDataSource( + request: Request, + instanceId: str = Path(...), + dataSourceId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Delete a DataSource.""" + _validateInstanceAccess(instanceId, context) + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getService("chat", ctx) + chatService.deleteDataSource(dataSourceId) + return JSONResponse({"success": True}) + + +@router.get("/{instanceId}/connections/{connectionId}/services") +@limiter.limit("30/minute") +async def listConnectionServices( + request: Request, + instanceId: str = Path(...), + connectionId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Return the available services for a specific UserConnection.""" + _validateInstanceAccess(instanceId, context) + try: + from modules.connectors.connectorResolver import ConnectorResolver + from modules.serviceCenter import getService as getSvc + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getSvc("chat", ctx) + securityService = getSvc("security", ctx) + dbInterface = _buildResolverDbInterface(chatService) + resolver = ConnectorResolver(securityService, dbInterface) + provider = await resolver.resolve(connectionId) + services = provider.getAvailableServices() + _serviceLabels = { + "sharepoint": "SharePoint", + "outlook": "Outlook", + "teams": "Teams", + "onedrive": "OneDrive", + "drive": "Google Drive", + "gmail": "Gmail", + "files": "Files (FTP)", + } + _serviceIcons = { + "sharepoint": "sharepoint", + "outlook": "mail", + "teams": "chat", + "onedrive": "cloud", + "drive": "cloud", + "gmail": "mail", + "files": "folder", + } + items = [ + { + "service": s, + "label": _serviceLabels.get(s, s), + "icon": _serviceIcons.get(s, "folder"), + } + for s in services + ] + return JSONResponse({"services": items}) + except Exception as e: + logger.error(f"Error listing services for connection {connectionId}: {e}") + return JSONResponse({"services": [], "error": str(e)}, status_code=400) + + +@router.get("/{instanceId}/connections/{connectionId}/browse") +@limiter.limit("60/minute") +async def browseConnectionService( + request: Request, + instanceId: str = Path(...), + connectionId: str = Path(...), + service: str = Query(..., description="Service name (e.g. sharepoint, onedrive, outlook)"), + path: str = Query("/", description="Path within the service to browse"), + context: RequestContext = Depends(getRequestContext), +): + """Browse folders/items within a connection's service at a given path.""" + _validateInstanceAccess(instanceId, context) + try: + from modules.connectors.connectorResolver import ConnectorResolver + from modules.serviceCenter import getService as getSvc + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=context.user, + mandate_id=str(context.mandateId) if context.mandateId else None, + feature_instance_id=instanceId, + ) + chatService = getSvc("chat", ctx) + securityService = getSvc("security", ctx) + dbInterface = _buildResolverDbInterface(chatService) + resolver = ConnectorResolver(securityService, dbInterface) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.browse(path, filter=None) + items = [] + for entry in (entries or []): + items.append({ + "name": entry.name, + "path": entry.path, + "isFolder": entry.isFolder, + "size": entry.size, + "mimeType": entry.mimeType, + "metadata": entry.metadata if hasattr(entry, "metadata") else {}, + }) + return JSONResponse({"items": items, "path": path, "service": service}) + except Exception as e: + logger.error(f"Error browsing {service} for connection {connectionId} at '{path}': {e}") + return JSONResponse({"items": [], "error": str(e)}, status_code=400) + + +# --------------------------------------------------------------------------- +# Voice endpoints +# --------------------------------------------------------------------------- + +@router.post("/{instanceId}/voice/transcribe") +@limiter.limit("30/minute") +async def transcribeVoice( + request: Request, + instanceId: str = Path(...), + audio: UploadFile = File(...), + context: RequestContext = Depends(getRequestContext), +): + """Transcribe audio to text using speech-to-text.""" + _validateInstanceAccess(instanceId, context) + audioBytes = await audio.read() + try: + import aiohttp + formData = aiohttp.FormData() + formData.add_field("audio", audioBytes, filename=audio.filename or "audio.webm") + async with aiohttp.ClientSession() as session: + async with session.post( + f"{request.base_url}api/voice-google/speech-to-text", + data=formData, + ) as resp: + if resp.status == 200: + result = await resp.json() + return JSONResponse({"text": result.get("text", "")}) + return JSONResponse({"text": "", "error": f"STT failed: {resp.status}"}) + except Exception as e: + logger.error(f"Voice transcription error: {e}") + return JSONResponse({"text": "", "error": str(e)}) + + +@router.post("/{instanceId}/voice/synthesize") +@limiter.limit("30/minute") +async def synthesizeVoice( + request: Request, + instanceId: str = Path(...), + body: dict = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Synthesize text to speech audio.""" + _validateInstanceAccess(instanceId, context) + text = body.get("text", "") + if not text: + raise HTTPException(status_code=400, detail="text is required") + return JSONResponse({"audio": None, "note": "TTS via browser Speech Synthesis API recommended"}) + + +# ========================================================================= +# Voice Settings Endpoints +# ========================================================================= + +@router.get("/{instanceId}/settings/voice") +@limiter.limit("30/minute") +async def getVoiceSettings( + request: Request, + instanceId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Load voice settings for the current user and instance.""" + _validateInstanceAccess(instanceId, context) + dbMgmt = _getDbManagement(context, instanceId) + userId = str(context.user.id) + try: + vs = dbMgmt.getVoiceSettings(userId) + if not vs: + logger.info(f"GET voice settings: not found for user={userId}, creating defaults") + vs = dbMgmt.getOrCreateVoiceSettings(userId) + result = vs.model_dump() if vs else {} + mapKeys = list(result.get("ttsVoiceMap", {}).keys()) if result else [] + logger.info(f"GET voice settings for user={userId}: ttsVoiceMap languages={mapKeys}") + return JSONResponse(result) + except Exception as e: + logger.error(f"Failed to load voice settings for user={userId}: {e}", exc_info=True) + return JSONResponse({"ttsVoiceMap": {}}, status_code=200) + + +@router.put("/{instanceId}/settings/voice") +@limiter.limit("30/minute") +async def updateVoiceSettings( + request: Request, + instanceId: str = Path(...), + body: dict = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Update voice settings for the current user and instance.""" + _validateInstanceAccess(instanceId, context) + dbMgmt = _getDbManagement(context, instanceId) + userId = str(context.user.id) + + try: + logger.info(f"PUT voice settings for user={userId}, instance={instanceId}, body keys={list(body.keys())}") + vs = dbMgmt.getVoiceSettings(userId) + if not vs: + logger.info(f"No existing voice settings, creating new for user={userId}") + createData = { + "userId": userId, + "mandateId": str(context.mandateId) if context.mandateId else "", + "featureInstanceId": instanceId, + } + createData.update(body) + created = dbMgmt.createVoiceSettings(createData) + logger.info(f"Created voice settings for user={userId}, ttsVoiceMap keys={list((created or {}).get('ttsVoiceMap', {}).keys())}") + return JSONResponse(created) + + updateData = {k: v for k, v in body.items() if k not in ("id", "userId", "mandateId", "featureInstanceId", "creationDate")} + logger.info(f"Updating voice settings for user={userId}, update keys={list(updateData.keys())}") + updated = dbMgmt.updateVoiceSettings(userId, updateData) + logger.info(f"Updated voice settings for user={userId}, ttsVoiceMap keys={list((updated or {}).get('ttsVoiceMap', {}).keys())}") + return JSONResponse(updated) + except Exception as e: + logger.error(f"Failed to update voice settings for user={userId}: {e}", exc_info=True) + return JSONResponse({"error": str(e)}, status_code=500) + + +@router.get("/{instanceId}/voice/languages") +@limiter.limit("30/minute") +async def getVoiceLanguages( + request: Request, + instanceId: str = Path(...), + context: RequestContext = Depends(getRequestContext), +): + """Return available TTS languages.""" + mandateId = _validateInstanceAccess(instanceId, context) + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + voiceInterface = getVoiceInterface(context.user, mandateId) + languagesResult = await voiceInterface.getAvailableLanguages() + languageList = languagesResult.get("languages", []) if isinstance(languagesResult, dict) else languagesResult + return JSONResponse({"languages": languageList}) + + +@router.get("/{instanceId}/voice/voices") +@limiter.limit("30/minute") +async def getVoiceVoices( + request: Request, + instanceId: str = Path(...), + language: str = Query("de-DE"), + context: RequestContext = Depends(getRequestContext), +): + """Return available TTS voices for a given language.""" + mandateId = _validateInstanceAccess(instanceId, context) + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + voiceInterface = getVoiceInterface(context.user, mandateId) + voicesResult = await voiceInterface.getAvailableVoices(language) + voiceList = voicesResult.get("voices", []) if isinstance(voicesResult, dict) else voicesResult + return JSONResponse({"voices": voiceList}) + + +@router.post("/{instanceId}/voice/test") +@limiter.limit("10/minute") +async def testVoice( + request: Request, + instanceId: str = Path(...), + body: dict = Body(...), + context: RequestContext = Depends(getRequestContext), +): + """Test a specific voice with a sample text.""" + import base64 + mandateId = _validateInstanceAccess(instanceId, context) + text = body.get("text", "Hallo, das ist ein Stimmtest.") + language = body.get("language", "de-DE") + voiceId = body.get("voiceId") + + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + voiceInterface = getVoiceInterface(context.user, mandateId) + + try: + result = await voiceInterface.textToSpeech(text=text, languageCode=language, voiceName=voiceId) + if result and isinstance(result, dict): + audioContent = result.get("audioContent") + if audioContent: + audioB64 = base64.b64encode( + audioContent if isinstance(audioContent, bytes) else audioContent.encode() + ).decode() + return JSONResponse({"success": True, "audio": audioB64, "format": "mp3", "text": text}) + return JSONResponse({"success": False, "error": "TTS returned no audio"}) + except Exception as e: + logger.error(f"Voice test failed: {e}") + raise HTTPException(status_code=500, detail=f"TTS test failed: {str(e)}") diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index f6a2c41b..d53c9b5a 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -4,7 +4,7 @@ import logging import asyncio import uuid import base64 -from typing import Dict, Any, List, Union, Tuple, Optional, Callable +from typing import Dict, Any, List, Union, Tuple, Optional, Callable, AsyncGenerator from dataclasses import dataclass, field import time @@ -84,15 +84,16 @@ class AiObjects: # AI for Extraction, Processing, Generation async def callWithTextContext(self, request: AiCallRequest) -> AiCallResponse: - """Call AI model for traditional text/context calls with fallback mechanism.""" + """Call AI model for traditional text/context calls with fallback mechanism. + + Supports two modes: + - Legacy: prompt + context → constructs messages internally + - Agent: request.messages provided → passes through directly + """ prompt = request.prompt context = request.context or "" options = request.options - # Input bytes will be calculated inside _callWithModel - - # Generation parameters are handled inside _callWithModel - # Get failover models for this operation type availableModels = modelRegistry.getAvailableModels() @@ -127,10 +128,12 @@ class AiObjects: try: logger.info(f"Attempting AI call with model: {model.name} (attempt {attempt + 1}/{len(failoverModelList)})") - # Call the model directly - no truncation or compression here - response = await self._callWithModel(model, prompt, context, options) + if request.messages: + response = await self._callWithMessages(model, request.messages, options, request.tools) + else: + response = await self._callWithModel(model, prompt, context, options) - logger.info(f"✅ AI call successful with model: {model.name}") + logger.info(f"AI call successful with model: {model.name}") return response except Exception as e: @@ -142,8 +145,7 @@ class AiObjects: logger.info(f"Trying next failover model...") continue else: - # All models failed - logger.error(f"💥 All {len(failoverModelList)} models failed for operation {options.operationType}") + logger.error(f"All {len(failoverModelList)} models failed for operation {options.operationType}") break # All failover attempts failed - return error response @@ -254,6 +256,242 @@ class AiObjects: return response + async def _callWithMessages(self, model: AiModel, messages: List[Dict[str, Any]], + options: AiCallOptions = None, + tools: List[Dict[str, Any]] = None) -> AiCallResponse: + """Call a model with pre-built messages (agent mode). Supports tools for native function calling.""" + import json as _json + + inputBytes = sum(len(str(m.get("content", "")).encode("utf-8")) for m in messages) + startTime = time.time() + + if not model.functionCall: + raise ValueError(f"Model {model.name} has no function call defined") + + modelCall = AiModelCall( + messages=messages, + model=model, + options=options or {}, + tools=tools + ) + + modelResponse = await model.functionCall(modelCall) + + if not modelResponse.success: + raise ValueError(f"Model call failed: {modelResponse.error}") + + endTime = time.time() + processingTime = endTime - startTime + content = modelResponse.content + outputBytes = len(content.encode("utf-8")) + priceCHF = model.calculatepriceCHF(processingTime, inputBytes, outputBytes) + + # Extract tool calls from metadata if present (native function calling) + responseToolCalls = None + if modelResponse.metadata: + responseToolCalls = modelResponse.metadata.get("toolCalls") + + response = AiCallResponse( + content=content, + modelName=model.name, + provider=model.connectorType, + priceCHF=priceCHF, + processingTime=processingTime, + bytesSent=inputBytes, + bytesReceived=outputBytes, + errorCount=0, + toolCalls=responseToolCalls + ) + + if self.billingCallback: + try: + self.billingCallback(response) + except Exception as e: + logger.error(f"BILLING: Failed to record billing for model {model.name}: {e}") + + return response + + async def callWithTextContextStream( + self, request: AiCallRequest + ) -> AsyncGenerator[Union[str, AiCallResponse], None]: + """Streaming variant of callWithTextContext. Yields str deltas, then final AiCallResponse.""" + options = request.options + availableModels = modelRegistry.getAvailableModels() + + allowedProviders = getattr(options, 'allowedProviders', None) if options else None + if allowedProviders: + filtered = [m for m in availableModels if m.connectorType in allowedProviders] + if filtered: + availableModels = filtered + + failoverModelList = modelSelector.getFailoverModelList( + request.prompt, request.context or "", options, availableModels + ) + if not failoverModelList: + yield AiCallResponse( + content=f"No suitable models found for operation {options.operationType}", + modelName="error", priceCHF=0.0, processingTime=0.0, + bytesSent=0, bytesReceived=0, errorCount=1, + ) + return + + lastError = None + for attempt, model in enumerate(failoverModelList): + try: + logger.info(f"Streaming AI call with model: {model.name} (attempt {attempt + 1})") + async for chunk in self._callWithMessagesStream(model, request.messages, options, request.tools): + yield chunk + return + except Exception as e: + lastError = e + logger.warning(f"Streaming AI call failed with {model.name}: {e}") + modelSelector.reportFailure(model.name) + if attempt < len(failoverModelList) - 1: + continue + break + + yield AiCallResponse( + content=f"All models failed (stream). Last error: {lastError}", + modelName="error", priceCHF=0.0, processingTime=0.0, + bytesSent=0, bytesReceived=0, errorCount=1, + ) + + async def _callWithMessagesStream( + self, model: AiModel, messages: List[Dict[str, Any]], + options: AiCallOptions = None, tools: List[Dict[str, Any]] = None, + ) -> AsyncGenerator[Union[str, AiCallResponse], None]: + """Stream a model call. Yields str deltas, then final AiCallResponse with billing.""" + from modules.datamodels.datamodelAi import AiModelCall, AiModelResponse + + inputBytes = sum(len(str(m.get("content", "")).encode("utf-8")) for m in messages) + startTime = time.time() + + if not model.functionCallStream: + response = await self._callWithMessages(model, messages, options, tools) + if response.content: + yield response.content + yield response + return + + modelCall = AiModelCall( + messages=messages, model=model, + options=options or {}, tools=tools, + ) + + finalModelResponse = None + async for item in model.functionCallStream(modelCall): + if isinstance(item, AiModelResponse): + finalModelResponse = item + else: + yield item + + if not finalModelResponse: + raise ValueError(f"Stream from {model.name} produced no final AiModelResponse") + + endTime = time.time() + processingTime = endTime - startTime + content = finalModelResponse.content + outputBytes = len(content.encode("utf-8")) + priceCHF = model.calculatepriceCHF(processingTime, inputBytes, outputBytes) + + responseToolCalls = None + if finalModelResponse.metadata: + responseToolCalls = finalModelResponse.metadata.get("toolCalls") + + response = AiCallResponse( + content=content, + modelName=model.name, + provider=model.connectorType, + priceCHF=priceCHF, + processingTime=processingTime, + bytesSent=inputBytes, + bytesReceived=outputBytes, + errorCount=0, + toolCalls=responseToolCalls, + ) + + if self.billingCallback: + try: + self.billingCallback(response) + except Exception as e: + logger.error(f"BILLING: Failed to record stream billing for {model.name}: {e}") + + yield response + + async def callEmbedding(self, texts: List[str], options: AiCallOptions = None) -> AiCallResponse: + """Generate embeddings for a list of texts using the best available embedding model. + + Uses the standard model selector with OperationTypeEnum.EMBEDDING to pick the model. + Failover across providers (OpenAI → Mistral) works identically to chat models. + + Returns: + AiCallResponse with metadata["embeddings"] containing the vectors. + """ + if options is None: + options = AiCallOptions(operationType=OperationTypeEnum.EMBEDDING) + else: + options.operationType = OperationTypeEnum.EMBEDDING + + combinedText = " ".join(texts[:3])[:500] + availableModels = modelRegistry.getAvailableModels() + failoverModelList = modelSelector.getFailoverModelList( + combinedText, "", options, availableModels + ) + + if not failoverModelList: + return AiCallResponse( + content="", modelName="error", priceCHF=0.0, + processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=1 + ) + + lastError = None + for attempt, model in enumerate(failoverModelList): + try: + logger.info(f"Embedding call with {model.name} (attempt {attempt + 1}/{len(failoverModelList)})") + inputBytes = sum(len(t.encode("utf-8")) for t in texts) + startTime = time.time() + + modelCall = AiModelCall( + model=model, options=options, embeddingInput=texts + ) + modelResponse = await model.functionCall(modelCall) + + if not modelResponse.success: + raise ValueError(f"Embedding call failed: {modelResponse.error}") + + processingTime = time.time() - startTime + priceCHF = model.calculatepriceCHF(processingTime, inputBytes, 0) + embeddings = (modelResponse.metadata or {}).get("embeddings", []) + + response = AiCallResponse( + content="", modelName=model.name, provider=model.connectorType, + priceCHF=priceCHF, processingTime=processingTime, + bytesSent=inputBytes, bytesReceived=0, errorCount=0, + metadata={"embeddings": embeddings} + ) + + if self.billingCallback: + try: + self.billingCallback(response) + except Exception as e: + logger.error(f"BILLING: Failed to record billing for embedding {model.name}: {e}") + + return response + + except Exception as e: + lastError = e + logger.warning(f"Embedding call failed with {model.name}: {str(e)}") + modelSelector.reportFailure(model.name) + if attempt < len(failoverModelList) - 1: + continue + break + + errorMsg = f"All embedding models failed. Last error: {str(lastError)}" + logger.error(errorMsg) + return AiCallResponse( + content=errorMsg, modelName="error", priceCHF=0.0, + processingTime=0.0, bytesSent=0, bytesReceived=0, errorCount=1 + ) # Utility methods async def listAvailableModels(self, connectorType: str = None) -> List[Dict[str, Any]]: diff --git a/modules/interfaces/interfaceDbBilling.py b/modules/interfaces/interfaceDbBilling.py index 8ffd7254..08c43189 100644 --- a/modules/interfaces/interfaceDbBilling.py +++ b/modules/interfaces/interfaceDbBilling.py @@ -764,7 +764,11 @@ class BillingObjects: featureCode: str = None, aicoreProvider: str = None, aicoreModel: str = None, - description: str = "AI Usage" + description: str = "AI Usage", + processingTime: float = None, + bytesSent: int = None, + bytesReceived: int = None, + errorCount: int = None ) -> Optional[Dict[str, Any]]: """ Record usage cost as a billing transaction. @@ -774,20 +778,6 @@ class BillingObjects: - PREPAY_USER: deduct from user's own balance - PREPAY_MANDATE: deduct from mandate pool balance - CREDIT_POSTPAY: deduct from mandate pool balance - - Args: - mandateId: Mandate ID - userId: User ID - priceCHF: Cost in CHF - workflowId: Optional workflow ID - featureInstanceId: Optional feature instance ID - featureCode: Optional feature code - aicoreProvider: AICore provider name (e.g., 'anthropic', 'openai') - aicoreModel: AICore model name (e.g., 'claude-4-sonnet', 'gpt-4o') - description: Transaction description - - Returns: - Created transaction dict or None """ if priceCHF <= 0: return None @@ -816,7 +806,11 @@ class BillingObjects: featureCode=featureCode, aicoreProvider=aicoreProvider, aicoreModel=aicoreModel, - createdByUserId=userId + createdByUserId=userId, + processingTime=processingTime, + bytesSent=bytesSent, + bytesReceived=bytesReceived, + errorCount=errorCount ) # Determine where to deduct balance @@ -828,6 +822,20 @@ class BillingObjects: poolAccount = self.getOrCreateMandateAccount(mandateId) return self.createTransaction(transaction, balanceAccountId=poolAccount["id"]) + # ========================================================================= + # Workflow Cost Query + # ========================================================================= + + def getWorkflowCost(self, workflowId: str) -> float: + """Sum of all transaction amounts for a workflow.""" + if not workflowId: + return 0.0 + transactions = self.db.getRecordset( + BillingTransaction, + recordFilter={"workflowId": workflowId} + ) + return sum(t.get("amount", 0.0) for t in transactions) + # ========================================================================= # Billing Model Switch Operations # ========================================================================= diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index 5ed7fa6c..9ad072ad 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -18,7 +18,6 @@ from modules.datamodels.datamodelUam import AccessLevel from modules.datamodels.datamodelChat import ( ChatDocument, - ChatStat, ChatLog, ChatMessage, ChatWorkflow, @@ -663,10 +662,8 @@ class ChatObjects: workflow = workflows[0] try: - # Load related data from normalized tables logs = self.getLogs(workflowId) messages = self.getMessages(workflowId) - stats = self.getStats(workflowId) # Validate workflow data against ChatWorkflow model # Explicit type coercion: DB may store numeric fields as TEXT on some platforms @@ -694,8 +691,7 @@ class ChatObjects: lastActivity=_toFloat(workflow.get("lastActivity")), startedAt=_toFloat(workflow.get("startedAt")), logs=logs, - messages=messages, - stats=stats + messages=messages ) except Exception as e: logger.error(f"Error validating workflow data: {str(e)}") @@ -731,7 +727,7 @@ class ChatObjects: except Exception as e: logger.warning(f"Could not get Root mandate: {e}") # Note: ChatWorkflow has featureInstanceId for multi-tenancy isolation. - # Child tables (ChatMessage, ChatLog, ChatStat, ChatDocument) are user-owned + # Child tables (ChatMessage, ChatLog, ChatDocument) are user-owned # and do NOT store featureInstanceId - they inherit isolation from ChatWorkflow. # Ensure featureInstanceId is set from context if not already in workflowData if "featureInstanceId" not in workflowData or not workflowData.get("featureInstanceId"): @@ -760,7 +756,7 @@ class ChatObjects: logs=[], messages=[], stats=[], - workflowMode=created["workflowMode"], + workflowMode=created.get("workflowMode", "Dynamic"), maxSteps=created.get("maxSteps", 1) ) @@ -789,23 +785,20 @@ class ChatObjects: # Load fresh data from normalized tables logs = self.getLogs(workflowId) messages = self.getMessages(workflowId) - stats = self.getStats(workflowId) - # Convert to ChatWorkflow model return ChatWorkflow( id=updated["id"], status=updated.get("status", workflow.status), name=updated.get("name", workflow.name), - currentRound=updated.get("currentRound", workflow.currentRound), - currentTask=updated.get("currentTask", workflow.currentTask), - currentAction=updated.get("currentAction", workflow.currentAction), - totalTasks=updated.get("totalTasks", workflow.totalTasks), - totalActions=updated.get("totalActions", workflow.totalActions), + currentRound=updated.get("currentRound") or getattr(workflow, "currentRound", 0) or 0, + currentTask=updated.get("currentTask") or getattr(workflow, "currentTask", 0) or 0, + currentAction=updated.get("currentAction") or getattr(workflow, "currentAction", 0) or 0, + totalTasks=updated.get("totalTasks") or getattr(workflow, "totalTasks", 0) or 0, + totalActions=updated.get("totalActions") or getattr(workflow, "totalActions", 0) or 0, lastActivity=updated.get("lastActivity", workflow.lastActivity), startedAt=updated.get("startedAt", workflow.startedAt), logs=logs, - messages=messages, - stats=stats + messages=messages ) def deleteWorkflow(self, workflowId: str) -> bool: @@ -827,7 +820,6 @@ class ChatObjects: messageId = message.id if messageId: # Delete message documents (but NOT the files!) - # Note: ChatStat does NOT have messageId - stats are only at workflow level try: existing_docs = self._getRecordset(ChatDocument, recordFilter={"messageId": messageId}) for doc in existing_docs: @@ -839,11 +831,7 @@ class ChatObjects: self.db.recordDelete(ChatMessage, messageId) # 2. Delete workflow stats - existing_stats = self._getRecordset(ChatStat, recordFilter={"workflowId": workflowId}) - for stat in existing_stats: - self.db.recordDelete(ChatStat, stat["id"]) - - # 3. Delete workflow logs + # 2. Delete workflow logs existing_logs = self._getRecordset(ChatLog, recordFilter={"workflowId": workflowId}) for log in existing_logs: self.db.recordDelete(ChatLog, log["id"]) @@ -1270,7 +1258,6 @@ class ChatObjects: self.db.recordDelete(ChatDocument, doc["id"]) # 2. Finally delete the message itself - # Note: ChatStat has no messageId field -- stats are workflow-level, not message-level success = self.db.recordDelete(ChatMessage, messageId) return success @@ -1517,74 +1504,10 @@ class ChatObjects: # Return validated ChatLog instance return ChatLog(**createdLog) - # Stats methods - - def getStats(self, workflowId: str) -> List[ChatStat]: - """Returns list of statistics for a workflow if user has access.""" - # Check workflow access first (without calling getWorkflow to avoid circular reference) - # Use RBAC filtering - workflows = self._getRecordset(ChatWorkflow, recordFilter={"id": workflowId}) - - if not workflows: - return [] - - # Get stats for this workflow from normalized table - stats = self._getRecordset(ChatStat, recordFilter={"workflowId": workflowId}) - - if not stats: - return [] - - # Return all stats records sorted by creation time. - # Use parseTimestamp to tolerate mixed DB types (float/string) on INT. - # DB uses _createdAt (camelCase system field). - stats.sort(key=lambda x: parseTimestamp(x.get("_createdAt"), default=0)) - - # Convert to ChatStat objects, preserving _createdAt via extra="allow" - result = [] - for stat in stats: - chat_stat = ChatStat(**stat) - # Explicitly preserve _createdAt from raw DB record - if "_createdAt" in stat: - setattr(chat_stat, '_createdAt', stat["_createdAt"]) - result.append(chat_stat) - - return result - - - def createStat(self, statData: Dict[str, Any]) -> ChatStat: - """Creates a new stats record and returns it.""" - try: - # Ensure workflowId is present in statData - if "workflowId" not in statData: - raise ValueError("workflowId is required in statData") - - # Note: Chat data is user-owned, no mandate/featureInstance context stored - # mandateId/featureInstanceId removed from ChatStat model - - # Validate the stat data against ChatStat model - stat = ChatStat(**statData) - - logger.debug(f"Creating stat for workflow {statData.get('workflowId')}: " - f"process={statData.get('process')}, " - f"priceCHF={statData.get('priceCHF', 0):.4f}, " - f"processingTime={statData.get('processingTime', 0):.2f}s") - - # Create the stat record in the database - created = self.db.recordCreate(ChatStat, stat) - - logger.info(f"Created stat {created.get('id')} for workflow {statData.get('workflowId')}") - - # Return the created ChatStat - return ChatStat(**created) - except Exception as e: - logger.error(f"Error creating workflow stat: {str(e)}") - raise - - - def getUnifiedChatData(self, workflowId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: + def getUnifiedChatData(self, workflowId: str, afterTimestamp: Optional[float] = None, workflowCost: float = 0.0) -> Dict[str, Any]: """ - Returns unified chat data (messages, logs, stats) for a workflow in chronological order. - Uses timestamp-based selective data transfer for efficient polling. + Returns unified chat data (messages, logs) for a workflow in chronological order, + plus workflowCost from billing transactions (single source of truth). """ # Check workflow access first # Use RBAC filtering @@ -1652,29 +1575,10 @@ class ChatObjects: "item": chatLog }) - # Get stats - ChatStat model supports _createdAt via model_config extra="allow" - stats = self.getStats(workflowId) - for stat in stats: - # Apply timestamp filtering in Python - # Use _createdAt (system field from DB, preserved via model_config extra="allow") - stat_timestamp = getattr(stat, '_createdAt', None) or getUtcTimestamp() - if afterTimestamp is not None and stat_timestamp <= afterTimestamp: - continue - - # Convert to dict and include _createdAt for frontend - stat_dict = stat.model_dump() if hasattr(stat, 'model_dump') else stat.dict() - stat_dict['_createdAt'] = stat_timestamp - - items.append({ - "type": "stat", - "createdAt": stat_timestamp, - "item": stat_dict - }) - # Sort all items by createdAt timestamp for chronological order items.sort(key=lambda x: parseTimestamp(x.get("createdAt"), default=0)) - return {"items": items} + return {"items": items, "workflowCost": workflowCost} def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': diff --git a/modules/interfaces/interfaceDbKnowledge.py b/modules/interfaces/interfaceDbKnowledge.py new file mode 100644 index 00000000..e15f19c2 --- /dev/null +++ b/modules/interfaces/interfaceDbKnowledge.py @@ -0,0 +1,234 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Interface to the Knowledge Store database (poweron_knowledge). +Provides CRUD for FileContentIndex, ContentChunk, WorkflowMemory +and semantic search via pgvector. +""" + +import logging +from typing import Dict, Any, List, Optional + +from modules.connectors.connectorDbPostgre import _get_cached_connector +from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk, WorkflowMemory +from modules.datamodels.datamodelUam import User +from modules.shared.configuration import APP_CONFIG +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +_instances: Dict[str, "KnowledgeObjects"] = {} + + +class KnowledgeObjects: + """Interface to the Knowledge Store database. + Manages FileContentIndex, ContentChunk, and WorkflowMemory with semantic search.""" + + def __init__(self): + self.currentUser: Optional[User] = None + self.userId: Optional[str] = None + self._initializeDatabase() + + def _initializeDatabase(self): + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_knowledge" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + + self.db = _get_cached_connector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId, + ) + logger.info("Knowledge Store database initialized") + + def setUserContext(self, user: User): + self.currentUser = user + self.userId = user.id if user else None + if self.userId: + self.db.updateContext(self.userId) + + # ========================================================================= + # FileContentIndex CRUD + # ========================================================================= + + def upsertFileContentIndex(self, index: FileContentIndex) -> Dict[str, Any]: + """Create or update a FileContentIndex entry.""" + data = index.model_dump() + existing = self.db._loadRecord(FileContentIndex, index.id) + if existing: + return self.db.recordModify(FileContentIndex, index.id, data) + return self.db.recordCreate(FileContentIndex, data) + + def getFileContentIndex(self, fileId: str) -> Optional[Dict[str, Any]]: + """Get a FileContentIndex by file ID.""" + return self.db._loadRecord(FileContentIndex, fileId) + + def getFileContentIndexByUser( + self, userId: str, featureInstanceId: str = None + ) -> List[Dict[str, Any]]: + """Get all FileContentIndex entries for a user.""" + recordFilter = {"userId": userId} + if featureInstanceId: + recordFilter["featureInstanceId"] = featureInstanceId + return self.db.getRecordset(FileContentIndex, recordFilter=recordFilter) + + def updateFileStatus(self, fileId: str, status: str) -> bool: + """Update the processing status of a FileContentIndex.""" + existing = self.db._loadRecord(FileContentIndex, fileId) + if not existing: + return False + self.db.recordModify(FileContentIndex, fileId, {"status": status}) + return True + + def deleteFileContentIndex(self, fileId: str) -> bool: + """Delete a FileContentIndex and all associated ContentChunks.""" + chunks = self.db.getRecordset(ContentChunk, recordFilter={"fileId": fileId}) + for chunk in chunks: + self.db.recordDelete(ContentChunk, chunk["id"]) + return self.db.recordDelete(FileContentIndex, fileId) + + # ========================================================================= + # ContentChunk CRUD + # ========================================================================= + + def upsertContentChunk(self, chunk: ContentChunk) -> Dict[str, Any]: + """Create or update a ContentChunk.""" + data = chunk.model_dump() + existing = self.db._loadRecord(ContentChunk, chunk.id) + if existing: + return self.db.recordModify(ContentChunk, chunk.id, data) + return self.db.recordCreate(ContentChunk, data) + + def upsertContentChunks(self, chunks: List[ContentChunk]) -> int: + """Batch upsert multiple ContentChunks. Returns count of upserted chunks.""" + count = 0 + for chunk in chunks: + self.upsertContentChunk(chunk) + count += 1 + return count + + def getContentChunks(self, fileId: str) -> List[Dict[str, Any]]: + """Get all ContentChunks for a file.""" + return self.db.getRecordset(ContentChunk, recordFilter={"fileId": fileId}) + + def deleteContentChunks(self, fileId: str) -> int: + """Delete all ContentChunks for a file. Returns count of deleted chunks.""" + chunks = self.db.getRecordset(ContentChunk, recordFilter={"fileId": fileId}) + count = 0 + for chunk in chunks: + if self.db.recordDelete(ContentChunk, chunk["id"]): + count += 1 + return count + + # ========================================================================= + # WorkflowMemory CRUD + # ========================================================================= + + def upsertWorkflowMemory(self, memory: WorkflowMemory) -> Dict[str, Any]: + """Create or update a WorkflowMemory entry.""" + data = memory.model_dump() + existing = self.db._loadRecord(WorkflowMemory, memory.id) + if existing: + return self.db.recordModify(WorkflowMemory, memory.id, data) + return self.db.recordCreate(WorkflowMemory, data) + + def getWorkflowEntities(self, workflowId: str) -> List[Dict[str, Any]]: + """Get all WorkflowMemory entries for a workflow.""" + return self.db.getRecordset(WorkflowMemory, recordFilter={"workflowId": workflowId}) + + def getWorkflowEntity(self, workflowId: str, key: str) -> Optional[Dict[str, Any]]: + """Get a specific WorkflowMemory entry by workflow and key.""" + results = self.db.getRecordset( + WorkflowMemory, recordFilter={"workflowId": workflowId, "key": key} + ) + return results[0] if results else None + + def deleteWorkflowMemory(self, workflowId: str) -> int: + """Delete all WorkflowMemory entries for a workflow. Returns count.""" + entries = self.db.getRecordset(WorkflowMemory, recordFilter={"workflowId": workflowId}) + count = 0 + for entry in entries: + if self.db.recordDelete(WorkflowMemory, entry["id"]): + count += 1 + return count + + # ========================================================================= + # Semantic Search + # ========================================================================= + + def semanticSearch( + self, + queryVector: List[float], + userId: str = None, + featureInstanceId: str = None, + mandateId: str = None, + isShared: bool = None, + limit: int = 10, + minScore: float = None, + contentType: str = None, + ) -> List[Dict[str, Any]]: + """Semantic search across ContentChunks using pgvector cosine similarity. + + Args: + queryVector: Query embedding vector. + userId: Filter by user (Instance Layer). + featureInstanceId: Filter by feature instance. + mandateId: Filter by mandate (for Shared Layer lookups). + isShared: If True, search Shared Layer via FileContentIndex join. + limit: Max results. + minScore: Minimum cosine similarity (0.0 - 1.0). + contentType: Filter by content type (text, image, etc.). + + Returns: + List of ContentChunk records with _score field, sorted by relevance. + """ + recordFilter = {} + if userId: + recordFilter["userId"] = userId + if featureInstanceId: + recordFilter["featureInstanceId"] = featureInstanceId + if contentType: + recordFilter["contentType"] = contentType + + return self.db.semanticSearch( + modelClass=ContentChunk, + vectorColumn="embedding", + queryVector=queryVector, + limit=limit, + recordFilter=recordFilter if recordFilter else None, + minScore=minScore, + ) + + def semanticSearchWorkflowMemory( + self, + queryVector: List[float], + workflowId: str, + limit: int = 5, + minScore: float = None, + ) -> List[Dict[str, Any]]: + """Semantic search across WorkflowMemory entries.""" + return self.db.semanticSearch( + modelClass=WorkflowMemory, + vectorColumn="embedding", + queryVector=queryVector, + limit=limit, + recordFilter={"workflowId": workflowId}, + minScore=minScore, + ) + + +def getInterface(currentUser: Optional[User] = None) -> KnowledgeObjects: + """Get or create a KnowledgeObjects singleton.""" + if "default" not in _instances: + _instances["default"] = KnowledgeObjects() + + interface = _instances["default"] + if currentUser: + interface.setUserContext(currentUser) + + return interface diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index b16f0b24..bedc6a81 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -58,7 +58,6 @@ TABLE_NAMESPACE = { "ChatWorkflow": "chat", "ChatMessage": "chat", "ChatLog": "chat", - "ChatStat": "chat", "ChatDocument": "chat", "Prompt": "chat", # Chatbot (poweron_chatbot) - per feature-instance isolation @@ -69,13 +68,20 @@ TABLE_NAMESPACE = { # Files - benutzer-eigen "FileItem": "files", "FileData": "files", + "FileFolder": "files", # Automation - benutzer-eigen "AutomationDefinition": "automation", "AutomationTemplate": "automation", + # Knowledge Store - benutzer-eigen + "FileContentIndex": "knowledge", + "ContentChunk": "knowledge", + "WorkflowMemory": "knowledge", + # Data Sources - benutzer-eigen + "DataSource": "datasource", } # Namespaces ohne Mandantenkontext - GROUP wird auf MY gemappt -USER_OWNED_NAMESPACES = {"chat", "chatbot", "files", "automation"} +USER_OWNED_NAMESPACES = {"chat", "chatbot", "files", "automation", "knowledge", "datasource"} def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str: @@ -175,7 +181,7 @@ def getRecordsetWithRBAC( whereValues = [] # CRITICAL: Only pass featureInstanceId to WHERE clause if the model actually has - # this column. Chat child tables (ChatMessage, ChatLog, ChatStat, ChatDocument) + # this column. Chat child tables (ChatMessage, ChatLog, ChatDocument) # are user-owned and do NOT have featureInstanceId - only ChatWorkflow does. # Without this check, the SQL query would reference a non-existent column, # causing a silent error that returns empty results. diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 18e26335..227fbb4c 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -247,19 +247,13 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict # Get FeatureAccess for this user and instance (Pydantic model) featureAccess = rootInterface.getFeatureAccess(userId, instanceId) - logger.debug(f"_getInstancePermissions: userId={userId}, instanceId={instanceId}, featureAccess={featureAccess is not None}") - if not featureAccess: - logger.debug(f"_getInstancePermissions: No FeatureAccess found for user {userId} and instance {instanceId}") return permissions # Get role IDs via interface method roleIds = rootInterface.getRoleIdsForFeatureAccess(str(featureAccess.id)) - logger.debug(f"_getInstancePermissions: featureAccessId={featureAccess.id}, roleIds={roleIds}") - if not roleIds: - logger.debug(f"_getInstancePermissions: No roles found for FeatureAccess {featureAccess.id}") return permissions # Check if user has admin role @@ -274,8 +268,6 @@ def _getInstancePermissions(rootInterface, userId: str, instanceId: str) -> Dict # Get all rules for this role (returns Pydantic models) accessRules = rootInterface.getAccessRules(roleId=roleId) - logger.debug(f"_getInstancePermissions: roleId={roleId}, accessRules={len(accessRules) if accessRules else 0}") - for rule in accessRules: context = rule.context item = rule.item or "" diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index 2f4aea86..194ebba0 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -21,7 +21,7 @@ from modules.auth import limiter, requireSysAdminRole, getRequestContext, Reques # Import billing components from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface, _getRootInterface -from modules.services.serviceBilling.mainServiceBilling import getService as getBillingService +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict from modules.routes.routeDataUsers import _applyFiltersAndSort from modules.datamodels.datamodelBilling import ( @@ -162,6 +162,23 @@ def _isAdminOfMandate(ctx: RequestContext, targetMandateId: str) -> bool: return False +def _isMemberOfMandate(ctx: RequestContext, targetMandateId: str) -> bool: + """Check if user has any enabled membership in the specified mandate.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + rootInterface = getRootInterface() + userMandates = rootInterface.getUserMandates(str(ctx.user.id)) + for um in userMandates: + if str(getattr(um, 'mandateId', None)) != str(targetMandateId): + continue + if not getattr(um, 'enabled', True): + continue + return True + return False + except Exception: + return False + + def _filterTransactionsByScope(transactions: list, scope: BillingDataScope) -> list: """ Filter a list of transaction dicts based on the user's BillingDataScope. @@ -720,11 +737,11 @@ def createCheckoutSession( targetMandateId: str = Path(..., description="Mandate ID"), checkoutRequest: CheckoutCreateRequest = Body(...), ctx: RequestContext = Depends(getRequestContext), - _admin = Depends(requireSysAdminRole) ): """ Create Stripe Checkout Session for credit top-up. Returns redirect URL. - SysAdmin only. Amount is validated server-side against allowed presets. + RBAC: PREPAY_USER requires mandate membership (user loads own account), + PREPAY_MANDATE requires mandate admin role. """ try: billingInterface = getBillingInterface(ctx.user, targetMandateId) @@ -738,10 +755,17 @@ def createCheckoutSession( if billingModel == BillingModelEnum.PREPAY_USER: if not checkoutRequest.userId: raise HTTPException(status_code=400, detail="userId is required for PREPAY_USER model") - elif billingModel not in [BillingModelEnum.PREPAY_MANDATE, BillingModelEnum.CREDIT_POSTPAY]: + if str(checkoutRequest.userId) != str(ctx.user.id): + raise HTTPException(status_code=403, detail="Users can only load credit to their own account") + if not _isMemberOfMandate(ctx, targetMandateId): + raise HTTPException(status_code=403, detail="User is not a member of this mandate") + elif billingModel in [BillingModelEnum.PREPAY_MANDATE, BillingModelEnum.CREDIT_POSTPAY]: + if not _isAdminOfMandate(ctx, targetMandateId): + raise HTTPException(status_code=403, detail="Mandate admin role required to load mandate credit") + else: raise HTTPException(status_code=400, detail=f"Cannot add credit to {billingModel.value} billing model") - from modules.services.serviceBilling.stripeCheckout import create_checkout_session + from modules.serviceCenter.services.serviceBilling.stripeCheckout import create_checkout_session redirect_url = create_checkout_session( mandate_id=targetMandateId, user_id=checkoutRequest.userId, @@ -768,7 +792,7 @@ async def stripeWebhook( No JWT auth - Stripe authenticates via Stripe-Signature header. """ from modules.shared.configuration import APP_CONFIG - from modules.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF + from modules.serviceCenter.services.serviceBilling.stripeCheckout import ALLOWED_AMOUNTS_CHF webhook_secret = APP_CONFIG.get("STRIPE_WEBHOOK_SECRET") if not webhook_secret: diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index e8fceaff..46182f60 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -19,6 +19,114 @@ from modules.datamodels.datamodelPagination import PaginationParams, PaginatedRe # Configure logger logger = logging.getLogger(__name__) + +async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user): + """Background task: pre-scan + extraction + knowledge indexing. + Step 1: Structure Pre-Scan (AI-free) -> FileContentIndex (persisted) + Step 2: Content extraction via runExtraction -> ContentParts + Step 3: KnowledgeService.indexFile -> chunking + embedding -> Knowledge Store""" + userId = user.id if hasattr(user, "id") else str(user) + try: + mgmtInterface = interfaceDbManagement.getInterface(user) + mgmtInterface.updateFile(fileId, {"status": "processing"}) + + rawBytes = mgmtInterface.getFileData(fileId) + if not rawBytes: + logger.warning(f"Auto-index: no file data for {fileId}, skipping") + mgmtInterface.updateFile(fileId, {"status": "active"}) + return + + logger.info(f"Auto-index starting for {fileName} ({len(rawBytes)} bytes, {mimeType})") + + # Step 1: Structure Pre-Scan (AI-free) + from modules.serviceCenter.services.serviceKnowledge.subPreScan import preScanDocument + contentIndex = await preScanDocument( + fileData=rawBytes, + mimeType=mimeType, + fileId=fileId, + fileName=fileName, + userId=userId, + ) + logger.info( + f"Pre-scan complete for {fileName}: " + f"{contentIndex.totalObjects} objects" + ) + + # Persist FileContentIndex immediately + from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface + knowledgeDb = getKnowledgeInterface() + knowledgeDb.upsertFileContentIndex(contentIndex) + + # Step 2: Content extraction (AI-free, produces ContentParts) + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry, ChunkerRegistry + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction + from modules.datamodels.datamodelExtraction import ExtractionOptions + + extractorRegistry = ExtractorRegistry() + chunkerRegistry = ChunkerRegistry() + options = ExtractionOptions() + + extracted = runExtraction( + extractorRegistry, chunkerRegistry, + rawBytes, fileName, mimeType, options, + ) + + contentObjects = [] + for part in extracted.parts: + contentType = "text" + if part.typeGroup == "image": + contentType = "image" + elif part.typeGroup in ("binary", "container"): + contentType = "other" + + if not part.data or not part.data.strip(): + continue + + contentObjects.append({ + "contentObjectId": part.id, + "contentType": contentType, + "data": part.data, + "contextRef": { + "containerPath": fileName, + "location": part.label or "file", + **(part.metadata or {}), + }, + }) + + logger.info(f"Extracted {len(contentObjects)} content objects from {fileName}") + + if not contentObjects: + knowledgeDb.updateFileStatus(fileId, "indexed") + mgmtInterface.updateFile(fileId, {"status": "active"}) + return + + # Step 3: Knowledge indexing (chunking + embedding) + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + + ctx = ServiceCenterContext(user=user, mandate_id="", feature_instance_id="") + knowledgeService = getService("knowledge", ctx) + + await knowledgeService.indexFile( + fileId=fileId, + fileName=fileName, + mimeType=mimeType, + userId=userId, + contentObjects=contentObjects, + structure=contentIndex.structure, + ) + + mgmtInterface.updateFile(fileId, {"status": "active"}) + logger.info(f"Auto-index complete for file {fileId} ({fileName})") + + except Exception as e: + logger.error(f"Auto-index failed for file {fileId}: {e}", exc_info=True) + try: + errMgmt = interfaceDbManagement.getInterface(user) + errMgmt.updateFile(fileId, {"status": "active"}) + except Exception: + pass + # Model attributes for FileItem fileAttributes = getModelAttributeDefinitions(FileItem) @@ -111,6 +219,7 @@ async def upload_file( request: Request, file: UploadFile = File(...), workflowId: Optional[str] = Form(None), + featureInstanceId: Optional[str] = Form(None), currentUser: User = Depends(getCurrentUser) ) -> JSONResponse: # Add fileName property to UploadFile for consistency with backend model @@ -132,6 +241,10 @@ async def upload_file( # Save file via LucyDOM interface in the database fileItem, duplicateType = managementInterface.saveUploadedFile(fileContent, file.filename) + + if featureInstanceId and not fileItem.featureInstanceId: + managementInterface.updateFile(fileItem.id, {"featureInstanceId": featureInstanceId}) + fileItem.featureInstanceId = featureInstanceId # Determine response message based on duplicate type if duplicateType == "exact_duplicate": @@ -148,6 +261,32 @@ async def upload_file( if workflowId: fileMeta["workflowId"] = workflowId + # Trigger background auto-index pipeline (non-blocking) + # Also runs for duplicates in case the original was never successfully indexed + shouldIndex = duplicateType == "new_file" + if not shouldIndex: + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as _getKnowledgeInterface + _kDb = _getKnowledgeInterface() + _existingIndex = _kDb.getFileContentIndex(fileItem.id) + if not _existingIndex: + shouldIndex = True + logger.info(f"Re-triggering auto-index for duplicate {fileItem.id} (not yet indexed)") + except Exception: + shouldIndex = True + + if shouldIndex: + try: + import asyncio + asyncio.ensure_future(_autoIndexFile( + fileId=fileItem.id, + fileName=fileItem.fileName, + mimeType=fileItem.mimeType, + user=currentUser, + )) + except Exception as indexErr: + logger.warning(f"Auto-index trigger failed (non-blocking): {indexErr}") + # Response with duplicate information return JSONResponse({ "message": message, diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 9717a602..16742388 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -764,7 +764,7 @@ def send_password_link( expiryHours = int(APP_CONFIG.get("Auth_RESET_TOKEN_EXPIRY_HOURS", "24")) try: - from modules.services import Services + from modules.serviceHub import Services services = Services(targetUser) emailSubject = "PowerOn - Passwort setzen" diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index 3615d76e..0b4784ff 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -395,7 +395,7 @@ def trigger_subscription( ) # Get messaging service from request app state - from modules.services import getInterface as getServicesInterface + from modules.serviceHub import getInterface as getServicesInterface services = getServicesInterface(context.user, None, mandateId=str(context.mandateId)) # Konvertiere Dict zu Pydantic Model diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index cfaddc22..ad0dcd52 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -87,9 +87,10 @@ CLIENT_SECRET = APP_CONFIG.get("Service_GOOGLE_CLIENT_SECRET") REDIRECT_URI = APP_CONFIG.get("Service_GOOGLE_REDIRECT_URI") SCOPES = [ "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/drive.readonly", "https://www.googleapis.com/auth/userinfo.profile", "https://www.googleapis.com/auth/userinfo.email", - "openid" + "openid", ] @router.get("/config") @@ -488,7 +489,7 @@ async def auth_callback(code: str, state: str, request: Request, response: Respo connection.externalUsername = user_info.get("email") connection.externalEmail = user_info.get("email") # Store actually granted scopes for this connection - granted_scopes_list = granted_scopes.split(" ") if granted_scopes else SCOPES + granted_scopes_list = granted_scopes if isinstance(granted_scopes, list) else (granted_scopes.split(" ") if granted_scopes else SCOPES) connection.grantedScopes = granted_scopes_list logger.info(f"Storing granted scopes for connection {connection_id}: {granted_scopes_list}") diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 11d35915..97604e67 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -59,6 +59,7 @@ SCOPES = [ "Mail.Send", # Send mail "Files.ReadWrite.All", # Read and write files (SharePoint/OneDrive) "Sites.ReadWrite.All", # Read and write SharePoint sites + "Team.ReadBasic.All", # List joined teams and channels # Teams Bot: Meeting and chat access (requires admin consent) "OnlineMeetings.Read", # Read user's Teams meeting details (delegated scope) "Chat.ReadWrite", # Read and write Teams chat messages diff --git a/modules/routes/routeSharepoint.py b/modules/routes/routeSharepoint.py index 2719562a..9bf5b633 100644 --- a/modules/routes/routeSharepoint.py +++ b/modules/routes/routeSharepoint.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Query, Request, sta from modules.auth import limiter, getCurrentUser from modules.datamodels.datamodelUam import User, UserConnection from modules.interfaces.interfaceDbApp import getInterface -from modules.services import getInterface as getServices +from modules.serviceHub import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/routes/routeSystem.py b/modules/routes/routeSystem.py index 2ef07db7..95d90aa6 100644 --- a/modules/routes/routeSystem.py +++ b/modules/routes/routeSystem.py @@ -123,6 +123,9 @@ def _getFeatureUiObjects(featureCode: str) -> List[Dict[str, Any]]: elif featureCode == "commcoach": from modules.features.commcoach.mainCommcoach import UI_OBJECTS return UI_OBJECTS + elif featureCode == "workspace": + from modules.features.workspace.mainWorkspace import UI_OBJECTS + return UI_OBJECTS else: logger.warning(f"Unknown feature code: {featureCode}") return [] diff --git a/modules/serviceCenter/__init__.py b/modules/serviceCenter/__init__.py index 4f1a5be5..fb40df65 100644 --- a/modules/serviceCenter/__init__.py +++ b/modules/serviceCenter/__init__.py @@ -26,7 +26,6 @@ logger = logging.getLogger(__name__) def getService( key: str, context: ServiceCenterContext, - legacy_hub: Optional[Any] = None, ) -> Any: """ Get a service instance by key for the given context. @@ -34,14 +33,13 @@ def getService( Args: key: Service key (e.g., "web", "extraction", "utils") context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow - legacy_hub: Optional legacy Services instance for fallback when service not yet migrated Returns: Service instance """ cache = get_resolution_cache() resolving = set() - return resolve(key, context, cache, resolving, legacy_hub=legacy_hub) + return resolve(key, context, cache, resolving) def preWarm(service_keys: Optional[List[str]] = None) -> None: diff --git a/modules/serviceCenter/core/serviceStreaming/eventManager.py b/modules/serviceCenter/core/serviceStreaming/eventManager.py index ae603fad..00e29f17 100644 --- a/modules/serviceCenter/core/serviceStreaming/eventManager.py +++ b/modules/serviceCenter/core/serviceStreaming/eventManager.py @@ -22,6 +22,8 @@ class EventManager: """Initialize the event manager.""" self._queues: Dict[str, asyncio.Queue] = {} self._cleanup_tasks: Dict[str, asyncio.Task] = {} + self._agent_tasks: Dict[str, asyncio.Task] = {} + self._cancelled: Dict[str, bool] = {} def create_queue(self, workflow_id: str) -> asyncio.Queue: """ @@ -33,9 +35,22 @@ class EventManager: Returns: Async queue for events """ + if workflow_id in self._cleanup_tasks: + self._cleanup_tasks[workflow_id].cancel() + del self._cleanup_tasks[workflow_id] + logger.debug(f"Cancelled pending cleanup for workflow {workflow_id}") + if workflow_id not in self._queues: self._queues[workflow_id] = asyncio.Queue() logger.debug(f"Created event queue for workflow {workflow_id}") + else: + old = self._queues[workflow_id] + while not old.empty(): + try: + old.get_nowait() + except asyncio.QueueEmpty: + break + logger.debug(f"Reusing event queue for workflow {workflow_id} (drained stale events)") return self._queues[workflow_id] def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]: @@ -62,6 +77,31 @@ class EventManager: """ return workflow_id in self._queues + def register_agent_task(self, workflow_id: str, task: asyncio.Task) -> None: + """Register the asyncio Task running the agent for a workflow.""" + self._agent_tasks[workflow_id] = task + self._cancelled.pop(workflow_id, None) + + def is_cancelled(self, workflow_id: str) -> bool: + """Check if a workflow has been cancelled.""" + return self._cancelled.get(workflow_id, False) + + async def cancel_agent(self, workflow_id: str) -> bool: + """Cancel the running agent task for a workflow. Returns True if cancelled.""" + self._cancelled[workflow_id] = True + task = self._agent_tasks.pop(workflow_id, None) + if task and not task.done(): + task.cancel() + logger.info(f"Cancelled agent task for workflow {workflow_id}") + return True + logger.debug(f"No running agent task found for workflow {workflow_id}") + return False + + def _unregister_agent_task(self, workflow_id: str) -> None: + """Remove the agent task reference after completion.""" + self._agent_tasks.pop(workflow_id, None) + self._cancelled.pop(workflow_id, None) + async def emit_event( self, context_id: str, @@ -97,7 +137,8 @@ class EventManager: try: await queue.put(event) - logger.debug(f"Emitted {event_type} event for workflow {context_id}") + if event_type not in ("chunk",): + logger.debug(f"Emitted {event_type} event for workflow {context_id}") except Exception as e: logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True) diff --git a/modules/serviceCenter/registry.py b/modules/serviceCenter/registry.py index 409c72fd..900f9f0e 100644 --- a/modules/serviceCenter/registry.py +++ b/modules/serviceCenter/registry.py @@ -98,6 +98,20 @@ IMPORTABLE_SERVICES: Dict[str, Dict[str, Any]] = { "objectKey": "service.neutralization", "label": {"en": "Neutralization", "de": "Neutralisierung", "fr": "Neutralisation"}, }, + "agent": { + "module": "modules.serviceCenter.services.serviceAgent.mainServiceAgent", + "class": "AgentService", + "dependencies": ["ai", "chat", "utils", "extraction", "billing", "streaming", "knowledge"], + "objectKey": "service.agent", + "label": {"en": "Agent", "de": "Agent", "fr": "Agent"}, + }, + "knowledge": { + "module": "modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge", + "class": "KnowledgeService", + "dependencies": ["ai"], + "objectKey": "service.knowledge", + "label": {"en": "Knowledge Store", "de": "Wissensspeicher", "fr": "Base de connaissances"}, + }, } # RBAC objects for service-level access control (for catalog registration) diff --git a/modules/serviceCenter/resolver.py b/modules/serviceCenter/resolver.py index dd525491..5d400760 100644 --- a/modules/serviceCenter/resolver.py +++ b/modules/serviceCenter/resolver.py @@ -2,7 +2,7 @@ # All rights reserved. """ Service Center Resolver. -Resolution logic, dependency injection, and optional legacy fallback. +Resolution logic and dependency injection for service instantiation. """ import importlib @@ -14,7 +14,6 @@ from modules.serviceCenter.registry import CORE_SERVICES, IMPORTABLE_SERVICES logger = logging.getLogger(__name__) -# Type for get_service callable passed to services GetServiceFunc = Callable[[str], Any] @@ -29,50 +28,15 @@ def _load_service_class(module_path: str, class_name: str): return getattr(module, class_name) -def _create_legacy_hub(ctx: ServiceCenterContext) -> Any: - """Create legacy Services instance for fallback when service not yet migrated.""" - from modules.services import getInterface - return getInterface( - ctx.user, - workflow=ctx.workflow, - mandateId=ctx.mandate_id, - featureInstanceId=ctx.feature_instance_id, - ) - - -def _get_from_legacy(legacy_hub: Any, key: str) -> Any: - """Map service key to legacy hub attribute (for fallback when service center module fails).""" - key_to_attr = { - "utils": "utils", - "security": "security", - "streaming": "streaming", - "ticket": "ticket", - "messaging": "messaging", - "billing": "billing", - "sharepoint": "sharepoint", - "chat": "chat", - "extraction": "extraction", - "generation": "generation", - "ai": "ai", - "web": "web", - "neutralization": "neutralization", - } - attr = key_to_attr.get(key) - if attr and hasattr(legacy_hub, attr): - return getattr(legacy_hub, attr) - return None - - def resolve( key: str, context: ServiceCenterContext, cache: Dict[str, Any], resolving: Set[str], - legacy_hub: Optional[Any] = None, ) -> Any: """ Resolve a service by key. Uses cache, resolves dependencies recursively. - Falls back to legacy_hub if service module cannot be loaded. + Raises KeyError if the service is not registered. """ cache_key = f"{_make_context_id(context)}_{key}" if cache_key in cache: @@ -82,59 +46,20 @@ def resolve( raise RuntimeError(f"Circular dependency detected for service: {key}") def get_service(dep_key: str) -> Any: - return resolve(dep_key, context, cache, resolving, legacy_hub) + return resolve(dep_key, context, cache, resolving) - # Try core first - if key in CORE_SERVICES: - spec = CORE_SERVICES[key] + spec = CORE_SERVICES.get(key) or IMPORTABLE_SERVICES.get(key) + if spec: + cls = _load_service_class(spec["module"], spec["class"]) + resolving.add(key) try: - cls = _load_service_class(spec["module"], spec["class"]) - resolving.add(key) - try: - for dep in spec.get("dependencies", []): - get_service(dep) - finally: - resolving.discard(key) - instance = cls(context, get_service) - cache[cache_key] = instance - return instance - except (ImportError, ModuleNotFoundError, AttributeError) as e: - logger.debug(f"Could not load core service '{key}' from service center: {e}") - if legacy_hub: - fallback = _get_from_legacy(legacy_hub, key) - if fallback is not None: - cache[cache_key] = fallback - return fallback - raise - - # Try importable - if key in IMPORTABLE_SERVICES: - spec = IMPORTABLE_SERVICES[key] - try: - cls = _load_service_class(spec["module"], spec["class"]) - resolving.add(key) - try: - for dep in spec.get("dependencies", []): - get_service(dep) - finally: - resolving.discard(key) - instance = cls(context, get_service) - cache[cache_key] = instance - return instance - except (ImportError, ModuleNotFoundError, AttributeError) as e: - logger.debug(f"Could not load importable service '{key}' from service center: {e}") - if legacy_hub: - fallback = _get_from_legacy(legacy_hub, key) - if fallback is not None: - cache[cache_key] = fallback - return fallback - raise - - if legacy_hub: - fallback = _get_from_legacy(legacy_hub, key) - if fallback is not None: - cache[cache_key] = fallback - return fallback + for dep in spec.get("dependencies", []): + get_service(dep) + finally: + resolving.discard(key) + instance = cls(context, get_service) + cache[cache_key] = instance + return instance raise KeyError(f"Unknown service: {key}") diff --git a/modules/serviceCenter/services/serviceAgent/__init__.py b/modules/serviceCenter/services/serviceAgent/__init__.py new file mode 100644 index 00000000..05d5452b --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""serviceAgent: AI Agent with ReAct loop and native function calling.""" diff --git a/modules/serviceCenter/services/serviceAgent/actionToolAdapter.py b/modules/serviceCenter/services/serviceAgent/actionToolAdapter.py new file mode 100644 index 00000000..c80ffdeb --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/actionToolAdapter.py @@ -0,0 +1,162 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""ActionToolAdapter: wraps existing workflow actions (dynamicMode=True) as agent tools.""" + +import logging +from typing import Dict, Any, List, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( + ToolDefinition, ToolResult +) +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry + +logger = logging.getLogger(__name__) + + +class ActionToolAdapter: + """Wraps existing Workflow-Actions as Agent-Tools. + + Iterates over discovered methods, finds actions with dynamicMode=True, + and registers them in the ToolRegistry with a compound name (method.action). + """ + + def __init__(self, actionExecutor): + self._actionExecutor = actionExecutor + self._registeredTools: List[str] = [] + + def registerAll(self, toolRegistry: ToolRegistry): + """Discover and register all dynamicMode actions as agent tools.""" + from modules.workflows.processing.shared.methodDiscovery import methods + + registered = 0 + for methodName, methodInfo in methods.items(): + if not methodName[0].isupper(): + continue + + shortName = methodName.replace("Method", "").lower() + methodInstance = methodInfo["instance"] + + for actionName, actionInfo in methodInfo["actions"].items(): + actionDef = methodInstance._actions.get(actionName) + if not actionDef or not getattr(actionDef, "dynamicMode", False): + continue + + compoundName = f"{shortName}.{actionName}" + toolDef = _buildToolDefinition(compoundName, actionDef, actionInfo) + + handler = _createDispatchHandler(self._actionExecutor, shortName, actionName) + toolRegistry.registerFromDefinition(toolDef, handler) + self._registeredTools.append(compoundName) + registered += 1 + + logger.info(f"ActionToolAdapter: registered {registered} tools from workflow actions") + + @property + def registeredTools(self) -> List[str]: + """Names of all tools registered by this adapter.""" + return list(self._registeredTools) + + +def _buildToolDefinition(compoundName: str, actionDef, actionInfo: Dict[str, Any]) -> ToolDefinition: + """Build a ToolDefinition from a WorkflowActionDefinition.""" + parameters = _convertParameterSchema(actionInfo.get("parameters", {})) + + return ToolDefinition( + name=compoundName, + description=actionDef.description or actionInfo.get("description", ""), + parameters=parameters, + readOnly=False + ) + + +def _convertParameterSchema(actionParams: Dict[str, Any]) -> Dict[str, Any]: + """Convert workflow action parameter schema to JSON Schema for tool definitions.""" + properties = {} + required = [] + + for paramName, paramInfo in actionParams.items(): + paramType = paramInfo.get("type", "str") if isinstance(paramInfo, dict) else "str" + paramDesc = paramInfo.get("description", "") if isinstance(paramInfo, dict) else "" + paramRequired = paramInfo.get("required", False) if isinstance(paramInfo, dict) else False + + jsonType = _pythonTypeToJsonType(paramType) + properties[paramName] = { + "type": jsonType, + "description": paramDesc + } + + if paramRequired: + required.append(paramName) + + return { + "type": "object", + "properties": properties, + "required": required + } + + +def _pythonTypeToJsonType(pythonType: str) -> str: + """Map Python type strings to JSON Schema types.""" + mapping = { + "str": "string", + "int": "integer", + "float": "number", + "bool": "boolean", + "list": "array", + "dict": "object", + "List[str]": "array", + "List[int]": "array", + "List[dict]": "array", + "Dict[str, Any]": "object", + } + return mapping.get(pythonType, "string") + + +def _createDispatchHandler(actionExecutor, methodName: str, actionName: str): + """Create an async handler that dispatches to the ActionExecutor.""" + async def _handler(args: Dict[str, Any], context: Dict[str, Any]) -> ToolResult: + try: + result = await actionExecutor.executeAction(methodName, actionName, args) + data = _formatActionResult(result) + return ToolResult( + toolCallId="", + toolName=f"{methodName}.{actionName}", + success=result.success, + data=data, + error=result.error + ) + except Exception as e: + logger.error(f"ActionToolAdapter dispatch failed for {methodName}.{actionName}: {e}") + return ToolResult( + toolCallId="", + toolName=f"{methodName}.{actionName}", + success=False, + error=str(e) + ) + return _handler + + +def _formatActionResult(result) -> str: + """Format an ActionResult into a text representation for the agent.""" + parts = [] + + if result.resultLabel: + parts.append(f"Result: {result.resultLabel}") + + if result.error: + parts.append(f"Error: {result.error}") + + if result.documents: + parts.append(f"Documents ({len(result.documents)}):") + for doc in result.documents: + docName = getattr(doc, "documentName", "unnamed") + docType = getattr(doc, "mimeType", "unknown") + parts.append(f" - {docName} ({docType})") + docData = getattr(doc, "documentData", None) + if docData and isinstance(docData, str) and len(docData) < 2000: + parts.append(f" Content: {docData[:2000]}") + + if not parts: + parts.append("Action completed successfully." if result.success else "Action failed.") + + return "\n".join(parts) diff --git a/modules/serviceCenter/services/serviceAgent/agentLoop.py b/modules/serviceCenter/services/serviceAgent/agentLoop.py new file mode 100644 index 00000000..17d603dd --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/agentLoop.py @@ -0,0 +1,406 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Agent loop: ReAct pattern with native function calling, budget control, and error handling.""" + +import asyncio +import logging +import time +import json +import re +from typing import List, Dict, Any, Optional, AsyncGenerator, Callable, Awaitable + +from modules.datamodels.datamodelAi import ( + AiCallRequest, AiCallOptions, AiCallResponse, OperationTypeEnum +) +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( + AgentState, AgentStatusEnum, AgentConfig, AgentEvent, AgentEventTypeEnum, + ToolCallRequest, ToolResult, ToolCallLog, AgentRoundLog, AgentTrace +) +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry +from modules.serviceCenter.services.serviceAgent.conversationManager import ( + ConversationManager, buildSystemPrompt +) +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +MAX_RETRIES_PER_TOOL = 3 +RETRY_BASE_DELAY_S = 1.0 + + +async def runAgentLoop( + prompt: str, + toolRegistry: ToolRegistry, + config: AgentConfig, + aiCallFn: Callable[[AiCallRequest], Awaitable[AiCallResponse]], + getWorkflowCostFn: Callable[[], Awaitable[float]], + workflowId: str, + userId: str = "", + featureInstanceId: str = "", + buildRagContextFn: Callable[..., Awaitable[str]] = None, + mandateId: str = "", + aiCallStreamFn: Callable = None, + userLanguage: str = "", +) -> AsyncGenerator[AgentEvent, None]: + """Run the agent loop. Yields AgentEvent for each step (SSE-ready). + + Args: + prompt: User prompt + toolRegistry: Registry with available tools + config: Agent configuration (maxRounds, maxCostCHF, etc.) + aiCallFn: Function to call the AI (wraps serviceAi.callAi with billing) + getWorkflowCostFn: Function to get current workflow cost + workflowId: Workflow ID for tracking + userId: User ID for tracing + featureInstanceId: Feature instance ID for tracing + buildRagContextFn: Optional async function to build RAG context before each round + mandateId: Mandate ID for RAG scoping + userLanguage: ISO 639-1 language code for agent responses + """ + state = AgentState(workflowId=workflowId, maxRounds=config.maxRounds) + trace = AgentTrace( + workflowId=workflowId, userId=userId, + featureInstanceId=featureInstanceId + ) + + tools = toolRegistry.getTools() + toolDefinitions = toolRegistry.formatToolsForFunctionCalling() + toolsText = toolRegistry.formatToolsForPrompt() + + systemPrompt = buildSystemPrompt(tools, toolsText, userLanguage=userLanguage) + conversation = ConversationManager(systemPrompt) + conversation.addUserMessage(prompt) + + while state.status == AgentStatusEnum.RUNNING and state.currentRound < state.maxRounds: + await asyncio.sleep(0) + state.currentRound += 1 + roundStartTime = time.time() + roundLog = AgentRoundLog(roundNumber=state.currentRound) + + # RAG context injection (before each round for fresh relevance) + if buildRagContextFn: + try: + latestUserMsg = "" + for msg in reversed(conversation.messages): + if msg.get("role") == "user": + latestUserMsg = msg.get("content", "") + break + ragContext = await buildRagContextFn( + currentPrompt=latestUserMsg or prompt, + workflowId=workflowId, + userId=userId, + featureInstanceId=featureInstanceId, + mandateId=mandateId, + ) + if ragContext: + conversation.injectRagContext(ragContext) + except Exception as ragErr: + logger.warning(f"RAG context injection failed (non-blocking): {ragErr}") + + # Budget check + budgetExceeded = await _checkBudget(config, getWorkflowCostFn) + if budgetExceeded: + state.status = AgentStatusEnum.BUDGET_EXCEEDED + state.abortReason = "Workflow cost budget exceeded" + yield AgentEvent( + type=AgentEventTypeEnum.FINAL, + content=_buildProgressSummary(state, "Budget exceeded. Here is the progress so far.") + ) + break + + logger.info(f"Agent round {state.currentRound}/{state.maxRounds} for workflow {workflowId} (tools={state.totalToolCalls}, cost={state.totalCostCHF:.4f})") + yield AgentEvent( + type=AgentEventTypeEnum.AGENT_PROGRESS, + data={ + "round": state.currentRound, + "maxRounds": state.maxRounds, + "totalAiCalls": state.totalAiCalls, + "totalToolCalls": state.totalToolCalls, + "costCHF": state.totalCostCHF + } + ) + + # Progressive summarization + if conversation.needsSummarization(state.currentRound): + async def _summarizeCall(summaryPrompt: str) -> str: + req = AiCallRequest( + prompt=summaryPrompt, + options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE) + ) + resp = await aiCallFn(req) + state.totalCostCHF += resp.priceCHF + state.totalAiCalls += 1 + return resp.content + + await conversation.summarize(state.currentRound, _summarizeCall) + + # AI call + aiRequest = AiCallRequest( + prompt="", + options=AiCallOptions( + operationType=OperationTypeEnum.AGENT, + temperature=config.temperature + ), + messages=conversation.messages, + tools=toolDefinitions + ) + + try: + aiResponse = None + streamedText = "" + isFirstChunkOfRound = True + + if aiCallStreamFn: + async for chunk in aiCallStreamFn(aiRequest): + if isinstance(chunk, str): + if isFirstChunkOfRound and state.currentRound > 1: + chunk = "\n\n" + chunk + isFirstChunkOfRound = False + elif isFirstChunkOfRound: + isFirstChunkOfRound = False + streamedText += chunk + yield AgentEvent(type=AgentEventTypeEnum.CHUNK, content=chunk) + else: + aiResponse = chunk + + if aiResponse is None: + raise RuntimeError("Stream ended without final AiCallResponse") + else: + aiResponse = await aiCallFn(aiRequest) + + except Exception as e: + logger.error(f"AI call failed in round {state.currentRound}: {e}", exc_info=True) + state.status = AgentStatusEnum.ERROR + state.abortReason = f"AI call error: {e}" + yield AgentEvent(type=AgentEventTypeEnum.ERROR, content=str(e)) + break + + state.totalAiCalls += 1 + state.totalCostCHF += aiResponse.priceCHF + state.totalProcessingTime += aiResponse.processingTime + roundLog.aiModel = aiResponse.modelName + roundLog.costCHF = aiResponse.priceCHF + + if aiResponse.errorCount > 0: + state.status = AgentStatusEnum.ERROR + state.abortReason = f"AI returned error: {aiResponse.content}" + yield AgentEvent(type=AgentEventTypeEnum.ERROR, content=aiResponse.content) + break + + # Parse response for tool calls + toolCalls = _parseToolCalls(aiResponse) + textContent = _extractTextContent(aiResponse) + + if textContent and not streamedText: + yield AgentEvent(type=AgentEventTypeEnum.MESSAGE, content=textContent) + + if not toolCalls: + state.status = AgentStatusEnum.COMPLETED + conversation.addAssistantMessage(aiResponse.content) + roundLog.durationMs = int((time.time() - roundStartTime) * 1000) + trace.rounds.append(roundLog) + yield AgentEvent(type=AgentEventTypeEnum.FINAL, content=textContent or aiResponse.content) + break + + # Add assistant message with tool calls to conversation + assistantToolCalls = _formatAssistantToolCalls(toolCalls) + conversation.addAssistantMessage(textContent or "", assistantToolCalls) + + # Execute tool calls + for tc in toolCalls: + yield AgentEvent( + type=AgentEventTypeEnum.TOOL_CALL, + data={"toolName": tc.name, "args": tc.args} + ) + + results = await _executeToolCalls(toolCalls, toolRegistry, { + "workflowId": workflowId, + "userId": userId, + "featureInstanceId": featureInstanceId, + "mandateId": mandateId, + }) + state.totalToolCalls += len(results) + + for result in results: + roundLog.toolCalls.append(ToolCallLog( + toolName=result.toolName, + args=next((tc.args for tc in toolCalls if tc.id == result.toolCallId), {}), + success=result.success, + durationMs=result.durationMs, + error=result.error + )) + if not result.success: + logger.warning(f"Tool '{result.toolName}' failed: {result.error}") + yield AgentEvent( + type=AgentEventTypeEnum.TOOL_RESULT, + data={ + "toolName": result.toolName, + "success": result.success, + "data": result.data[:500] if result.data else "", + "error": result.error + } + ) + if result.sideEvents: + for sideEvt in result.sideEvents: + evtType = sideEvt.get("type", "") + try: + evtEnum = AgentEventTypeEnum(evtType) + except (ValueError, KeyError): + continue + yield AgentEvent( + type=evtEnum, + data=sideEvt.get("data"), + content=sideEvt.get("content"), + ) + + # Add tool results to conversation + toolResultMessages = [ + {"toolCallId": r.toolCallId, "toolName": r.toolName, + "content": r.data if r.success else f"Error: {r.error}"} + for r in results + ] + conversation.addToolResults(toolResultMessages) + + roundLog.durationMs = int((time.time() - roundStartTime) * 1000) + trace.rounds.append(roundLog) + + # maxRounds reached + if state.currentRound >= state.maxRounds and state.status == AgentStatusEnum.RUNNING: + state.status = AgentStatusEnum.MAX_ROUNDS_REACHED + state.abortReason = f"Maximum rounds ({state.maxRounds}) reached" + yield AgentEvent( + type=AgentEventTypeEnum.FINAL, + content=_buildProgressSummary(state, "Maximum rounds reached.") + ) + + # Agent summary + trace.completedAt = getUtcTimestamp() + trace.status = state.status + trace.totalRounds = state.currentRound + trace.totalToolCalls = state.totalToolCalls + trace.totalCostCHF = state.totalCostCHF + trace.abortReason = state.abortReason + + yield AgentEvent( + type=AgentEventTypeEnum.AGENT_SUMMARY, + data={ + "rounds": state.currentRound, + "totalAiCalls": state.totalAiCalls, + "totalToolCalls": state.totalToolCalls, + "costCHF": round(state.totalCostCHF, 4), + "processingTime": round(state.totalProcessingTime, 2), + "status": state.status.value, + "abortReason": state.abortReason + } + ) + + +async def _checkBudget(config: AgentConfig, + getWorkflowCostFn: Callable[[], Awaitable[float]]) -> bool: + """Check if workflow budget is exceeded. Returns True if exceeded.""" + if config.maxCostCHF is None: + return False + try: + currentCost = await getWorkflowCostFn() + return currentCost > config.maxCostCHF + except Exception as e: + logger.warning(f"Could not check workflow cost: {e}") + return False + + +async def _executeToolCalls(toolCalls: List[ToolCallRequest], + toolRegistry: ToolRegistry, + context: Dict[str, Any]) -> List[ToolResult]: + """Execute tool calls: readOnly tools in parallel, others sequentially.""" + readOnlyCalls = [tc for tc in toolCalls if toolRegistry.isReadOnly(tc.name)] + writeCalls = [tc for tc in toolCalls if not toolRegistry.isReadOnly(tc.name)] + + results: Dict[str, ToolResult] = {} + + if readOnlyCalls: + readResults = await asyncio.gather(*[ + toolRegistry.dispatch(tc, context) for tc in readOnlyCalls + ]) + for tc, result in zip(readOnlyCalls, readResults): + results[tc.id] = result + + for tc in writeCalls: + results[tc.id] = await toolRegistry.dispatch(tc, context) + + return [results[tc.id] for tc in toolCalls] + + +def _parseToolCalls(aiResponse: AiCallResponse) -> List[ToolCallRequest]: + """Parse tool calls from AI response. Supports native function calling and text-based fallback.""" + toolCalls = [] + + # Native function calling: check response metadata + if hasattr(aiResponse, 'toolCalls') and aiResponse.toolCalls: + for tc in aiResponse.toolCalls: + rawArgs = tc["function"]["arguments"] + if isinstance(rawArgs, str): + rawArgs = rawArgs.strip() + try: + parsedArgs = json.loads(rawArgs) if rawArgs else {} + except json.JSONDecodeError: + logger.warning(f"Failed to parse tool args for '{tc['function']['name']}': {rawArgs[:200]}") + parsedArgs = {} + else: + parsedArgs = rawArgs if rawArgs else {} + toolCalls.append(ToolCallRequest( + id=tc.get("id", str(len(toolCalls))), + name=tc["function"]["name"], + args=parsedArgs, + )) + return toolCalls + + # Text-based fallback: parse ```tool_call blocks + content = aiResponse.content or "" + pattern = r"```tool_call\s*\n\s*tool:\s*(\S+)\s*\n\s*args:\s*(\{.*?\})\s*\n\s*```" + matches = re.finditer(pattern, content, re.DOTALL) + + for match in matches: + toolName = match.group(1).strip() + argsStr = match.group(2).strip() + try: + args = json.loads(argsStr) + except json.JSONDecodeError: + logger.warning(f"Failed to parse tool args for '{toolName}': {argsStr}") + args = {} + toolCalls.append(ToolCallRequest(name=toolName, args=args)) + + return toolCalls + + +def _extractTextContent(aiResponse: AiCallResponse) -> str: + """Extract text content from AI response, removing tool_call blocks.""" + content = aiResponse.content or "" + cleaned = re.sub(r"```tool_call\s*\n.*?\n\s*```", "", content, flags=re.DOTALL) + return cleaned.strip() + + +def _formatAssistantToolCalls(toolCalls: List[ToolCallRequest]) -> List[Dict[str, Any]]: + """Format tool calls for the conversation history (OpenAI tool_calls format).""" + return [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.args) + } + } + for tc in toolCalls + ] + + +def _buildProgressSummary(state: AgentState, reason: str) -> str: + """Build a human-readable summary of agent progress for graceful termination.""" + return ( + f"{reason}\n\n" + f"Progress after {state.currentRound} rounds:\n" + f"- AI calls: {state.totalAiCalls}\n" + f"- Tool calls: {state.totalToolCalls}\n" + f"- Cost: {state.totalCostCHF:.4f} CHF\n" + f"- Processing time: {state.totalProcessingTime:.1f}s" + ) diff --git a/modules/serviceCenter/services/serviceAgent/conversationManager.py b/modules/serviceCenter/services/serviceAgent/conversationManager.py new file mode 100644 index 00000000..a5a8d6ea --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/conversationManager.py @@ -0,0 +1,280 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Conversation manager for the Agent service. +Handles message history, context window management, and progressive summarization.""" + +import logging +from typing import List, Dict, Any, Optional + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolDefinition + +logger = logging.getLogger(__name__) + +FIRST_SUMMARY_ROUND = 4 +META_SUMMARY_ROUND = 7 +KEEP_RECENT_MESSAGES = 4 +MAX_ESTIMATED_TOKENS = 60000 + + +class ConversationManager: + """Manages the conversation history and context window for agent runs. + + Progressive summarization strategy: + - Rounds 1-3: full conversation retained + - Round 4+: older messages compressed into a running summary + - Round 7+: meta-summary replaces prior summaries + Supports RAG context injection before each round via injectRagContext.""" + + def __init__(self, systemPrompt: str): + self._messages: List[Dict[str, Any]] = [ + {"role": "system", "content": systemPrompt} + ] + self._summaries: List[Dict[str, Any]] = [] + self._lastSummarizedRound: int = 0 + self._ragContextInjected: bool = False + + @property + def messages(self) -> List[Dict[str, Any]]: + """Current messages for the next AI call (internal markers stripped).""" + return [ + {k: v for k, v in msg.items() if not k.startswith("_")} + for msg in self._messages + ] + + def addUserMessage(self, content: str): + """Add a user message.""" + self._messages.append({"role": "user", "content": content}) + + def addAssistantMessage(self, content: str, toolCalls: List[Dict[str, Any]] = None): + """Add an assistant message, optionally with tool calls.""" + msg: Dict[str, Any] = {"role": "assistant", "content": content} + if toolCalls: + msg["tool_calls"] = toolCalls + self._messages.append(msg) + + def addToolResults(self, results: List[Dict[str, Any]]): + """Add tool results to the conversation. + Each result: {toolCallId, toolName, content}.""" + for result in results: + self._messages.append({ + "role": "tool", + "tool_call_id": result["toolCallId"], + "content": result["content"] + }) + + def addToolResultsAsText(self, resultText: str): + """Add combined tool results as a user message (text-based fallback).""" + self._messages.append({ + "role": "user", + "content": f"Tool Results:\n{resultText}" + }) + + def injectRagContext(self, ragContext: str): + """Inject RAG context as a system message right after the main system prompt. + + Called before each agent round by the agent loop if KnowledgeService is available. + Replaces any previously injected RAG context to keep the context fresh.""" + if not ragContext: + return + + ragMessage = { + "role": "system", + "content": f"Relevant Knowledge (from indexed documents and workflow context):\n{ragContext}", + "_isRagContext": True, + } + + # Replace existing RAG message if present, otherwise insert after system prompt + for i, msg in enumerate(self._messages): + if msg.get("_isRagContext"): + self._messages[i] = ragMessage + self._ragContextInjected = True + return + + # Insert after the first system prompt + self._messages.insert(1, ragMessage) + self._ragContextInjected = True + + def getMessageCount(self) -> int: + """Get the number of messages (excluding system prompt).""" + return len(self._messages) - 1 + + def estimateTokenCount(self) -> int: + """Rough estimate of total tokens in the conversation (4 chars ≈ 1 token).""" + totalChars = sum(len(str(m.get("content", ""))) for m in self._messages) + return totalChars // 4 + + def needsSummarization(self, currentRound: int) -> bool: + """Check if progressive summarization should be triggered. + + Triggers: + - At round FIRST_SUMMARY_ROUND (4) if not yet summarized + - At round META_SUMMARY_ROUND (7) for meta-summary + - Every 5 rounds after that + - When estimated token count exceeds MAX_ESTIMATED_TOKENS + """ + if currentRound >= FIRST_SUMMARY_ROUND and self._lastSummarizedRound < currentRound: + if currentRound == FIRST_SUMMARY_ROUND or currentRound == META_SUMMARY_ROUND: + return True + if (currentRound - META_SUMMARY_ROUND) % 5 == 0 and currentRound > META_SUMMARY_ROUND: + return True + if self.estimateTokenCount() > MAX_ESTIMATED_TOKENS: + return True + return False + + async def summarize(self, currentRound: int, aiCallFn) -> Optional[str]: + """Perform progressive summarization of older messages. + + Rounds 1-3: full history retained, no summarization. + Round 4+: compress older messages into a running summary. + Round 7+: meta-summary that consolidates prior summaries. + """ + if currentRound < FIRST_SUMMARY_ROUND and self.estimateTokenCount() <= MAX_ESTIMATED_TOKENS: + return None + + systemMsgs = [m for m in self._messages if m.get("role") == "system"] + nonSystemMessages = [m for m in self._messages if m.get("role") != "system"] + + keepRecent = min(KEEP_RECENT_MESSAGES, len(nonSystemMessages)) + if len(nonSystemMessages) <= keepRecent + 1: + return None + + splitIdx = len(nonSystemMessages) - keepRecent + # Ensure the split doesn't orphan tool messages from their assistant. + # Walk backwards from splitIdx: if we're landing in the middle of a + # tool-call sequence (assistant+tool_calls → tool → tool …), include + # the entire sequence in recentMessages. + while splitIdx > 0 and nonSystemMessages[splitIdx].get("role") == "tool": + splitIdx -= 1 + # Also include the assistant message that triggered the tool calls. + if splitIdx > 0 and splitIdx < len(nonSystemMessages) and \ + nonSystemMessages[splitIdx].get("role") == "assistant" and \ + nonSystemMessages[splitIdx].get("tool_calls"): + pass # splitIdx already points at the assistant; keep it in recent + elif splitIdx == 0: + return None # nothing to summarize + + messagesToSummarize = nonSystemMessages[:splitIdx] + recentMessages = nonSystemMessages[splitIdx:] + + summaryInput = _formatMessagesForSummary(messagesToSummarize) + previousSummary = self._summaries[-1]["content"] if self._summaries else "" + + isMetaSummary = currentRound >= META_SUMMARY_ROUND and len(self._summaries) >= 2 + summaryPrompt = _buildSummaryPrompt(summaryInput, previousSummary, isMetaSummary) + + try: + summaryText = await aiCallFn(summaryPrompt) + except Exception as e: + logger.error(f"Progressive summarization failed: {e}") + return None + + self._summaries.append({ + "round": currentRound, + "content": summaryText, + "isMeta": isMetaSummary, + }) + self._lastSummarizedRound = currentRound + + mainSystem = systemMsgs[0] if systemMsgs else {"role": "system", "content": ""} + ragMessages = [m for m in systemMsgs if m.get("_isRagContext")] + + self._messages = [ + mainSystem, + *ragMessages, + {"role": "system", "content": f"Conversation Summary (rounds 1-{currentRound - keepRecent}):\n{summaryText}"}, + *recentMessages, + ] + + logger.info( + f"Progressive summarization at round {currentRound}: " + f"compressed {len(messagesToSummarize)} messages into " + f"{'meta-' if isMetaSummary else ''}summary" + ) + return summaryText + + +def _formatMessagesForSummary(messages: List[Dict[str, Any]]) -> str: + """Format messages into a text block for summarization.""" + parts = [] + for msg in messages: + role = msg.get("role", "unknown") + content = msg.get("content", "") + if role == "tool": + toolName = msg.get("tool_call_id", "tool") + parts.append(f"[Tool Result ({toolName})]:\n{content}") + elif role == "assistant" and msg.get("tool_calls"): + calls = msg["tool_calls"] + callNames = [c.get("function", {}).get("name", "?") for c in calls] + parts.append(f"[Assistant → Tool Calls: {', '.join(callNames)}]") + if content: + parts.append(f"[Assistant]: {content}") + else: + parts.append(f"[{role.capitalize()}]: {content}") + return "\n\n".join(parts) + + +def _buildSummaryPrompt(messagesText: str, previousSummary: str, isMetaSummary: bool = False) -> str: + """Build the prompt for progressive summarization.""" + if isMetaSummary: + prompt = ( + "Create a comprehensive meta-summary consolidating the previous summary " + "and the new messages. Preserve all key facts, decisions, entities (names, " + "numbers, dates), tool results, and action outcomes. Be concise but complete.\n\n" + ) + else: + prompt = ( + "Summarize the following conversation concisely. Preserve all key facts, " + "decisions, entities (names, numbers, dates), and tool results. " + "Do not lose any important information.\n\n" + ) + if previousSummary: + prompt += f"Previous Summary:\n{previousSummary}\n\n" + prompt += f"New Messages to Summarize:\n{messagesText}\n\nProvide a concise, factual summary:" + return prompt + + +_LANGUAGE_NAMES = { + "de": "German", "en": "English", "fr": "French", "it": "Italian", + "es": "Spanish", "pt": "Portuguese", "nl": "Dutch", "ja": "Japanese", + "zh": "Chinese", "ko": "Korean", "ar": "Arabic", "ru": "Russian", +} + + +def buildSystemPrompt( + tools: List[ToolDefinition], + toolsFormatted: str = None, + userLanguage: str = "", +) -> str: + """Build the system prompt for the agent. + + Args: + tools: Available tool definitions. + toolsFormatted: Pre-formatted tool descriptions for text-based fallback. + userLanguage: ISO 639-1 language code (e.g. "de", "en"). The agent will + respond in this language. + """ + langName = _LANGUAGE_NAMES.get(userLanguage, "") + langInstruction = ( + f"IMPORTANT: Always respond in {langName} ({userLanguage}). " + f"The user's language is {langName}. All your messages, explanations, " + f"and summaries MUST be in {langName}. " + f"Only use English for tool call arguments and technical identifiers.\n\n" + ) if langName else "" + + prompt = ( + f"{langInstruction}" + "You are an AI agent with access to tools. " + "Use the provided tools to accomplish the user's task. " + "Think step by step. Call tools when you need information or need to perform actions. " + "When you have enough information to answer, respond directly without calling tools.\n\n" + ) + if toolsFormatted: + prompt += f"Available Tools:\n{toolsFormatted}\n\n" + prompt += ( + "To call a tool, use this format:\n" + "```tool_call\n" + "tool: \n" + 'args: {"param": "value"}\n' + "```\n\n" + ) + return prompt diff --git a/modules/serviceCenter/services/serviceAgent/datamodelAgent.py b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py new file mode 100644 index 00000000..b786b550 --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py @@ -0,0 +1,132 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Data models for the Agent service.""" + +from typing import List, Dict, Any, Optional +from enum import Enum +from pydantic import BaseModel, Field +from modules.shared.timeUtils import getUtcTimestamp +import uuid + + +class AgentStatusEnum(str, Enum): + RUNNING = "running" + COMPLETED = "completed" + MAX_ROUNDS_REACHED = "maxRoundsReached" + BUDGET_EXCEEDED = "budgetExceeded" + ERROR = "error" + STOPPED = "stopped" + + +class AgentEventTypeEnum(str, Enum): + MESSAGE = "message" + CHUNK = "chunk" + TOOL_CALL = "toolCall" + TOOL_RESULT = "toolResult" + AGENT_PROGRESS = "agentProgress" + AGENT_SUMMARY = "agentSummary" + FILE_CREATED = "fileCreated" + DATA_SOURCE_ACCESS = "dataSourceAccess" + VOICE_RESPONSE = "voiceResponse" + FINAL = "final" + ERROR = "error" + + +class ToolDefinition(BaseModel): + """Schema for a tool available to the agent.""" + name: str = Field(description="Unique tool name") + description: str = Field(description="What this tool does") + parameters: Dict[str, Any] = Field( + default_factory=dict, + description="JSON Schema for tool parameters" + ) + readOnly: bool = Field( + default=False, + description="If True, tool can run in parallel with other readOnly tools" + ) + featureType: Optional[str] = Field( + default=None, + description="Feature scope for this tool (None = available to all)" + ) + + +class ToolCallRequest(BaseModel): + """A tool call requested by the AI model.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + name: str + args: Dict[str, Any] = Field(default_factory=dict) + + +class ToolResult(BaseModel): + """Result from executing a tool.""" + toolCallId: str + toolName: str + success: bool = True + data: str = "" + error: Optional[str] = None + durationMs: int = 0 + sideEvents: Optional[List[Dict[str, Any]]] = None + + +class AgentEvent(BaseModel): + """Event emitted during agent execution for SSE streaming.""" + type: AgentEventTypeEnum + content: Optional[str] = None + data: Optional[Dict[str, Any]] = None + + +class AgentConfig(BaseModel): + """Configuration for an agent run.""" + maxRounds: int = Field(default=25, ge=1, le=100) + maxCostCHF: Optional[float] = Field(default=None, ge=0.0) + entityCacheEnabled: bool = Field(default=False) + toolSet: str = Field(default="core") + temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) + + +class AgentState(BaseModel): + """Tracks state across an agent loop execution.""" + workflowId: str + currentRound: int = 0 + maxRounds: int = 25 + totalAiCalls: int = 0 + totalToolCalls: int = 0 + totalCostCHF: float = 0.0 + totalProcessingTime: float = 0.0 + status: AgentStatusEnum = AgentStatusEnum.RUNNING + abortReason: Optional[str] = None + + +class ToolCallLog(BaseModel): + """Log of a single tool call for observability.""" + toolName: str + args: Dict[str, Any] = Field(default_factory=dict) + success: bool = True + durationMs: int = 0 + error: Optional[str] = None + + +class AgentRoundLog(BaseModel): + """Log of a single agent round for observability.""" + roundNumber: int + aiModel: str = "" + inputTokens: int = 0 + outputTokens: int = 0 + costCHF: float = 0.0 + toolCalls: List[ToolCallLog] = Field(default_factory=list) + durationMs: int = 0 + + +class AgentTrace(BaseModel): + """Full trace of an agent workflow for observability.""" + workflowId: str + userId: str = "" + featureInstanceId: str = "" + startedAt: float = Field(default_factory=getUtcTimestamp) + completedAt: Optional[float] = None + status: AgentStatusEnum = AgentStatusEnum.RUNNING + totalRounds: int = 0 + totalToolCalls: int = 0 + totalCostCHF: float = 0.0 + abortReason: Optional[str] = None + rounds: List[AgentRoundLog] = Field(default_factory=list) diff --git a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py new file mode 100644 index 00000000..ac44f86e --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py @@ -0,0 +1,1983 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Agent service: entry point for running AI agents with tool use.""" + +import logging +from typing import Any, Callable, Dict, List, Optional, AsyncGenerator + +from modules.datamodels.datamodelAi import ( + AiCallRequest, AiCallOptions, AiCallResponse, OperationTypeEnum +) +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( + AgentConfig, AgentEvent, AgentEventTypeEnum +) +from modules.serviceCenter.services.serviceAgent.toolRegistry import ToolRegistry +from modules.serviceCenter.services.serviceAgent.agentLoop import runAgentLoop +from modules.serviceCenter.services.serviceAgent.actionToolAdapter import ActionToolAdapter +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import ( + getService as getBillingService, + InsufficientBalanceException, + BillingContextError +) + +logger = logging.getLogger(__name__) + +_MAX_TOOL_RESULT_CHARS = 50_000 + +_BINARY_SIGNATURES = (b"%PDF", b"\x89PNG", b"\xff\xd8\xff", b"GIF8", b"PK\x03\x04", b"Rar!", b"\x1f\x8b") + + +def _looksLikeBinary(data: bytes, sampleSize: int = 1024) -> bool: + """Detect binary content by checking for magic bytes and non-printable char ratio.""" + if any(data[:8].startswith(sig) for sig in _BINARY_SIGNATURES): + return True + sample = data[:sampleSize] + if not sample: + return False + nonPrintable = sum(1 for b in sample if b < 0x09 or (0x0E <= b < 0x20 and b != 0x1B)) + return nonPrintable / len(sample) > 0.10 + + +class _ServicesAdapter: + """Adapter providing service access from (context, get_service).""" + + def __init__(self, context, getService: Callable[[str], Any]): + self._context = context + self._getService = getService + self.user = context.user + self.mandateId = context.mandate_id + self.featureInstanceId = context.feature_instance_id + + @property + def workflow(self): + return self._context.workflow + + @property + def ai(self): + return self._getService("ai") + + @property + def chat(self): + return self._getService("chat") + + @property + def streaming(self): + return self._getService("streaming") + + @property + def billing(self): + return self._getService("billing") + + @property + def utils(self): + return self._getService("utils") + + @property + def extraction(self): + return self._getService("extraction") + + def getService(self, name: str): + """Access any service by name.""" + return self._getService(name) + + @property + def featureCode(self) -> Optional[str]: + w = self.workflow + if w and hasattr(w, "feature") and w.feature: + return getattr(w.feature, "code", None) + return getattr(w, "featureCode", None) if w else None + + +class AgentService: + """Service for running AI agents with ReAct loop and tool use. + + Registered as IMPORTABLE_SERVICE with objectKey 'service.agent'. + Uses serviceAi for model selection/billing, streaming for SSE events. + """ + + def __init__(self, context, get_service: Callable[[str], Any]): + self._context = context + self._getService = get_service + self.services = _ServicesAdapter(context, get_service) + + async def runAgent( + self, + prompt: str, + fileIds: List[str] = None, + config: AgentConfig = None, + toolSet: str = "core", + workflowId: str = None, + additionalTools: List[Dict[str, Any]] = None, + userLanguage: str = "", + ) -> AsyncGenerator[AgentEvent, None]: + """Run an agent with the given prompt and tools. + + Args: + prompt: User prompt + fileIds: Optional list of file IDs to include as context + config: Agent configuration + toolSet: Which tool set to activate + workflowId: Workflow ID for tracking and billing + additionalTools: Extra tool definitions to register dynamically + userLanguage: ISO 639-1 language code; falls back to user.language from profile + + Yields: + AgentEvent for each step (SSE-ready) + """ + if config is None: + config = AgentConfig(toolSet=toolSet) + + if workflowId is None: + workflowId = getattr(self.services.workflow, "id", "unknown") if self.services.workflow else "unknown" + + resolvedLanguage = userLanguage or getattr(self.services.user, "language", "") or "de" + + enrichedPrompt = await self._enrichPromptWithFiles(prompt, fileIds) + + toolRegistry = self._buildToolRegistry(config) + + aiCallFn = self._createAiCallFn() + aiCallStreamFn = self._createAiCallStreamFn() + getWorkflowCostFn = self._createGetWorkflowCostFn(workflowId) + buildRagContextFn = self._createBuildRagContextFn() + + async for event in runAgentLoop( + prompt=enrichedPrompt, + toolRegistry=toolRegistry, + config=config, + aiCallFn=aiCallFn, + getWorkflowCostFn=getWorkflowCostFn, + workflowId=workflowId, + userId=self.services.user.id if self.services.user else "", + featureInstanceId=self.services.featureInstanceId or "", + buildRagContextFn=buildRagContextFn, + mandateId=self.services.mandateId or "", + aiCallStreamFn=aiCallStreamFn, + userLanguage=resolvedLanguage, + ): + if event.type == AgentEventTypeEnum.AGENT_SUMMARY: + await self._persistTrace(workflowId, event.data or {}) + logger.debug(f"runAgent yielding event type={event.type}") + yield event + logger.info(f"runAgent loop completed for workflow {workflowId}") + + async def _enrichPromptWithFiles(self, prompt: str, fileIds: List[str] = None) -> str: + """Resolve file metadata + FileContentIndex for attached fileIds and prepend to prompt. + + The FileContentIndex is produced by the upload pipeline (AI-free extraction) + and tells the agent exactly which content objects (text, images, tables, etc.) + exist inside a file, so the agent can work with them directly via tools. + """ + if not fileIds: + return prompt + try: + chatService = self.services.chat + knowledgeDb = None + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as _getKnowledgeInterface + knowledgeDb = _getKnowledgeInterface() + except Exception: + pass + + fileDescriptions = [] + for fid in fileIds: + try: + info = chatService.getFileInfo(fid) + fileName = info.get("fileName", fid) if info else fid + mimeType = info.get("mimeType", "unknown") if info else "unknown" + fileSize = info.get("size", "?") if info else "?" + + desc = f"### File: {fileName}\n - id: {fid}\n - type: {mimeType}\n - size: {fileSize} bytes" + + if knowledgeDb: + contentIndex = knowledgeDb.getFileContentIndex(fid) + if contentIndex: + structure = contentIndex.get("structure", {}) + totalObjects = contentIndex.get("totalObjects", 0) + desc += f"\n - indexed: yes ({totalObjects} content objects)" + if structure: + structParts = [] + for key, val in structure.items(): + if isinstance(val, (int, str)): + structParts.append(f"{key}: {val}") + if structParts: + desc += f"\n - structure: {', '.join(structParts)}" + + objectSummary = contentIndex.get("objectSummary", []) + if objectSummary: + desc += "\n - content objects:" + for obj in objectSummary[:20]: + objType = obj.get("type", obj.get("contentType", "?")) + objRef = obj.get("ref", {}) + objLabel = objRef.get("location", "") if isinstance(objRef, dict) else "" + objId = obj.get("id", obj.get("contentObjectId", "")) + desc += f"\n * [{objType}] {objLabel}" + (f" (id: {objId})" if objId else "") + if len(objectSummary) > 20: + desc += f"\n ... and {len(objectSummary) - 20} more objects" + else: + desc += "\n - indexed: no (use readFile to trigger extraction)" + + fileDescriptions.append(desc) + except Exception: + fileDescriptions.append(f"### File id: {fid}") + + if fileDescriptions: + header = ( + "## Attached Files\n" + "These files have been uploaded and processed through the extraction pipeline.\n" + "Use `readFile(fileId)` to read text content, `readContentObjects(fileId)` for structured access, " + "or `describeImage(fileId)` for image analysis.\n" + "When generating documents with `renderDocument`, embed images using `![alt text](file:fileId)` in the markdown content.\n\n" + ) + header += "\n\n".join(fileDescriptions) + return f"{header}\n\n---\n\nUser request: {prompt}" + except Exception as e: + logger.warning(f"Could not enrich prompt with file metadata: {e}") + return prompt + + def _buildToolRegistry(self, config: AgentConfig) -> ToolRegistry: + """Build a tool registry with core tools and ActionToolAdapter tools.""" + registry = ToolRegistry() + + _registerCoreTools(registry, self.services) + + try: + from modules.workflows.processing.core.actionExecutor import ActionExecutor + actionExecutor = ActionExecutor(self.services) + adapter = ActionToolAdapter(actionExecutor) + adapter.registerAll(registry) + except Exception as e: + logger.warning(f"Could not register action tools: {e}") + + return registry + + async def _persistTrace(self, workflowId: str, summaryData: Dict[str, Any]): + """Persist the agent trace as a workflow memory entry in the knowledge store.""" + try: + knowledgeService = self._getService("knowledge") + userId = self.services.user.id if self.services.user else "" + featureInstanceId = self.services.featureInstanceId or "" + + import json + traceValue = json.dumps(summaryData, default=str) + + await knowledgeService.storeEntity( + workflowId=workflowId, + userId=userId, + featureInstanceId=featureInstanceId, + key="_agentTrace", + value=traceValue, + source="agent", + ) + logger.info(f"Persisted agent trace for workflow {workflowId}") + except Exception as e: + logger.warning(f"Could not persist agent trace: {e}") + + def _createAiCallFn(self) -> Callable[[AiCallRequest], AiCallResponse]: + """Create the AI call function that wraps serviceAi with billing.""" + async def _aiCallFn(request: AiCallRequest) -> AiCallResponse: + aiService = self.services.ai + return await aiService.callAi(request) + return _aiCallFn + + def _createAiCallStreamFn(self): + """Create the streaming AI call function. Yields str deltas, then AiCallResponse.""" + async def _aiCallStreamFn(request: AiCallRequest): + aiService = self.services.ai + async for chunk in aiService.callAiStream(request): + yield chunk + return _aiCallStreamFn + + def _createGetWorkflowCostFn(self, workflowId: str) -> Callable[[], float]: + """Create a function that returns the current workflow cost.""" + async def _getWorkflowCost() -> float: + try: + billingService = self.services.billing + return await billingService.getWorkflowCost(workflowId) + except Exception: + return 0.0 + return _getWorkflowCost + + def _createBuildRagContextFn(self): + """Create the RAG context builder function that delegates to KnowledgeService.""" + async def _buildRagContext( + currentPrompt: str, workflowId: str, userId: str, + featureInstanceId: str, mandateId: str, **kwargs + ) -> str: + try: + knowledgeService = self.services.getService("knowledge") + return await knowledgeService.buildAgentContext( + currentPrompt=currentPrompt, + workflowId=workflowId, + userId=userId, + featureInstanceId=featureInstanceId, + mandateId=mandateId, + ) + except Exception as e: + logger.debug(f"RAG context not available: {e}") + return "" + return _buildRagContext + + +def _registerCoreTools(registry: ToolRegistry, services): + """Register built-in core tools: file operations, search, and folder management.""" + from modules.serviceCenter.services.serviceAgent.datamodelAgent import ToolResult + + # ---- Read-only tools ---- + + async def _readFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="readFile", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None + + # 1) Knowledge Store: return already-extracted text chunks + if knowledgeService: + fileStatus = knowledgeService.getFileStatus(fileId) + if fileStatus == "indexed": + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + textChunks = [ + c for c in (chunks or []) + if c.get("contentType") != "image" and c.get("data") + ] + if textChunks: + assembled = "\n\n".join(c["data"] for c in textChunks) + if len(assembled) > _MAX_TOOL_RESULT_CHARS: + assembled = assembled[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(assembled)}]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=assembled, + ) + elif fileStatus in ("processing", "embedding", "extracted"): + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[File {fileId} is currently being processed (status: {fileStatus}). Try again shortly.]", + ) + + # 2) Not indexed yet: try on-demand extraction + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) + if not fileInfo: + return ToolResult(toolCallId="", toolName="readFile", success=True, data="File not found.") + + fileName = fileInfo.get("fileName", fileId) + mimeType = fileInfo.get("mimeType", "") + + _BINARY_TYPES = ("application/pdf", "image/", "application/vnd.", "application/zip", + "application/x-zip", "application/x-tar", "application/x-7z", + "application/msword", "application/octet-stream") + isBinary = any(mimeType.startswith(t) for t in _BINARY_TYPES) + + rawBytes = chatService.getFileData(fileId) + if not rawBytes: + return ToolResult(toolCallId="", toolName="readFile", success=True, data="File data not accessible.") + + if not isBinary: + isBinary = _looksLikeBinary(rawBytes) + + if isBinary: + try: + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry, ChunkerRegistry + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction + from modules.datamodels.datamodelExtraction import ExtractionOptions + + extracted = runExtraction( + ExtractorRegistry(), ChunkerRegistry(), + rawBytes, fileName, mimeType, ExtractionOptions(), + ) + + contentObjects = [] + for part in extracted.parts: + tg = (part.typeGroup or "").lower() + ct = "image" if tg == "image" else "text" + if not part.data or not part.data.strip(): + continue + contentObjects.append({ + "contentObjectId": part.id, + "contentType": ct, + "data": part.data, + "contextRef": { + "containerPath": fileName, + "location": part.label or "file", + **(part.metadata or {}), + }, + }) + + if contentObjects: + if knowledgeService: + try: + userId = context.get("userId", "") + await knowledgeService.indexFile( + fileId=fileId, fileName=fileName, mimeType=mimeType, + userId=userId, contentObjects=contentObjects, + ) + except Exception: + pass + + textParts = [o["data"] for o in contentObjects if o["contentType"] != "image"] + if textParts: + joined = "\n\n".join(textParts) + if len(joined) > _MAX_TOOL_RESULT_CHARS: + joined = joined[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(joined)}]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=joined, + ) + imgCount = sum(1 for o in contentObjects if o["contentType"] == "image") + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[Extracted {len(contentObjects)} content objects from '{fileName}' " + f"({imgCount} images, no readable text). " + f"Use describeImage(fileId='{fileId}') to analyze visual content.]", + ) + except Exception as extractErr: + logger.warning(f"readFile extraction failed for {fileId} ({fileName}): {extractErr}") + + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=f"[Binary file: '{fileName}', type={mimeType}, size={len(rawBytes)} bytes. " + f"Text extraction not available. Use describeImage for images.]", + ) + + # 3) Text file: decode raw bytes + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + text = rawBytes.decode(encoding) + if text.strip(): + if len(text) > _MAX_TOOL_RESULT_CHARS: + text = text[:_MAX_TOOL_RESULT_CHARS] + f"\n\n[Truncated – showing first {_MAX_TOOL_RESULT_CHARS} chars of {len(text)}]" + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data=text, + ) + except (UnicodeDecodeError, ValueError): + continue + + return ToolResult( + toolCallId="", toolName="readFile", success=True, + data="File is empty or could not be decoded.", + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="readFile", success=False, error=str(e)) + + async def _listFiles(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + files = chatService.listFiles( + folderId=args.get("folderId"), + tags=args.get("tags"), + search=args.get("search"), + ) + fileList = "\n".join( + f"- {f.get('fileName', 'unknown')} (id: {f.get('id', '?')}, " + f"type: {f.get('mimeType', '?')}, size: {f.get('fileSize', '?')}, " + f"tags: {f.get('tags', [])}, status: {f.get('status', 'n/a')})" + for f in files + ) if files else "No files found." + return ToolResult(toolCallId="", toolName="listFiles", success=True, data=fileList) + except Exception as e: + return ToolResult(toolCallId="", toolName="listFiles", success=False, error=str(e)) + + async def _searchFiles(args: Dict[str, Any], context: Dict[str, Any]): + query = args.get("query", "") + if not query: + return ToolResult(toolCallId="", toolName="searchFiles", success=False, error="query is required") + try: + chatService = services.chat + files = chatService.listFiles(search=query, tags=args.get("tags")) + fileList = "\n".join( + f"- {f.get('fileName', 'unknown')} (id: {f.get('id', '?')})" + for f in files + ) if files else "No files matching query." + return ToolResult(toolCallId="", toolName="searchFiles", success=True, data=fileList) + except Exception as e: + return ToolResult(toolCallId="", toolName="searchFiles", success=False, error=str(e)) + + async def _listFolders(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + folders = chatService.listFolders(parentId=args.get("parentId")) + folderList = "\n".join( + f"- {f.get('name', 'unnamed')} (id: {f.get('id', '?')})" + for f in folders + ) if folders else "No folders found." + return ToolResult(toolCallId="", toolName="listFolders", success=True, data=folderList) + except Exception as e: + return ToolResult(toolCallId="", toolName="listFolders", success=False, error=str(e)) + + async def _webSearch(args: Dict[str, Any], context: Dict[str, Any]): + query = args.get("query", "") + if not query: + return ToolResult(toolCallId="", toolName="webSearch", success=False, error="query is required") + try: + webService = services.getService("web") + result = await webService.performWebResearch( + prompt=query, + urls=[], + country=None, + language=args.get("language"), + ) + summary = result.get("summary", "") if isinstance(result, dict) else str(result) + return ToolResult( + toolCallId="", toolName="webSearch", success=True, + data=summary or str(result) + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="webSearch", success=False, error=str(e)) + + # ---- Write tools ---- + + async def _tagFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + tags = args.get("tags", []) + if not fileId: + return ToolResult(toolCallId="", toolName="tagFile", success=False, error="fileId is required") + try: + chatService = services.chat + chatService.interfaceDbComponent.updateFile(fileId, {"tags": tags}) + return ToolResult( + toolCallId="", toolName="tagFile", success=True, + data=f"Tags updated to {tags} for file {fileId}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="tagFile", success=False, error=str(e)) + + async def _moveFile(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + targetFolderId = args.get("targetFolderId") + if not fileId: + return ToolResult(toolCallId="", toolName="moveFile", success=False, error="fileId is required") + try: + chatService = services.chat + chatService.interfaceDbComponent.updateFile(fileId, {"folderId": targetFolderId}) + return ToolResult( + toolCallId="", toolName="moveFile", success=True, + data=f"File {fileId} moved to folder {targetFolderId or 'root'}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="moveFile", success=False, error=str(e)) + + async def _createFolder(args: Dict[str, Any], context: Dict[str, Any]): + name = args.get("name", "") + if not name: + return ToolResult(toolCallId="", toolName="createFolder", success=False, error="name is required") + try: + chatService = services.chat + folder = chatService.createFolder(name=name, parentId=args.get("parentId")) + return ToolResult( + toolCallId="", toolName="createFolder", success=True, + data=f"Folder '{name}' created (id: {folder.get('id', '?')})" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="createFolder", success=False, error=str(e)) + + async def _writeFile(args: Dict[str, Any], context: Dict[str, Any]): + name = args.get("name", "") + content = args.get("content", "") + if not name: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error="name is required") + try: + chatService = services.chat + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile( + content.encode("utf-8"), name + ) + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fileItem.id, {"featureInstanceId": fiId}) + if args.get("folderId"): + chatService.interfaceDbComponent.updateFile(fileItem.id, {"folderId": args["folderId"]}) + if args.get("tags"): + chatService.interfaceDbComponent.updateFile(fileItem.id, {"tags": args["tags"]}) + return ToolResult( + toolCallId="", toolName="writeFile", success=True, + data=f"File '{name}' created (id: {fileItem.id})", + sideEvents=[{ + "type": "fileCreated", + "data": { + "fileId": fileItem.id, + "fileName": name, + "mimeType": fileItem.mimeType, + "fileSize": fileItem.fileSize, + }, + }], + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="writeFile", success=False, error=str(e)) + + # ---- Register all tools ---- + + registry.register( + "readFile", _readFile, + description="Read the content of a file by its fileId.", + parameters={ + "type": "object", + "properties": {"fileId": {"type": "string", "description": "The file ID to read"}}, + "required": ["fileId"] + }, + readOnly=True + ) + + registry.register( + "listFiles", _listFiles, + description="List LOCAL workspace files (uploaded/generated). NOT for external data sources -- use browseDataSource instead.", + parameters={ + "type": "object", + "properties": { + "folderId": {"type": "string", "description": "Filter by folder ID"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Filter by tags (any match)"}, + "search": {"type": "string", "description": "Search in file names and descriptions"}, + } + }, + readOnly=True + ) + + registry.register( + "searchFiles", _searchFiles, + description="Search LOCAL workspace files by name, description, or tags. NOT for external data sources -- use searchDataSource instead.", + parameters={ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Additional tag filter"}, + }, + "required": ["query"] + }, + readOnly=True + ) + + registry.register( + "listFolders", _listFolders, + description="List LOCAL workspace folders. NOT for external data sources -- use browseDataSource instead.", + parameters={ + "type": "object", + "properties": { + "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, + } + }, + readOnly=True + ) + + registry.register( + "webSearch", _webSearch, + description="Search the web for information.", + parameters={ + "type": "object", + "properties": {"query": {"type": "string", "description": "Search query"}}, + "required": ["query"] + }, + readOnly=True + ) + + registry.register( + "tagFile", _tagFile, + description="Set tags on a file for categorization.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags to set"}, + }, + "required": ["fileId", "tags"] + }, + readOnly=False + ) + + registry.register( + "moveFile", _moveFile, + description="Move a file to a different folder.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID to move"}, + "targetFolderId": {"type": "string", "description": "Target folder ID (null for root)"}, + }, + "required": ["fileId"] + }, + readOnly=False + ) + + registry.register( + "createFolder", _createFolder, + description="Create a new file folder.", + parameters={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "Folder name"}, + "parentId": {"type": "string", "description": "Parent folder ID (omit for root)"}, + }, + "required": ["name"] + }, + readOnly=False + ) + + registry.register( + "writeFile", _writeFile, + description="Create a new file with text content.", + parameters={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "File name including extension"}, + "content": {"type": "string", "description": "File content as text"}, + "folderId": {"type": "string", "description": "Target folder ID"}, + "tags": {"type": "array", "items": {"type": "string"}, "description": "Tags"}, + }, + "required": ["name", "content"] + }, + readOnly=False + ) + + # ---- Connection tools (external data sources) ---- + + def _buildResolverDb(): + """Build a DB adapter that ConnectorResolver can use to load UserConnections. + interfaceDbApp has getUserConnectionById; ConnectorResolver expects getUserConnection.""" + chatService = services.chat + appIf = getattr(chatService, "interfaceDbApp", None) + if appIf and hasattr(appIf, "getUserConnectionById"): + class _Adapter: + def __init__(self, app): + self._app = app + def getUserConnection(self, connectionId: str): + return self._app.getUserConnectionById(connectionId) + return _Adapter(appIf) + return getattr(chatService, "interfaceDbComponent", None) + + async def _listConnections(args: Dict[str, Any], context: Dict[str, Any]): + try: + chatService = services.chat + connections = chatService.getUserConnections() if hasattr(chatService, "getUserConnections") else [] + if not connections: + return ToolResult(toolCallId="", toolName="listConnections", success=True, data="No connections available.") + lines = [] + for conn in connections: + connId = conn.get("id", "?") if isinstance(conn, dict) else getattr(conn, "id", "?") + authority = conn.get("authority", "?") if isinstance(conn, dict) else getattr(conn, "authority", "?") + email = conn.get("externalEmail", "") if isinstance(conn, dict) else getattr(conn, "externalEmail", "") + lines.append(f"- {authority} ({email}) id: {connId}") + return ToolResult(toolCallId="", toolName="listConnections", success=True, data="\n".join(lines)) + except Exception as e: + return ToolResult(toolCallId="", toolName="listConnections", success=False, error=str(e)) + + async def _externalBrowse(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + service = args.get("service", "") + path = args.get("path", "/") + if not connectionId or not service: + return ToolResult(toolCallId="", toolName="externalBrowse", success=False, error="connectionId and service are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.browse(path, filter=args.get("filter")) + entryLines = "\n".join( + f"- {'[DIR]' if e.isFolder else '[FILE]'} {e.name} ({e.size or '?'} bytes)" + for e in entries + ) if entries else "Empty directory." + return ToolResult(toolCallId="", toolName="externalBrowse", success=True, data=entryLines) + except Exception as e: + return ToolResult(toolCallId="", toolName="externalBrowse", success=False, error=str(e)) + + async def _externalDownload(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + service = args.get("service", "") + path = args.get("path", "") + if not connectionId or not service or not path: + return ToolResult(toolCallId="", toolName="externalDownload", success=False, error="connectionId, service, and path are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + fileBytes = await adapter.download(path) + if not fileBytes: + return ToolResult(toolCallId="", toolName="externalDownload", success=False, error="Download returned empty") + fileName = path.split("/")[-1] or "downloaded_file" + chatService = services.chat + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(fileBytes, fileName) + ext = fileName.rsplit(".", 1)[-1].lower() if "." in fileName else "" + hint = "Use readFile to read text content." if ext in ("doc", "docx", "txt", "csv", "json", "xml", "html", "md", "rtf", "odt", "xls", "xlsx", "pptx") else "Use readFile to access the content." + return ToolResult( + toolCallId="", toolName="externalDownload", success=True, + data=f"Downloaded '{fileName}' ({len(fileBytes)} bytes) → local file id: {fileItem.id}. {hint}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="externalDownload", success=False, error=str(e)) + + async def _externalUpload(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + service = args.get("service", "") + path = args.get("path", "") + fileId = args.get("fileId", "") + if not connectionId or not service or not path or not fileId: + return ToolResult(toolCallId="", toolName="externalUpload", success=False, error="connectionId, service, path, and fileId are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + chatService = services.chat + fileContent = chatService.getFileContent(fileId) + if not fileContent: + return ToolResult(toolCallId="", toolName="externalUpload", success=False, error="File not found") + fileData = fileContent.get("data", b"") if isinstance(fileContent, dict) else b"" + if isinstance(fileData, str): + fileData = fileData.encode("utf-8") + fileName = fileContent.get("fileName", "file") if isinstance(fileContent, dict) else "file" + result = await adapter.upload(path, fileData, fileName) + return ToolResult(toolCallId="", toolName="externalUpload", success=True, data=str(result)) + except Exception as e: + return ToolResult(toolCallId="", toolName="externalUpload", success=False, error=str(e)) + + async def _externalSearch(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + service = args.get("service", "") + query = args.get("query", "") + if not connectionId or not service or not query: + return ToolResult(toolCallId="", toolName="externalSearch", success=False, error="connectionId, service, and query are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.search(query, path=args.get("path")) + resultLines = "\n".join( + f"- {e.name} ({e.path})" + for e in entries + ) if entries else "No results found." + return ToolResult(toolCallId="", toolName="externalSearch", success=True, data=resultLines) + except Exception as e: + return ToolResult(toolCallId="", toolName="externalSearch", success=False, error=str(e)) + + async def _sendMail(args: Dict[str, Any], context: Dict[str, Any]): + connectionId = args.get("connectionId", "") + to = args.get("to", []) + subject = args.get("subject", "") + body = args.get("body", "") + if not connectionId or not to or not subject: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error="connectionId, to, and subject are required") + try: + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, "outlook") + if hasattr(adapter, "sendMail"): + result = await adapter.sendMail(to=to, subject=subject, body=body, cc=args.get("cc")) + return ToolResult(toolCallId="", toolName="sendMail", success=True, data=str(result)) + return ToolResult(toolCallId="", toolName="sendMail", success=False, error="Mail not supported by this adapter") + except Exception as e: + return ToolResult(toolCallId="", toolName="sendMail", success=False, error=str(e)) + + _connToolParams = { + "connectionId": {"type": "string", "description": "UserConnection ID"}, + "service": {"type": "string", "description": "Service name (sharepoint, outlook, drive, etc.)"}, + } + + registry.register( + "listConnections", _listConnections, + description="List available external connections and their services.", + parameters={"type": "object", "properties": {}}, + readOnly=True, + ) + + registry.register( + "externalBrowse", _externalBrowse, + description="Browse files in an external source by connectionId+service. For ATTACHED data sources, prefer browseDataSource instead.", + parameters={ + "type": "object", + "properties": { + **_connToolParams, + "path": {"type": "string", "description": "Path to browse"}, + "filter": {"type": "string", "description": "Filter pattern (e.g. '*.pdf')"}, + }, + "required": ["connectionId", "service"], + }, + readOnly=True, + ) + + registry.register( + "externalDownload", _externalDownload, + description="Download a file from an external source into local storage + auto-index.", + parameters={ + "type": "object", + "properties": { + **_connToolParams, + "path": {"type": "string", "description": "File path to download"}, + }, + "required": ["connectionId", "service", "path"], + }, + readOnly=False, + ) + + registry.register( + "externalUpload", _externalUpload, + description="Upload a local file to an external data source.", + parameters={ + "type": "object", + "properties": { + **_connToolParams, + "path": {"type": "string", "description": "Destination path"}, + "fileId": {"type": "string", "description": "Local file ID to upload"}, + }, + "required": ["connectionId", "service", "path", "fileId"], + }, + readOnly=False, + ) + + registry.register( + "externalSearch", _externalSearch, + description="Search files in an external source by connectionId+service. For ATTACHED data sources, prefer searchDataSource instead.", + parameters={ + "type": "object", + "properties": { + **_connToolParams, + "query": {"type": "string", "description": "Search query"}, + "path": {"type": "string", "description": "Scope to a specific path"}, + }, + "required": ["connectionId", "service", "query"], + }, + readOnly=True, + ) + + registry.register( + "sendMail", _sendMail, + description="Send an email via a connected mail service (Outlook, Gmail).", + parameters={ + "type": "object", + "properties": { + "connectionId": {"type": "string", "description": "UserConnection ID"}, + "to": {"type": "array", "items": {"type": "string"}, "description": "Recipient email addresses"}, + "subject": {"type": "string", "description": "Email subject"}, + "body": {"type": "string", "description": "Email body text"}, + "cc": {"type": "array", "items": {"type": "string"}, "description": "CC addresses"}, + }, + "required": ["connectionId", "to", "subject", "body"], + }, + readOnly=False, + ) + + # ---- DataSource convenience tools ---- + _SOURCE_TYPE_TO_SERVICE = { + "sharepointFolder": "sharepoint", + "onedriveFolder": "onedrive", + "outlookFolder": "outlook", + "googleDriveFolder": "drive", + "gmailFolder": "gmail", + "ftpFolder": "files", + } + + async def _resolveDataSource(dsId: str): + """Resolve a DataSource record and return (connectionId, service, path) or raise.""" + chatService = services.chat + ds = chatService.getDataSource(dsId) if hasattr(chatService, "getDataSource") else None + if not ds: + raise ValueError(f"DataSource '{dsId}' not found") + connectionId = ds.get("connectionId", "") + sourceType = ds.get("sourceType", "") + path = ds.get("path", "/") + label = ds.get("label", "") + service = _SOURCE_TYPE_TO_SERVICE.get(sourceType, sourceType) + if not connectionId: + raise ValueError(f"DataSource '{dsId}' has no connectionId") + logger.info(f"Resolved DataSource '{dsId}' ({label}): sourceType={sourceType}, service={service}, connectionId={connectionId}, path={path[:80]}") + return connectionId, service, path + + async def _browseDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + subPath = args.get("subPath", "") + if not dsId: + return ToolResult(toolCallId="", toolName="browseDataSource", success=False, error="dataSourceId is required") + try: + connectionId, service, basePath = await _resolveDataSource(dsId) + if subPath: + if subPath.startswith("/"): + browsePath = subPath + else: + browsePath = f"{basePath.rstrip('/')}/{subPath}" + else: + browsePath = basePath + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.browse(browsePath, filter=args.get("filter")) + if not entries: + return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data="Empty directory.") + lines = [] + for e in entries: + prefix = "[DIR]" if e.isFolder else "[FILE]" + sizeInfo = f" ({e.size} bytes)" if e.size else "" + lines.append(f"- {prefix} {e.name}{sizeInfo} path: {e.path}") + return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data="\n".join(lines)) + except Exception as e: + return ToolResult(toolCallId="", toolName="browseDataSource", success=False, error=str(e)) + + async def _searchDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + query = args.get("query", "") + if not dsId or not query: + return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error="dataSourceId and query are required") + try: + connectionId, service, basePath = await _resolveDataSource(dsId) + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + entries = await adapter.search(query, path=basePath) + if not entries: + return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data="No results found.") + lines = [f"- {e.name} (path: {e.path})" for e in entries] + return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data="\n".join(lines)) + except Exception as e: + return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error=str(e)) + + async def _downloadFromDataSource(args: Dict[str, Any], context: Dict[str, Any]): + dsId = args.get("dataSourceId", "") + filePath = args.get("filePath", "") + fileName = args.get("fileName", "") + if not dsId or not filePath: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="dataSourceId and filePath are required") + try: + connectionId, service, basePath = await _resolveDataSource(dsId) + fullPath = filePath if filePath.startswith("/") else f"{basePath.rstrip('/')}/{filePath}" + from modules.connectors.connectorResolver import ConnectorResolver + resolver = ConnectorResolver( + services.getService("security"), + _buildResolverDb(), + ) + adapter = await resolver.resolveService(connectionId, service) + fileBytes = await adapter.download(fullPath) + if not fileBytes: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error="Download returned empty") + if not fileName or "." not in fileName: + pathSegment = fullPath.split("/")[-1] or "downloaded_file" + fileName = fileName or pathSegment + if "." not in fileName: + try: + entries = await adapter.browse(basePath) + for entry in entries: + if getattr(entry, "path", "") == filePath or getattr(entry, "path", "").endswith(filePath): + if "." in entry.name: + fileName = entry.name + break + except Exception: + pass + if "." not in fileName: + import mimetypes as _mt + guessed = _mt.guess_type(f"file.{_mt.guess_extension('application/octet-stream') or ''}")[0] + if not guessed and fileBytes[:4] == b"%PDF": + fileName = f"{fileName}.pdf" + elif not guessed and fileBytes[:2] == b"PK": + fileName = f"{fileName}.zip" + chatService = services.chat + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(fileBytes, fileName) + ext = fileName.rsplit(".", 1)[-1].lower() if "." in fileName else "" + hint = "Use readFile to read the text content." if ext in ("doc", "docx", "txt", "csv", "json", "xml", "html", "md", "rtf", "odt", "xls", "xlsx", "pptx", "pdf") else "Use readFile to access the content." + return ToolResult( + toolCallId="", toolName="downloadFromDataSource", success=True, + data=f"Downloaded '{fileName}' ({len(fileBytes)} bytes) → local file id: {fileItem.id}. {hint}" + ) + except Exception as e: + return ToolResult(toolCallId="", toolName="downloadFromDataSource", success=False, error=str(e)) + + registry.register( + "browseDataSource", _browseDataSource, + description="Browse files AND folders in an ATTACHED data source by its dataSourceId. This is the PRIMARY tool for listing data source contents.", + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID (from the attached data sources in the prompt)"}, + "subPath": {"type": "string", "description": "Optional sub-path within the data source to browse"}, + "filter": {"type": "string", "description": "Optional filter pattern (e.g. '*.pdf')"}, + }, + "required": ["dataSourceId"], + }, + readOnly=True, + ) + + registry.register( + "searchDataSource", _searchDataSource, + description="Search for files within an attached data source by query.", + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID"}, + "query": {"type": "string", "description": "Search query"}, + }, + "required": ["dataSourceId", "query"], + }, + readOnly=True, + ) + + registry.register( + "downloadFromDataSource", _downloadFromDataSource, + description="Download a file from an attached data source into local storage. Returns the local file ID which can then be read with readFile. Always provide the fileName if known from the browse results.", + parameters={ + "type": "object", + "properties": { + "dataSourceId": {"type": "string", "description": "DataSource ID"}, + "filePath": {"type": "string", "description": "Path of the file to download (as returned by browseDataSource)"}, + "fileName": {"type": "string", "description": "Human-readable file name with extension (e.g. 'report.pdf'). Get this from browseDataSource results."}, + }, + "required": ["dataSourceId", "filePath"], + }, + readOnly=False, + ) + + # ---- Document tools (Smart Documents / Container Handling) ---- + + async def _browseContainer(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="browseContainer", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + index = knowledgeService.getFileContentIndex(fileId) + if not index: + return ToolResult(toolCallId="", toolName="browseContainer", success=True, data="No content index available for this file. It may not have been indexed yet.") + structure = index.get("structure", {}) if isinstance(index, dict) else {} + objectSummary = index.get("objectSummary", []) if isinstance(index, dict) else [] + totalObjects = index.get("totalObjects", 0) if isinstance(index, dict) else 0 + + result = f"File: {index.get('fileName', '?')} ({index.get('mimeType', '?')})\n" + result += f"Total content objects: {totalObjects}\n" + + sections = structure.get("sections", []) + if sections: + result += "\nSections:\n" + for s in sections: + result += f" [{s.get('id', '?')}] {s.get('title', 'Untitled')} (pages {s.get('startPage', '?')}-{s.get('endPage', '?')})\n" + + if structure.get("pageMap"): + pages = len(structure["pageMap"]) + result += f"\nPages: {pages}\n" + imgCount = structure.get("imageCount", 0) + tableCount = structure.get("tableCount", 0) + if imgCount: + result += f"Images: {imgCount}\n" + if tableCount: + result += f"Tables: {tableCount}\n" + + if structure.get("sheetMap"): + result += "\nSheets:\n" + for s in structure["sheetMap"]: + result += f" {s.get('sheetName', '?')} ({s.get('rows', '?')} rows x {s.get('columns', '?')} cols)\n" + + if structure.get("slideMap"): + result += "\nSlides:\n" + for s in structure["slideMap"]: + result += f" Slide {s.get('slideIndex', 0) + 1}: {s.get('title', '(no title)')}\n" + + return ToolResult(toolCallId="", toolName="browseContainer", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="browseContainer", success=False, error=str(e)) + + async def _readContentObjects(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + filterDict = {} + if args.get("pageIndex") is not None: + filterDict["pageIndex"] = args["pageIndex"] + if args.get("contentType"): + filterDict["contentType"] = args["contentType"] + if args.get("sectionId"): + filterDict["sectionId"] = args["sectionId"] + + objects = await knowledgeService.readContentObjects(fileId, filterDict) + if not objects: + return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data="No content objects found with the given filter.") + + result = f"Found {len(objects)} content objects:\n\n" + for obj in objects[:20]: + data = obj.get("data", "") + cType = obj.get("contentType", "?") + ref = obj.get("contextRef", {}) + location = ref.get("location", "") if isinstance(ref, dict) else "" + preview = data[:300] if cType == "text" else f"[{cType} data, {len(data)} chars]" + result += f"[{cType}] {location}: {preview}\n\n" + + if len(objects) > 20: + result += f"... and {len(objects) - 20} more objects" + + return ToolResult(toolCallId="", toolName="readContentObjects", success=True, data=result) + except Exception as e: + return ToolResult(toolCallId="", toolName="readContentObjects", success=False, error=str(e)) + + async def _extractContainerItem(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + containerPath = args.get("containerPath", "") + if not fileId or not containerPath: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error="fileId and containerPath are required") + try: + knowledgeService = services.getService("knowledge") + result = await knowledgeService.extractContainerItem(fileId, containerPath) + if result: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=True, data=str(result)) + return ToolResult(toolCallId="", toolName="extractContainerItem", success=True, data=f"On-demand extraction for '{containerPath}' queued.") + except Exception as e: + return ToolResult(toolCallId="", toolName="extractContainerItem", success=False, error=str(e)) + + async def _summarizeContent(args: Dict[str, Any], context: Dict[str, Any]): + fileId = args.get("fileId", "") + if not fileId: + return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error="fileId is required") + try: + knowledgeService = services.getService("knowledge") + filterDict = {} + if args.get("sectionId"): + filterDict["sectionId"] = args["sectionId"] + if args.get("pageIndex") is not None: + filterDict["pageIndex"] = args["pageIndex"] + if args.get("contentType"): + filterDict["contentType"] = args["contentType"] + + objects = await knowledgeService.readContentObjects(fileId, filterDict) + if not objects: + return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data="No content found to summarize.") + + textParts = [obj.get("data", "") for obj in objects if obj.get("contentType") != "image"] + combinedText = "\n\n".join(textParts)[:6000] + + aiService = services.ai + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + summaryRequest = AiCallRequest( + prompt=f"Summarize the following content concisely:\n\n{combinedText}", + options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE), + ) + response = await aiService.callAi(summaryRequest) + return ToolResult(toolCallId="", toolName="summarizeContent", success=True, data=response.content) + except Exception as e: + return ToolResult(toolCallId="", toolName="summarizeContent", success=False, error=str(e)) + + registry.register( + "browseContainer", _browseContainer, + description="Browse the structural index of a file/container (pages, sections, sheets, slides).", + parameters={ + "type": "object", + "properties": {"fileId": {"type": "string", "description": "The file ID to browse"}}, + "required": ["fileId"], + }, + readOnly=True, + ) + + registry.register( + "readContentObjects", _readContentObjects, + description="Read content objects from a file with optional filters (page, section, type).", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "pageIndex": {"type": "integer", "description": "Filter by page index"}, + "sectionId": {"type": "string", "description": "Filter by section ID"}, + "contentType": {"type": "string", "description": "Filter by content type (text, image, etc.)"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) + + registry.register( + "extractContainerItem", _extractContainerItem, + description="On-demand extraction of a specific item within a container (ZIP, nested file).", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The container file ID"}, + "containerPath": {"type": "string", "description": "Path within the container"}, + }, + "required": ["fileId", "containerPath"], + }, + readOnly=True, + ) + + registry.register( + "summarizeContent", _summarizeContent, + description="AI-powered summary of content objects from a file, optionally filtered.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID"}, + "sectionId": {"type": "string", "description": "Optional: summarize only this section"}, + "pageIndex": {"type": "integer", "description": "Optional: summarize only this page"}, + "contentType": {"type": "string", "description": "Optional: filter by content type"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) + + # ---- Vision tool ---- + + async def _describeImage(args: Dict[str, Any], context: Dict[str, Any]): + """Analyse an image using AI vision. Uses Knowledge Store chunks produced by Extractors.""" + fileId = args.get("fileId", "") + prompt = args.get("prompt", "Describe this image in detail. Extract all visible text, tables, and data.") + pageIndex = args.get("pageIndex") + + if not fileId: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error="fileId is required") + + try: + import base64 as _b64 + + imageData = None + mimeType = "image/png" + + knowledgeService = services.getService("knowledge") if hasattr(services, "getService") else None + + # 1) Knowledge Store: image chunks already produced by PdfExtractor / ImageExtractor + if knowledgeService: + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if pageIndex is not None: + imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] + if imageChunks: + imageData = imageChunks[0].get("data", "") + chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType") + if chunkMime: + mimeType = chunkMime + + # 2) File not yet indexed -> trigger extraction via ExtractionService, then retry + if not imageData and knowledgeService and not knowledgeService.isFileIndexed(fileId): + try: + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) + fileContent = chatService.getFileContent(fileId) + if fileContent and fileInfo: + rawData = fileContent.get("data", "") + if isinstance(rawData, str) and len(rawData) > 100: + rawBytes = _b64.b64decode(rawData) + elif isinstance(rawData, bytes): + rawBytes = rawData + else: + rawBytes = None + + if rawBytes: + from modules.serviceCenter.services.serviceExtraction.subRegistry import ExtractorRegistry + from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction + from modules.datamodels.datamodelExtraction import ExtractionOptions + + fileMime = fileInfo.get("mimeType", "application/octet-stream") + fileName = fileInfo.get("fileName", fileId) + extracted = runExtraction( + ExtractorRegistry(), None, + rawBytes, fileName, fileMime, ExtractionOptions(), + ) + + contentObjects = [] + for part in extracted.parts: + tg = (part.typeGroup or "").lower() + ct = "image" if tg == "image" else "text" + if not part.data or not part.data.strip(): + continue + contentObjects.append({ + "contentObjectId": part.id, + "contentType": ct, + "data": part.data, + "contextRef": {"containerPath": fileName, "location": part.label, **(part.metadata or {})}, + }) + + if contentObjects: + await knowledgeService.indexFile( + fileId=fileId, fileName=fileName, mimeType=fileMime, + userId=context.get("userId", ""), contentObjects=contentObjects, + ) + + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if pageIndex is not None: + imageChunks = [c for c in imageChunks if c.get("contextRef", {}).get("pageIndex") == pageIndex] + if imageChunks: + imageData = imageChunks[0].get("data", "") + except Exception as extractErr: + logger.warning(f"describeImage: on-demand extraction failed: {extractErr}") + + # 3) Direct image file (not a container) - use raw file data + if not imageData: + chatService = services.chat + fileContent = chatService.getFileContent(fileId) + if fileContent: + fileMimeType = fileContent.get("mimeType", "") + if fileMimeType.startswith("image/"): + imageData = fileContent.get("data", "") + mimeType = fileMimeType + + if not imageData: + chatService = services.chat + fileInfo = chatService.getFileInfo(fileId) if hasattr(chatService, "getFileInfo") else None + fileName = fileInfo.get("fileName", fileId) if fileInfo else fileId + fileMime = fileInfo.get("mimeType", "unknown") if fileInfo else "unknown" + return ToolResult(toolCallId="", toolName="describeImage", success=False, + error=f"No image data found in '{fileName}' (type: {fileMime}). " + f"This file likely contains text, not images. Use readFile(fileId=\"{fileId}\") to access its text content.") + + try: + rawHead = _b64.b64decode(imageData[:32]) + if rawHead[:3] == b"\xff\xd8\xff": + mimeType = "image/jpeg" + elif rawHead[:8] == b"\x89PNG\r\n\x1a\n": + mimeType = "image/png" + elif rawHead[:4] == b"GIF8": + mimeType = "image/gif" + elif rawHead[:4] == b"RIFF" and rawHead[8:12] == b"WEBP": + mimeType = "image/webp" + except Exception: + pass + dataUrl = f"data:{mimeType};base64,{imageData}" + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum as OTE + + visionRequest = AiCallRequest( + prompt=prompt, + options=AiCallOptions(operationType=OTE.IMAGE_ANALYSE), + messages=[{"role": "user", "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": dataUrl}}, + ]}], + ) + visionResponse = await services.ai.callAi(visionRequest) + + if visionResponse.errorCount > 0: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error=visionResponse.content) + return ToolResult(toolCallId="", toolName="describeImage", success=True, data=visionResponse.content) + + except Exception as e: + return ToolResult(toolCallId="", toolName="describeImage", success=False, error=str(e)) + + registry.register( + "describeImage", _describeImage, + description="Analyse an image using AI vision. Works with image files and images extracted from PDFs/DOCX/PPTX.", + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "The file ID containing the image or document with images"}, + "prompt": {"type": "string", "description": "What to look for in the image (default: describe everything)"}, + "pageIndex": {"type": "integer", "description": "Filter images by page index (0-based, for multi-page documents)"}, + }, + "required": ["fileId"], + }, + readOnly=True, + ) + + # ---- Document rendering tool ---- + + def _markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]: + """Convert markdown content to the standard document JSON format expected by renderers.""" + import re as _re + + sections = [] + order = 0 + lines = markdown.split("\n") + i = 0 + + def _nextId(): + nonlocal order + order += 1 + return f"s_{order}" + + while i < len(lines): + line = lines[i] + + # --- Headings --- + headingMatch = _re.match(r'^(#{1,6})\s+(.+)', line) + if headingMatch: + level = len(headingMatch.group(1)) + text = headingMatch.group(2).strip() + sections.append({ + "id": _nextId(), "content_type": "heading", "order": order, + "elements": [{"content": {"text": text, "level": level}}], + }) + i += 1 + continue + + # --- Fenced code blocks --- + codeMatch = _re.match(r'^```(\w*)', line) + if codeMatch: + lang = codeMatch.group(1) or "text" + codeLines = [] + i += 1 + while i < len(lines) and not lines[i].startswith("```"): + codeLines.append(lines[i]) + i += 1 + i += 1 + sections.append({ + "id": _nextId(), "content_type": "code_block", "order": order, + "elements": [{"content": {"code": "\n".join(codeLines), "language": lang}}], + }) + continue + + # --- Tables --- + tableMatch = _re.match(r'^\|(.+)\|$', line) + if tableMatch and (i + 1) < len(lines) and _re.match(r'^\|[\s\-:|]+\|$', lines[i + 1]): + headerCells = [c.strip() for c in tableMatch.group(1).split("|")] + i += 2 + rows = [] + while i < len(lines) and _re.match(r'^\|(.+)\|$', lines[i]): + rowCells = [c.strip() for c in lines[i][1:-1].split("|")] + rows.append(rowCells) + i += 1 + sections.append({ + "id": _nextId(), "content_type": "table", "order": order, + "elements": [{"content": {"headers": headerCells, "rows": rows}}], + }) + continue + + # --- Bullet / numbered lists --- + listMatch = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', line) + if listMatch: + isNumbered = bool(_re.match(r'\d+[.)]', listMatch.group(2))) + items = [] + while i < len(lines) and _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]): + m = _re.match(r'^(\s*)([-*+]|\d+[.)]) (.+)', lines[i]) + items.append({"text": m.group(3).strip()}) + i += 1 + sections.append({ + "id": _nextId(), "content_type": "bullet_list", "order": order, + "elements": [{"content": {"items": items, "list_type": "numbered" if isNumbered else "bullet"}}], + }) + continue + + # --- Empty lines (skip) --- + if not line.strip(): + i += 1 + continue + + # --- Images: ![alt](file:fileId) or ![alt](url) --- + imgMatch = _re.match(r'^!\[([^\]]*)\]\(([^)]+)\)', line) + if imgMatch: + altText = imgMatch.group(1).strip() or "Image" + src = imgMatch.group(2).strip() + fileId = "" + if src.startswith("file:"): + fileId = src[5:] + sections.append({ + "id": _nextId(), "content_type": "image", "order": order, + "elements": [{ + "content": { + "altText": altText, + "base64Data": "", + "_fileRef": fileId, + "_srcUrl": src if not fileId else "", + } + }], + }) + i += 1 + continue + + # --- Paragraph (collect consecutive non-empty lines) --- + paraLines = [] + while i < len(lines) and lines[i].strip() and not _re.match(r'^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )', lines[i]): + paraLines.append(lines[i]) + i += 1 + if paraLines: + sections.append({ + "id": _nextId(), "content_type": "paragraph", "order": order, + "elements": [{"content": {"text": " ".join(paraLines)}}], + }) + continue + + i += 1 + + if not sections: + sections.append({ + "id": _nextId(), "content_type": "paragraph", "order": order, + "elements": [{"content": {"text": markdown.strip() or "(empty)"}}], + }) + + return { + "metadata": { + "split_strategy": "single_document", + "source_documents": [], + "extraction_method": "agent_rendering", + "title": title, + "language": language, + }, + "documents": [{ + "id": "doc_1", + "title": title, + "sections": sections, + }], + } + + async def _renderDocument(args: Dict[str, Any], context: Dict[str, Any]): + """Render agent-produced markdown content into any document format via the RendererRegistry.""" + import re as _re + content = args.get("content", "") + outputFormat = args.get("outputFormat", "pdf") + title = args.get("title", "Document") + language = args.get("language", "de") + + if not content: + return ToolResult(toolCallId="", toolName="renderDocument", success=False, error="content is required") + + try: + structuredContent = _markdownToDocumentJson(content, title, language) + + # Resolve image file references (file:fileId) to base64 data from Knowledge Store + knowledgeService = None + try: + knowledgeService = services.getService("knowledge") + except Exception: + pass + resolvedImages = 0 + for doc in structuredContent.get("documents", []): + for section in doc.get("sections", []): + if section.get("content_type") != "image": + continue + for element in section.get("elements", []): + contentObj = element.get("content", {}) + fileRef = contentObj.get("_fileRef", "") + if not fileRef or contentObj.get("base64Data"): + continue + if knowledgeService: + chunks = knowledgeService._knowledgeDb.getContentChunks(fileRef) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if imageChunks: + contentObj["base64Data"] = imageChunks[0].get("data", "") + chunkMime = imageChunks[0].get("contextRef", {}).get("mimeType", "image/png") + contentObj["mimeType"] = chunkMime + resolvedImages += 1 + if not contentObj.get("base64Data"): + try: + rawBytes = services.chat.getFileData(fileRef) + if rawBytes: + import base64 as _b64 + contentObj["base64Data"] = _b64.b64encode(rawBytes).decode("ascii") + contentObj["mimeType"] = "image/png" + resolvedImages += 1 + except Exception: + pass + contentObj.pop("_fileRef", None) + contentObj.pop("_srcUrl", None) + + sectionCount = len(structuredContent.get("documents", [{}])[0].get("sections", [])) + logger.info(f"renderDocument: parsed {sectionCount} sections from markdown ({len(content)} chars), resolved {resolvedImages} image(s), format={outputFormat}") + + generationService = services.getService("generation") + documents = await generationService.renderReport( + extractedContent=structuredContent, + outputFormat=outputFormat, + language=language, + title=title, + userPrompt=content, + ) + + if not documents: + return ToolResult(toolCallId="", toolName="renderDocument", success=False, error="Rendering produced no output") + + savedFiles = [] + sideEvents = [] + chatService = services.chat + + sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "document" + + for doc in documents: + docData = doc.documentData if hasattr(doc, "documentData") else b"" + docName = doc.filename if hasattr(doc, "filename") else f"{sanitizedTitle}.{outputFormat}" + docMime = doc.mimeType if hasattr(doc, "mimeType") else "application/octet-stream" + + if not docName.lower().endswith(f".{outputFormat}"): + docName = f"{sanitizedTitle}.{outputFormat}" + + fileItem = None + if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): + fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) + else: + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) + + if fileItem: + fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) + savedFiles.append(f"- {docName} (id: {fid})") + sideEvents.append({ + "type": "fileCreated", + "data": { + "fileId": fid, + "fileName": docName, + "mimeType": docMime, + "fileSize": len(docData), + }, + }) + + result = f"Rendered {len(documents)} document(s):\n" + "\n".join(savedFiles) + return ToolResult(toolCallId="", toolName="renderDocument", success=True, data=result, sideEvents=sideEvents) + + except Exception as e: + logger.error(f"renderDocument failed: {e}") + return ToolResult(toolCallId="", toolName="renderDocument", success=False, error=str(e)) + + registry.register( + "renderDocument", _renderDocument, + description=( + "Render markdown content into a document file (PDF, DOCX, XLSX, PPTX, CSV, HTML, MD, JSON, TXT). " + "You write the full document content as markdown, then this tool converts and renders it. " + "To embed images from uploaded files, use markdown image syntax with the file ID: ![alt text](file:fileId). " + "The images will be resolved from the Knowledge Store and embedded in the output document." + ), + parameters={ + "type": "object", + "properties": { + "content": {"type": "string", "description": "Full document content as markdown (headings, tables, lists, code blocks, paragraphs, images via ![alt](file:fileId))"}, + "outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"}, + "title": {"type": "string", "description": "Document title", "default": "Document"}, + "language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"}, + }, + "required": ["content"], + }, + readOnly=False, + ) + + # ── textToSpeech tool ────────────────────────────────────────────── + def _stripMarkdownForTts(text: str) -> str: + """Strip markdown formatting so TTS reads clean speech text.""" + import re as _re + t = text + t = _re.sub(r'\*\*(.+?)\*\*', r'\1', t) + t = _re.sub(r'\*(.+?)\*', r'\1', t) + t = _re.sub(r'__(.+?)__', r'\1', t) + t = _re.sub(r'_(.+?)_', r'\1', t) + t = _re.sub(r'`[^`]+`', lambda m: m.group(0)[1:-1], t) + t = _re.sub(r'^#{1,6}\s*', '', t, flags=_re.MULTILINE) + t = _re.sub(r'^\s*[-*+]\s+', '', t, flags=_re.MULTILINE) + t = _re.sub(r'^\s*\d+\.\s+', '', t, flags=_re.MULTILINE) + t = _re.sub(r'\[(.+?)\]\(.+?\)', r'\1', t) + t = _re.sub(r'!\[.*?\]\(.*?\)', '', t) + t = _re.sub(r'\n{3,}', '\n\n', t) + return t.strip() + + async def _textToSpeech(args: Dict[str, Any], context: Dict[str, Any]): + """Convert text to speech using Google Cloud TTS, deliver audio via SSE.""" + import base64 as _b64 + text = args.get("text", "") + language = args.get("language", "auto") + voiceName = args.get("voiceName") + + if not text: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is required") + + cleanText = _stripMarkdownForTts(text) + if not cleanText: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="text is empty after stripping markdown") + + try: + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + mandateId = context.get("mandateId", "") + voiceInterface = getVoiceInterface(currentUser=None, mandateId=mandateId) + + _ISO_TO_BCP47 = { + "de": "de-DE", "en": "en-US", "fr": "fr-FR", "it": "it-IT", + "es": "es-ES", "pt": "pt-BR", "nl": "nl-NL", "pl": "pl-PL", + "ru": "ru-RU", "ja": "ja-JP", "zh": "zh-CN", "ko": "ko-KR", + "ar": "ar-XA", "hi": "hi-IN", "tr": "tr-TR", "sv": "sv-SE", + } + + if language == "auto": + try: + snippet = cleanText[:500] + detectResult = await voiceInterface.detectLanguage(snippet) + if detectResult and detectResult.get("success"): + detected = detectResult.get("language", "de") + language = _ISO_TO_BCP47.get(detected, detected) + if "-" not in language: + language = _ISO_TO_BCP47.get(language, f"{language}-{language.upper()}") + logger.info(f"textToSpeech: auto-detected language '{detected}' -> '{language}'") + else: + language = "de-DE" + except Exception as detectErr: + logger.warning(f"textToSpeech: language detection failed: {detectErr}, defaulting to de-DE") + language = "de-DE" + + if not voiceName: + try: + featureInstanceId = context.get("featureInstanceId", "") + userId = context.get("userId", "") + if featureInstanceId and userId: + dbMgmt = services.chat.interfaceDbApp if hasattr(services.chat, "interfaceDbApp") else None + if dbMgmt and hasattr(dbMgmt, "getVoiceSettings"): + vs = dbMgmt.getVoiceSettings(userId) + if vs: + voiceMap = {} + if hasattr(vs, "ttsVoiceMap") and vs.ttsVoiceMap: + voiceMap = vs.ttsVoiceMap if isinstance(vs.ttsVoiceMap, dict) else {} + if language in voiceMap: + voiceName = voiceMap[language].get("voiceName") if isinstance(voiceMap[language], dict) else voiceMap[language] + logger.info(f"textToSpeech: using configured voice '{voiceName}' for {language}") + elif hasattr(vs, "ttsVoice") and vs.ttsVoice and hasattr(vs, "ttsLanguage") and vs.ttsLanguage == language: + voiceName = vs.ttsVoice + except Exception as prefErr: + logger.debug(f"textToSpeech: could not load voice preferences: {prefErr}") + + ttsResult = await voiceInterface.textToSpeech( + text=cleanText, + languageCode=language, + voiceName=voiceName, + ) + + if not ttsResult or not ttsResult.get("success"): + errMsg = ttsResult.get("error", "TTS call failed") if ttsResult else "TTS returned None" + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=errMsg) + + audioContent = ttsResult.get("audioContent", "") + if not audioContent: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error="TTS returned no audio") + + if isinstance(audioContent, bytes): + audioB64 = _b64.b64encode(audioContent).decode("ascii") + elif isinstance(audioContent, str): + audioB64 = audioContent + else: + audioB64 = str(audioContent) + + audioFormat = ttsResult.get("audioFormat", "mp3") + charCount = len(cleanText) + usedVoice = voiceName or "default" + logger.info(f"textToSpeech: generated {audioFormat} audio for {charCount} chars, language={language}, voice={usedVoice}") + + return ToolResult( + toolCallId="", toolName="textToSpeech", success=True, + data=f"Audio generated ({charCount} characters, language={language}, voice={usedVoice}). Playing in chat.", + sideEvents=[{ + "type": "voiceResponse", + "data": { + "audio": audioB64, + "format": audioFormat, + "language": language, + "charCount": charCount, + }, + }], + ) + + except ImportError: + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, + error="Voice interface not available (missing dependency)") + except Exception as e: + logger.error(f"textToSpeech failed: {e}") + return ToolResult(toolCallId="", toolName="textToSpeech", success=False, error=str(e)) + + registry.register( + "textToSpeech", _textToSpeech, + description=( + "Convert text to speech audio. The audio is played directly in the chat. " + "Use this when the user asks you to read something aloud, narrate, or speak. " + "Language is auto-detected from the text content. You do NOT need to specify a language." + ), + parameters={ + "type": "object", + "properties": { + "text": {"type": "string", "description": "The text to convert to speech. Can include markdown (will be stripped automatically)."}, + "language": {"type": "string", "description": "BCP-47 language code (e.g. de-DE, en-US) or 'auto' for automatic detection", "default": "auto"}, + "voiceName": {"type": "string", "description": "Optional specific voice name. If omitted, uses the configured voice for the detected language."}, + }, + "required": ["text"], + }, + readOnly=False, + ) + + # ── generateImage tool ───────────────────────────────────────────── + + async def _generateImage(args: Dict[str, Any], context: Dict[str, Any]): + """Generate an image from a text prompt using AI (DALL-E).""" + import re as _re + + prompt = (args.get("prompt") or "").strip() + style = (args.get("style") or "").strip() or None + title = (args.get("title") or "").strip() or "Generated Image" + + if not prompt: + return ToolResult(toolCallId="", toolName="generateImage", success=False, error="prompt is required") + + try: + from modules.serviceCenter.services.serviceGeneration.paths.imagePath import ImageGenerationPath + + imagePath = ImageGenerationPath(services) + aiResponse = await imagePath.generateImages( + userPrompt=prompt, + count=1, + style=style, + format="png", + title=title, + ) + + if not aiResponse.documents: + return ToolResult(toolCallId="", toolName="generateImage", success=False, error="Image generation returned no image data") + + sideEvents = [] + savedFiles = [] + chatService = services.chat + sanitizedTitle = _re.sub(r'[^\w._-]', '_', title, flags=_re.UNICODE).strip('_') or "generated_image" + + for doc in aiResponse.documents: + docData = doc.documentData if hasattr(doc, "documentData") else b"" + docName = doc.documentName if hasattr(doc, "documentName") else f"{sanitizedTitle}.png" + docMime = doc.mimeType if hasattr(doc, "mimeType") else "image/png" + + if not docName.lower().endswith(".png"): + docName = f"{sanitizedTitle}.png" + + fileItem = None + if hasattr(chatService.interfaceDbComponent, "saveGeneratedFile"): + fileItem = chatService.interfaceDbComponent.saveGeneratedFile(docData, docName, docMime) + else: + fileItem, _ = chatService.interfaceDbComponent.saveUploadedFile(docData, docName) + + if fileItem: + fid = fileItem.id if hasattr(fileItem, "id") else fileItem.get("id", "?") + fiId = context.get("featureInstanceId") or (services.featureInstanceId if services else "") + if fiId: + chatService.interfaceDbComponent.updateFile(fid, {"featureInstanceId": fiId}) + savedFiles.append(f"- {docName} (id: {fid})") + sideEvents.append({ + "type": "fileCreated", + "data": { + "fileId": fid, + "fileName": docName, + "mimeType": docMime, + "fileSize": len(docData), + }, + }) + + result = f"Generated {len(aiResponse.documents)} image(s):\n" + "\n".join(savedFiles) + return ToolResult(toolCallId="", toolName="generateImage", success=True, data=result, sideEvents=sideEvents) + + except Exception as e: + logger.error(f"generateImage failed: {e}") + return ToolResult(toolCallId="", toolName="generateImage", success=False, error=str(e)) + + registry.register( + "generateImage", _generateImage, + description=( + "Generate an image from a text description using AI (DALL-E). " + "The generated image is saved as a file in the workspace. " + "Use this when the user asks to create, generate, draw, or design an image, illustration, icon, logo, diagram, or any visual content. " + "Provide a detailed, descriptive prompt for best results." + ), + parameters={ + "type": "object", + "properties": { + "prompt": {"type": "string", "description": "Detailed description of the image to generate. Be specific about subject, composition, colors, style, and mood."}, + "style": {"type": "string", "description": "Optional style modifier (e.g. 'photorealistic', 'watercolor', 'digital art', 'minimalist', 'sketch')"}, + "title": {"type": "string", "description": "Title/filename for the generated image", "default": "Generated Image"}, + }, + "required": ["prompt"], + }, + readOnly=False, + ) diff --git a/modules/serviceCenter/services/serviceAgent/toolRegistry.py b/modules/serviceCenter/services/serviceAgent/toolRegistry.py new file mode 100644 index 00000000..625001fb --- /dev/null +++ b/modules/serviceCenter/services/serviceAgent/toolRegistry.py @@ -0,0 +1,150 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Tool registry for the Agent service. Manages tool definitions and dispatch.""" + +import logging +import time +from typing import Dict, List, Any, Optional, Callable, Awaitable + +from modules.serviceCenter.services.serviceAgent.datamodelAgent import ( + ToolDefinition, ToolCallRequest, ToolResult +) + +logger = logging.getLogger(__name__) + + +class ToolRegistry: + """Registry for agent tools. Handles registration, lookup, and dispatch.""" + + def __init__(self): + self._tools: Dict[str, ToolDefinition] = {} + self._handlers: Dict[str, Callable[..., Awaitable[ToolResult]]] = {} + + def register(self, name: str, handler: Callable[..., Awaitable[ToolResult]], + description: str = "", parameters: Dict[str, Any] = None, + readOnly: bool = False, featureType: str = None): + """Register a tool with its handler function.""" + if name in self._tools: + logger.warning(f"Tool '{name}' already registered, overwriting") + + self._tools[name] = ToolDefinition( + name=name, + description=description, + parameters=parameters or {}, + readOnly=readOnly, + featureType=featureType + ) + self._handlers[name] = handler + logger.debug(f"Registered tool: {name} (readOnly={readOnly})") + + def registerFromDefinition(self, definition: ToolDefinition, + handler: Callable[..., Awaitable[ToolResult]]): + """Register a tool from a pre-built ToolDefinition.""" + self._tools[definition.name] = definition + self._handlers[definition.name] = handler + logger.debug(f"Registered tool: {definition.name} (readOnly={definition.readOnly})") + + def unregister(self, name: str): + """Remove a tool from the registry.""" + self._tools.pop(name, None) + self._handlers.pop(name, None) + + def getTools(self, toolSet: str = None, featureType: str = None) -> List[ToolDefinition]: + """Get available tools, optionally filtered by toolSet or featureType.""" + tools = list(self._tools.values()) + if featureType: + tools = [t for t in tools if t.featureType is None or t.featureType == featureType] + return tools + + def getToolNames(self) -> List[str]: + """Get names of all registered tools.""" + return list(self._tools.keys()) + + def getTool(self, name: str) -> Optional[ToolDefinition]: + """Get a single tool definition by name.""" + return self._tools.get(name) + + def isReadOnly(self, name: str) -> bool: + """Check if a tool is marked as readOnly.""" + tool = self._tools.get(name) + return tool.readOnly if tool else False + + def isValidTool(self, name: str) -> bool: + """Check if a tool name is valid (registered).""" + return name in self._tools + + async def dispatch(self, toolCall: ToolCallRequest, context: Dict[str, Any] = None) -> ToolResult: + """Execute a tool call and return the result.""" + startTime = time.time() + + if not self.isValidTool(toolCall.name): + return ToolResult( + toolCallId=toolCall.id, + toolName=toolCall.name, + success=False, + error=f"Unknown tool: '{toolCall.name}'. Available: {', '.join(self.getToolNames())}" + ) + + handler = self._handlers[toolCall.name] + argsSummary = ", ".join(f"{k}={str(v)[:80]}" for k, v in (toolCall.args or {}).items()) + logger.info(f"Tool dispatch: {toolCall.name}({argsSummary})") + try: + result = await handler(toolCall.args, context or {}) + durationMs = int((time.time() - startTime) * 1000) + + if isinstance(result, ToolResult): + result.toolCallId = toolCall.id + result.durationMs = durationMs + dataSummary = (result.data[:200] + "...") if result.data and len(result.data) > 200 else (result.data or "") + if result.success: + logger.info(f"Tool result: {toolCall.name} OK ({durationMs}ms) → {dataSummary}") + else: + logger.warning(f"Tool result: {toolCall.name} FAILED ({durationMs}ms) → {result.error}") + return result + + return ToolResult( + toolCallId=toolCall.id, + toolName=toolCall.name, + success=True, + data=str(result), + durationMs=durationMs + ) + + except Exception as e: + durationMs = int((time.time() - startTime) * 1000) + logger.error(f"Tool '{toolCall.name}' failed: {e}", exc_info=True) + return ToolResult( + toolCallId=toolCall.id, + toolName=toolCall.name, + success=False, + error=str(e), + durationMs=durationMs + ) + + def formatToolsForPrompt(self) -> str: + """Format all tools as text for system prompt (text-based fallback).""" + parts = [] + for tool in self._tools.values(): + paramStr = ", ".join( + f"{k}: {v}" for k, v in tool.parameters.items() + ) if tool.parameters else "none" + parts.append(f"- **{tool.name}**: {tool.description}\n Parameters: {{{paramStr}}}") + return "\n".join(parts) + + def formatToolsForFunctionCalling(self) -> List[Dict[str, Any]]: + """Format all tools as OpenAI-compatible function definitions for native function calling.""" + functions = [] + for tool in self._tools.values(): + functions.append({ + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": tool.parameters if tool.parameters else { + "type": "object", + "properties": {}, + "required": [] + } + } + }) + return functions diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index eff5671f..a24af9a9 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -64,6 +64,10 @@ class _ServicesAdapter: def interfaceDbChat(self): return self._get_service("chat").interfaceDbChat + @property + def interfaceDbComponent(self): + return self._get_service("chat").interfaceDbComponent + @property def featureCode(self) -> Optional[str]: w = self.workflow @@ -142,6 +146,8 @@ class AiService: 3. billingCallback on aiObjects: records one billing transaction per model call with exact provider + model name (set before AI call, invoked by _callWithModel) """ + await self.ensureAiObjectsInitialized() + # SPEECH_TEAMS: Dedicated pipeline, bypasses standard model selection if request.options and request.options.operationType == OperationTypeEnum.SPEECH_TEAMS: return await self._handleSpeechTeams(request) @@ -171,14 +177,27 @@ class AiService: else: response = await self.aiObjects.callWithTextContext(request) finally: - # Clear callback after call completes self.aiObjects.billingCallback = None - # Store workflow stats for analytics - self._storeAiCallStats(response, request) - return response + async def callAiStream(self, request: AiCallRequest): + """Streaming variant of callAi. Yields str deltas during generation, then final AiCallResponse.""" + await self.ensureAiObjectsInitialized() + self._preflightBillingCheck() + await self._checkBillingBeforeAiCall() + + effectiveProviders = self._calculateEffectiveProviders() + if effectiveProviders and request.options: + request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders}) + + self.aiObjects.billingCallback = self._createBillingCallback() + try: + async for chunk in self.aiObjects.callWithTextContextStream(request): + yield chunk + finally: + self.aiObjects.billingCallback = None + # ========================================================================= # SPEECH_TEAMS: Dedicated handler for Teams Meeting AI analysis # Bypasses standard model selection. Uses a fixed fast model. @@ -295,9 +314,6 @@ class AiService: except Exception as e: logger.error(f"BILLING: Failed to record billing for SPEECH_TEAMS: {e}") - # Store stats - self._storeAiCallStats(response, request) - logger.info(f"SPEECH_TEAMS call completed: model={model.name}, time={processingTime:.2f}s, cost={priceCHF:.4f} CHF") return response @@ -644,12 +660,12 @@ detectedIntent-Werte: billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) def _billingCallback(response) -> None: - """Record billing for a single AI model call.""" + """Record billing transaction with full AI call metadata.""" if not response or getattr(response, 'errorCount', 0) > 0: return - priceCHF = getattr(response, 'priceCHF', 0.0) - if not priceCHF or priceCHF <= 0: + basePriceCHF = getattr(response, 'priceCHF', 0.0) + if not basePriceCHF or basePriceCHF <= 0: return provider = getattr(response, 'provider', None) or 'unknown' @@ -657,20 +673,24 @@ detectedIntent-Werte: try: billingService.recordUsage( - priceCHF=priceCHF, + priceCHF=basePriceCHF, workflowId=workflowId, aicoreProvider=provider, aicoreModel=modelName, - description=f"AI: {modelName}" + description=f"AI: {modelName}", + processingTime=getattr(response, 'processingTime', None), + bytesSent=getattr(response, 'bytesSent', None), + bytesReceived=getattr(response, 'bytesReceived', None), + errorCount=getattr(response, 'errorCount', None) ) logger.debug( - f"Billed model call: {priceCHF:.4f} CHF, " + f"Billed model call: {basePriceCHF:.4f} CHF, " f"provider={provider}, model={modelName}, mandate={mandateId}" ) except Exception as e: logger.error( f"BILLING: Failed to record transaction! " - f"Cost={priceCHF:.4f} CHF, user={user.id}, mandate={mandateId}, " + f"Cost={basePriceCHF:.4f} CHF, user={user.id}, mandate={mandateId}, " f"provider={provider}, model={modelName}, error={e}" ) @@ -723,40 +743,6 @@ detectedIntent-Werte: logger.warning(f"Error calculating effective providers: {e}") return None - def _storeAiCallStats(self, response, request: AiCallRequest) -> None: - """Store workflow stats after an AI call. - - This method stores the AI call statistics (cost, processing time, bytes) - to the workflow stats collection for tracking and billing purposes. - - Args: - response: AiCallResponse with cost/timing data - request: Original AiCallRequest for context - """ - try: - # Skip if no workflow context - workflow = getattr(self.services, 'workflow', None) - if not workflow or not hasattr(workflow, 'id') or not workflow.id: - logger.debug("No workflow context - skipping stats storage") - return - - # Skip if response is an error - if not response or getattr(response, 'errorCount', 0) > 0: - logger.debug("Error response - skipping stats storage") - return - - # Determine process name from operation type - opType = getattr(request.options, 'operationType', 'unknown') if request.options else 'unknown' - process = f"ai.call.{opType}" - - # Store the stat - self.services.chat.storeWorkflowStat(workflow, response, process) - logger.debug(f"Stored AI call stat: {process}, cost={getattr(response, 'priceCHF', 0):.4f} CHF") - - except Exception as e: - # Log but don't fail - stats storage is not critical - logger.debug(f"Could not store AI call stat: {str(e)}") - async def ensureAiObjectsInitialized(self): """Ensure aiObjects is initialized and submodules are ready.""" if self.aiObjects is None: @@ -766,17 +752,17 @@ detectedIntent-Werte: self._initializeSubmodules() @classmethod - async def create(cls, legacy_services) -> "AiService": - """Create AiService from legacy Services hub. For backward compatibility with tests.""" + async def create(cls, servicesHub) -> "AiService": + """Create AiService from a ServiceHub instance.""" from modules.serviceCenter import getService from modules.serviceCenter.context import ServiceCenterContext ctx = ServiceCenterContext( - user=legacy_services.user, - mandate_id=legacy_services.mandateId, - feature_instance_id=legacy_services.featureInstanceId, - workflow=getattr(legacy_services, "workflow", None), + user=servicesHub.user, + mandate_id=servicesHub.mandateId, + feature_instance_id=servicesHub.featureInstanceId, + workflow=getattr(servicesHub, "workflow", None), ) - return getService("ai", ctx, legacy_hub=legacy_services) + return getService("ai", ctx) # Helper methods diff --git a/modules/serviceCenter/services/serviceAi/subAiCallLooping.py b/modules/serviceCenter/services/serviceAi/subAiCallLooping.py index 2e4edc3e..adfc4d8a 100644 --- a/modules/serviceCenter/services/serviceAi/subAiCallLooping.py +++ b/modules/serviceCenter/services/serviceAi/subAiCallLooping.py @@ -125,10 +125,11 @@ class AiCallLooper: logger.error(errorMsg) raise ValueError(errorMsg) - maxIterations = 50 # Prevent infinite loops + maxIterations = 10 iteration = 0 - allSections = [] # Accumulate all sections across iterations - lastRawResponse = None # Store last raw JSON response for continuation + result = "" + allSections = [] + lastRawResponse = None # JSON Base Iteration System: # - jsonBase: the merged JSON string (replaces accumulatedDirectJson array) diff --git a/modules/serviceCenter/services/serviceAi/subContentExtraction.py b/modules/serviceCenter/services/serviceAi/subContentExtraction.py index a7250a3a..1d1236da 100644 --- a/modules/serviceCenter/services/serviceAi/subContentExtraction.py +++ b/modules/serviceCenter/services/serviceAi/subContentExtraction.py @@ -261,35 +261,34 @@ class ContentExtractor: # Check if it's standardized JSON format (has "documents" or "sections") if document.mimeType == "application/json": - try: - docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) - if docBytes: + docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) + if docBytes: + try: docData = docBytes.decode('utf-8') jsonData = json.loads(docData) - - if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): - logger.info(f"Document is already in standardized JSON format, using as reference") - # Create reference ContentPart for structured JSON - contentPart = ContentPart( - id=f"ref_{document.id}", - label=f"Reference: {document.fileName}", - typeGroup="structure", - mimeType="application/json", - data=docData, - metadata={ - "contentFormat": "reference", - "documentId": document.id, - "documentReference": f"docItem:{document.id}:{document.fileName}", - "skipExtraction": True, - "intent": "reference" - } - ) - allContentParts.append(contentPart) - logger.info(f"✅ Using JSON document directly without extraction") - continue # Skip normal extraction for this document - except Exception as e: - logger.warning(f"Could not parse JSON document {document.fileName}, will extract normally: {str(e)}") - # Continue with normal extraction + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.warning(f"Could not parse JSON document {document.fileName}: {str(e)}") + jsonData = None + + if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): + logger.info(f"Document is already in standardized JSON format, using as reference") + contentPart = ContentPart( + id=f"ref_{document.id}", + label=f"Reference: {document.fileName}", + typeGroup="structure", + mimeType="application/json", + data=docData, + metadata={ + "contentFormat": "reference", + "documentId": document.id, + "documentReference": f"docItem:{document.id}:{document.fileName}", + "skipExtraction": True, + "intent": "reference" + } + ) + allContentParts.append(contentPart) + logger.info(f"✅ Using JSON document directly without extraction") + continue # Normal extraction path intent = getIntentForDocument(document.id, documentIntents) diff --git a/modules/serviceCenter/services/serviceAi/subDocumentIntents.py b/modules/serviceCenter/services/serviceAi/subDocumentIntents.py index 274a8a5a..42dfef14 100644 --- a/modules/serviceCenter/services/serviceAi/subDocumentIntents.py +++ b/modules/serviceCenter/services/serviceAi/subDocumentIntents.py @@ -230,9 +230,12 @@ class DocumentIntentAnalyzer: else: logger.debug(f"JSON document {document.id} has no documentData (actionType={actionType})") + return None + except (json.JSONDecodeError, UnicodeDecodeError) as e: + logger.debug(f"Error parsing document {document.fileName}: {str(e)}") return None except Exception as e: - logger.debug(f"Error resolving pre-extracted document {document.fileName}: {str(e)}") + logger.error(f"Error resolving pre-extracted document {document.fileName}: {str(e)}") return None def _buildIntentAnalysisPrompt( diff --git a/modules/serviceCenter/services/serviceAi/subJsonMerger.py b/modules/serviceCenter/services/serviceAi/subJsonMerger.py index c5a7b058..6b4e6c5e 100644 --- a/modules/serviceCenter/services/serviceAi/subJsonMerger.py +++ b/modules/serviceCenter/services/serviceAi/subJsonMerger.py @@ -330,17 +330,7 @@ class JsonMergeLogger: except Exception as e: logger.error(f"Failed to write merge log file: {e}") else: - # No log file set - write individual file (fallback) - currentFileDir = os.path.dirname(os.path.abspath(__file__)) - logDir = currentFileDir - os.makedirs(logDir, exist_ok=True) - logFilePath = os.path.join(logDir, f"{mergeId}.txt") - try: - with open(logFilePath, 'w', encoding='utf-8') as f: - f.write(logContent) - logger.info(f"JSON merge log written to: {logFilePath}") - except Exception as e: - logger.error(f"Failed to write merge log file: {e}") + logger.debug(f"JSON merge {mergeId} completed ({len(logContent)} chars log). Use initializeLogFile() to persist merge logs.") # Clear buffer for next merge JsonMergeLogger._logBuffer = [] diff --git a/modules/serviceCenter/services/serviceAi/subStructureFilling.py b/modules/serviceCenter/services/serviceAi/subStructureFilling.py index 4df52b56..6ba32dfd 100644 --- a/modules/serviceCenter/services/serviceAi/subStructureFilling.py +++ b/modules/serviceCenter/services/serviceAi/subStructureFilling.py @@ -25,7 +25,7 @@ class StructureFiller: """Handles filling document structure with content.""" # Default concurrency limit for parallel generation (chapters/sections) - DEFAULT_MAX_CONCURRENT_GENERATION = 16 + DEFAULT_MAX_CONCURRENT_GENERATION = 5 def __init__(self, services, aiService): """Initialize StructureFiller with service center and AI service access.""" @@ -568,11 +568,16 @@ class StructureFiller: all_sections_list: List[Dict[str, Any]], language: str, outputFormat: str = "txt", - calculateOverallProgress: callable = None + calculateOverallProgress: callable = None, + preExtractedText: Optional[str] = None ) -> List[Dict[str, Any]]: """ Process a single section and return its elements. Used for parallel processing of sections within a chapter. + + When preExtractedText is provided, the section uses the pre-extracted + content directly in its prompt instead of sending raw content parts + through the heavy extraction pipeline (avoids chunking + N*M AI calls). """ sectionId = section.get("id") sectionTitle = section.get("title", sectionId) @@ -600,6 +605,149 @@ class StructureFiller: elements = [] + # --- Fast path: use pre-extracted text instead of raw content parts --- + if preExtractedText and useAiCall and generationHint: + logger.info( + f"Section {sectionId}: Using pre-extracted text " + f"({len(preExtractedText):,} chars) - lightweight AI path" + ) + + for partId in contentPartIds: + part = self._findContentPartById(partId, contentParts) + if not part: + continue + cf = contentFormats.get(partId, part.metadata.get("contentFormat")) + if cf == "reference": + elements.append({ + "type": "reference", + "documentReference": part.metadata.get("documentReference"), + "label": part.metadata.get("usageHint", part.label) + }) + elif cf == "object": + if part.typeGroup == "image" and part.data: + caption = ( + section.get("caption") + or section.get("metadata", {}).get("caption") + or part.metadata.get("caption", "") + ) + elements.append({ + "type": "image", + "content": { + "base64Data": part.data, + "altText": part.metadata.get("usageHint", part.label), + "caption": caption + }, + "caption": caption + }) + + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( + section=section, + contentParts=[], + userPrompt=userPrompt, + generationHint=generationHint, + allSections=all_sections_list, + sectionIndex=sectionIndex, + isAggregation=False, + language=language, + outputFormat=outputFormat, + preExtractedText=preExtractedText + ) + + sectionOperationId = f"{fillOperationId}_section_{sectionId}" + self.services.chat.progressLogStart( + sectionOperationId, + "Section Generation (Pre-extracted)", + f"Section {sectionIndex + 1}/{totalSections}", + f"{sectionTitle} (pre-extracted)", + parentOperationId=chapterOperationId + ) + + try: + self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") + + operationType = OperationTypeEnum.DATA_ANALYSE + options = AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + + checkWorkflowStopped(self.services) + aiResponseJson = await self.aiService.callAiWithLooping( + prompt=generationPrompt, + options=options, + debugPrefix=f"{chapterId}_section_{sectionId}", + promptBuilder=self.buildSectionPromptWithContinuation, + promptArgs={ + "section": section, + "contentParts": [], + "userPrompt": userPrompt, + "generationHint": generationHint, + "allSections": all_sections_list, + "sectionIndex": sectionIndex, + "isAggregation": False, + "templateStructure": templateStructure, + "basePrompt": generationPrompt, + "language": language + }, + operationId=sectionOperationId, + userPrompt=userPrompt, + contentParts=None, + useCaseId="section_content" + ) + + try: + from modules.shared.jsonUtils import tryParseJson, repairBrokenJson + if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1): + generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId) + else: + parsedResponse, parseError, cleanedStr = tryParseJson(aiResponseJson) + if parsedResponse is None: + logger.warning(f"Section {sectionId}: tryParseJson failed, attempting repair") + repairedStr = repairBrokenJson(aiResponseJson) + parsedResponse, parseError2, _ = tryParseJson(repairedStr) + + if parsedResponse and isinstance(parsedResponse, dict): + generatedElements = parsedResponse.get("elements", []) + elif parsedResponse and isinstance(parsedResponse, list): + generatedElements = parsedResponse + else: + generatedElements = [] + except Exception as parseErr: + logger.error(f"Section {sectionId}: JSON parse error: {parseErr}") + generatedElements = [] + + self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") + + class _AiResponse: + def __init__(self, content): + self.content = content + + responseElements = await self._processAiResponseForSection( + aiResponse=_AiResponse(aiResponseJson), + contentType=contentType, + operationType=operationType, + sectionId=sectionId, + generationHint=generationHint, + generatedElements=generatedElements, + section=section + ) + elements.extend(responseElements) + self.services.chat.progressLogFinish(sectionOperationId, True) + + except Exception as e: + self.services.chat.progressLogFinish(sectionOperationId, False) + logger.error(f"Error in pre-extracted section {sectionId}: {e}") + elements.append({ + "type": "error", + "message": f"Error processing section {sectionId}: {str(e)}", + "sectionId": sectionId + }) + + return elements + + # --- Standard path: process content parts directly --- + # Prüfe ob Aggregation nötig ist needsAggregation = self._needsAggregation( contentType=contentType, @@ -1507,6 +1655,156 @@ class StructureFiller: return elements + async def _preExtractSharedContent( + self, + contentParts: List[ContentPart], + allSectionTasks: List[Dict[str, Any]], + userPrompt: str, + parentOperationId: str + ) -> Dict[str, str]: + """ + Pre-extract content from large/shared content parts ONCE before parallel + section filling. Returns dict mapping sectionId -> pre-extracted text. + + Extracts a comprehensive plain-text summary per content part, then gives + ALL sections referencing that part the SAME summary. Each section's own + generationHint focuses the AI on the relevant aspect during generation. + + This eliminates the N*M AI call explosion where N sections each independently + chunk and process the same M-byte content part through the extraction pipeline. + """ + SIZE_THRESHOLD = 100_000 + MIN_SHARED_SECTIONS = 2 + + partToSections: Dict[str, List[Dict[str, Any]]] = {} + for task in allSectionTasks: + section = task["section"] + for partId in section.get("contentPartIds", []): + if partId not in partToSections: + partToSections[partId] = [] + partToSections[partId].append(section) + + if not partToSections: + return {} + + preExtractedCache: Dict[str, str] = {} + + for partId, sections in partToSections.items(): + part = self._findContentPartById(partId, contentParts) + if not part: + continue + + contentFormat = part.metadata.get("contentFormat", "unknown") + if contentFormat != "extracted": + continue + + if part.typeGroup in ("image", "binary"): + continue + if part.mimeType and ( + part.mimeType.startswith("image/") + or part.mimeType.startswith("video/") + or part.mimeType.startswith("audio/") + ): + continue + + partSize = len(part.data) if part.data else 0 + numSections = len(sections) + + if numSections < MIN_SHARED_SECTIONS and partSize < SIZE_THRESHOLD: + continue + + fileName = part.metadata.get("originalFileName", partId) + logger.info( + f"Pre-extracting content part {partId} " + f"({partSize:,} bytes, referenced by {numSections} sections)" + ) + + topicLines = [] + for section in sections: + hint = ( + section.get("generationHint") + or section.get("generation_hint") + or section.get("title", "") + ) + topicLines.append(f"- {hint}") + topicsText = "\n".join(topicLines) + + extractionPrompt = ( + "# TASK: Extract key information from this document\n\n" + "Extract ALL relevant information from the provided content as " + "plain text. The extracted content will be used to generate a report " + "covering the topics listed below.\n\n" + f"## User Request\n{userPrompt}\n\n" + f"## Report topics that need data\n{topicsText}\n\n" + "## Instructions\n" + "- Extract key facts, data points, timestamps, error messages, " + "statistics, and specific findings\n" + "- Organize by theme but output as PLAIN TEXT (not JSON)\n" + "- Be comprehensive but concise - include specific data, " + "skip generic filler\n" + "- Include concrete examples with exact values from the source\n" + "- Do NOT add commentary or analysis - just extract the raw data\n" + ) + + try: + self.services.chat.progressLogUpdate( + parentOperationId, 0.05, + f"Pre-extracting content from {fileName} ({partSize:,} bytes)..." + ) + + def _preExtractionProgress(chunkProgress, message): + mapped = 0.05 + chunkProgress * 0.05 + self.services.chat.progressLogUpdate( + parentOperationId, mapped, + f"Pre-extraction: {message}" + ) + + request = AiCallRequest( + prompt=extractionPrompt, + contentParts=[part], + options=AiCallOptions( + operationType=OperationTypeEnum.DATA_ANALYSE, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + ) + + checkWorkflowStopped(self.services) + response = await self.aiService.callAi(request, progressCallback=_preExtractionProgress) + responseText = response.content if hasattr(response, "content") else str(response) + + if responseText and len(responseText.strip()) > 50: + for section in sections: + sId = section.get("id", "unknown") + preExtractedCache[sId] = responseText + logger.info( + f"Pre-extraction of {partId} successful: " + f"{len(responseText):,} chars summary for {numSections} sections" + ) + self.services.chat.progressLogUpdate( + parentOperationId, 0.10, + f"Pre-extraction complete ({len(responseText):,} chars). Starting section generation..." + ) + else: + logger.warning( + f"Pre-extraction of {partId} returned empty/short response " + f"({len(responseText) if responseText else 0} chars), " + "sections will fall back to direct extraction" + ) + except Exception as e: + logger.error( + f"Pre-extraction of {partId} failed: {e}. " + "Sections will fall back to direct extraction." + ) + + if preExtractedCache: + logger.info( + f"Pre-extraction complete: {len(preExtractedCache)} sections " + "have pre-extracted content (will use lightweight AI path)" + ) + + return preExtractedCache + async def _fillChapterSections( self, chapterStructure: Dict[str, Any], @@ -1564,27 +1862,42 @@ class StructureFiller: "docFormat": docFormat # Include output format }) + MAX_TOTAL_SECTIONS = 35 + if totalSections > MAX_TOTAL_SECTIONS: + logger.warning( + f"Structure has {totalSections} sections (limit {MAX_TOTAL_SECTIONS}). " + "Truncating to stay within budget." + ) + allSectionTasks = allSectionTasks[:MAX_TOTAL_SECTIONS] + totalSections = len(allSectionTasks) + + preExtractedCache = await self._preExtractSharedContent( + contentParts, allSectionTasks, userPrompt, fillOperationId + ) + logger.info(f"Starting FULLY PARALLEL section generation: {totalSections} sections across {totalChapters} chapters") # Create task wrapper for each section with progress tracking async def processSectionWithSemaphore(taskInfo): checkWorkflowStopped(self.services) + sectionId = taskInfo["section"].get("id", "unknown") async with sectionSemaphore: result = await self._processSingleSection( section=taskInfo["section"], sectionIndex=taskInfo["sectionIndex"], totalSections=taskInfo["chapterSectionCount"], - chapterIndex=0, # Not used for sequential logic anymore + chapterIndex=0, totalChapters=totalChapters, chapterId=taskInfo["chapterId"], - chapterOperationId=fillOperationId, # Use fillOperationId as parent (no chapter-level ops in parallel mode) + chapterOperationId=fillOperationId, fillOperationId=fillOperationId, contentParts=contentParts, userPrompt=userPrompt, all_sections_list=all_sections_list, language=taskInfo["docLanguage"], - outputFormat=taskInfo.get("docFormat", "txt"), # Pass output format - calculateOverallProgress=lambda *args: completedSections[0] / totalSections if totalSections > 0 else 1.0 + outputFormat=taskInfo.get("docFormat", "txt"), + calculateOverallProgress=lambda *args: completedSections[0] / totalSections if totalSections > 0 else 1.0, + preExtractedText=preExtractedCache.get(sectionId) ) # Update progress after each section completes @@ -1810,6 +2123,7 @@ GENERATION HINT: {generationHint} - Each section should serve a clear purpose with meaningful data - If no relevant data exists for a topic, do NOT create a section for it - Prefer ONE comprehensive section over multiple sparse sections +- HARD LIMIT: Maximum 5 sections per chapter. Combine related subtopics into single sections to stay within this limit. **CRITICAL**: The chapter's generationHint above describes what content this chapter should generate. If the generationHint references documents/images/data, then EACH section that generates content for this chapter MUST assign the relevant ContentParts from AVAILABLE CONTENT PARTS below. @@ -1893,7 +2207,8 @@ Return only valid JSON. Do not include any explanatory text outside the JSON. sectionIndex: Optional[int] = None, isAggregation: bool = False, language: str = "en", - outputFormat: str = "txt" + outputFormat: str = "txt", + preExtractedText: Optional[str] = None ) -> tuple[str, str]: """Baue Prompt für Section-Generierung mit vollständigem Kontext.""" # Filtere None-Werte @@ -2057,7 +2372,7 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, 5. For table: Extract all rows from the context. Return {{"headers": [...], "rows": []}} only if no data exists. 6. Format based on content_type ({effectiveContentType}). 7. No HTML/styling: Plain text only, no markup. -8. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. +8. Focus on the MOST RELEVANT information for this section's topic. Extract key facts, data, and findings. Omit redundant, repetitive, or tangential content. ## OUTPUT FORMAT @@ -2083,6 +2398,62 @@ Output requirements: {userPrompt} ``` +## CONTEXT +{contextText if contextText else ""} +""" + elif preExtractedText: + prompt = f"""# TASK: Generate Section Content from Pre-Extracted Data + +LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. + +## SECTION METADATA +- Section ID: {sectionId} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + +## PRE-EXTRACTED CONTENT FOR THIS SECTION +``` +{preExtractedText} +``` + +## INSTRUCTIONS +1. Use ONLY the pre-extracted content above. Never invent or generate data not present in it. +2. If the pre-extracted content is empty, return empty structures. +3. Format based on content_type ({effectiveContentType}). +4. Return only valid JSON with "elements" array. +5. No HTML/styling: Plain text only, no markup. +6. Focus on the MOST RELEVANT information. Be concise. + +## OUTPUT FORMAT +Return a JSON object with this structure: + +{{ + "elements": [ + {{ + "type": "{effectiveContentType}", + "content": {contentStructureExample} + }} + ] +}} + +Output requirements: +- "content" must be an object (never a string) +- Return only valid JSON - no text before, no text after, no comments, no explanations, no markdown code fences +- Start with {{ and end with }} - return ONLY the JSON object itself +- No invented data: Return empty structures if pre-extracted content is empty + +## USER REQUEST +``` +{userPrompt} +``` + ## CONTEXT {contextText if contextText else ""} """ @@ -2117,7 +2488,7 @@ LANGUAGE: Generate all content in {language.upper()} language. All text, titles, 3. Format based on content_type ({effectiveContentType}). 4. Return only valid JSON with "elements" array. 5. No HTML/styling: Plain text only, no markup. -6. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. +6. Focus on the MOST RELEVANT information for this section's topic. Extract key facts, data, and findings. Omit redundant, repetitive, or tangential content. ## OUTPUT FORMAT Return a JSON object with this structure: diff --git a/modules/serviceCenter/services/serviceAi/subStructureGeneration.py b/modules/serviceCenter/services/serviceAi/subStructureGeneration.py index 9795cf6f..72127c92 100644 --- a/modules/serviceCenter/services/serviceAi/subStructureGeneration.py +++ b/modules/serviceCenter/services/serviceAi/subStructureGeneration.py @@ -430,6 +430,7 @@ Then chapters that generate those generic content types MUST assign the relevant ## CHAPTER STRUCTURE REQUIREMENTS - Generate chapters based on USER REQUEST - analyze what structure the user wants - Create ONLY the minimum chapters needed to cover the user's request - avoid over-structuring +- HARD LIMIT: Maximum 7 chapters per document. If the topic can be covered in fewer, prefer fewer. Combine related topics into single chapters rather than creating many small ones. - IMPORTANT: Each chapter MUST have ALL these fields: - id: Unique identifier (e.g., "chapter_1") - level: Heading level (1, 2, 3, etc.) diff --git a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py index 128a307e..7a6951ae 100644 --- a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py +++ b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py @@ -205,36 +205,20 @@ class BillingService: workflowId: str = None, aicoreProvider: str = None, aicoreModel: str = None, - description: str = None + description: str = None, + processingTime: float = None, + bytesSent: int = None, + bytesReceived: int = None, + errorCount: int = None ) -> Optional[Dict[str, Any]]: - """ - Record AI usage cost as a billing transaction. - - This method: - 1. Applies the pricing markup - 2. Creates a DEBIT transaction - 3. Updates the account balance - - Args: - priceCHF: Base price from AI model (before markup) - workflowId: Optional workflow ID - aicoreProvider: AICore provider name (e.g., 'anthropic', 'openai') - aicoreModel: AICore model name (e.g., 'claude-4-sonnet', 'gpt-4o') - description: Optional description - - Returns: - Created transaction dict or None if not recorded - """ + """Record AI usage cost as a billing transaction with markup applied.""" if priceCHF <= 0: return None - # Apply markup finalPrice = self.calculatePriceWithMarkup(priceCHF) - if finalPrice <= 0: return None - # Build description if not description: description = f"AI Usage: {aicoreModel or aicoreProvider or 'unknown'}" @@ -247,9 +231,17 @@ class BillingService: featureCode=self.featureCode, aicoreProvider=aicoreProvider, aicoreModel=aicoreModel, - description=description + description=description, + processingTime=processingTime, + bytesSent=bytesSent, + bytesReceived=bytesReceived, + errorCount=errorCount ) + def getWorkflowCost(self, workflowId: str) -> float: + """Get total cost for a workflow from billing transactions.""" + return self._billingInterface.getWorkflowCost(workflowId) + # ========================================================================= # Provider Permission Check (via RBAC) # ========================================================================= diff --git a/modules/serviceCenter/services/serviceChat/mainServiceChat.py b/modules/serviceCenter/services/serviceChat/mainServiceChat.py index 6182f397..026dc70c 100644 --- a/modules/serviceCenter/services/serviceChat/mainServiceChat.py +++ b/modules/serviceCenter/services/serviceChat/mainServiceChat.py @@ -4,7 +4,7 @@ import logging from typing import Dict, Any, List, Optional, Callable from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum from modules.shared.progressLogger import ProgressLogger @@ -411,23 +411,159 @@ class ChatService: return None def getFileInfo(self, fileId: str) -> Dict[str, Any]: - """Get file information""" - file_item = self.interfaceDbComponent.getFile(fileId) - if file_item: + """Get file information including new fields (tags, folderId, description, status).""" + fileItem = self.interfaceDbComponent.getFile(fileId) + if fileItem: return { - "id": file_item.id, - "fileName": file_item.fileName, - "size": file_item.fileSize, - "mimeType": file_item.mimeType, - "fileHash": file_item.fileHash, - "creationDate": file_item.creationDate + "id": fileItem.id, + "fileName": fileItem.fileName, + "size": fileItem.fileSize, + "mimeType": fileItem.mimeType, + "fileHash": fileItem.fileHash, + "creationDate": fileItem.creationDate, + "tags": getattr(fileItem, "tags", None), + "folderId": getattr(fileItem, "folderId", None), + "description": getattr(fileItem, "description", None), + "status": getattr(fileItem, "status", None), } return None def getFileData(self, fileId: str) -> bytes: - """Get file data by ID""" + """Get file data by ID.""" return self.interfaceDbComponent.getFileData(fileId) - + + def getFileContent(self, fileId: str) -> Optional[Dict[str, Any]]: + """Get file content as text or base64 via FilePreview.""" + preview = self.interfaceDbComponent.getFileContent(fileId) + if preview: + return preview.toDictWithBase64Encoding() + return None + + def listFiles( + self, + folderId: str = None, + tags: List[str] = None, + search: str = None, + ) -> List[Dict[str, Any]]: + """List files for the current user with optional filters. + + Args: + folderId: Filter by folder (None = root / all). + tags: Filter by tags (any match). + search: Search in fileName and description. + + Returns: + List of file info dicts. + """ + allFiles = self.interfaceDbComponent.getAllFiles() + results = [] + for fileItem in allFiles: + if folderId is not None: + itemFolderId = getattr(fileItem, "folderId", None) + if itemFolderId != folderId: + continue + + if tags: + itemTags = getattr(fileItem, "tags", None) or [] + if not any(t in itemTags for t in tags): + continue + + if search: + searchLower = search.lower() + nameMatch = searchLower in (fileItem.fileName or "").lower() + descMatch = searchLower in (getattr(fileItem, "description", None) or "").lower() + if not nameMatch and not descMatch: + continue + + results.append({ + "id": fileItem.id, + "fileName": fileItem.fileName, + "mimeType": fileItem.mimeType, + "fileSize": fileItem.fileSize, + "creationDate": fileItem.creationDate, + "tags": getattr(fileItem, "tags", None), + "folderId": getattr(fileItem, "folderId", None), + "description": getattr(fileItem, "description", None), + "status": getattr(fileItem, "status", None), + }) + return results + + def listFolders(self, parentId: str = None) -> List[Dict[str, Any]]: + """List file folders for the current user. + + Args: + parentId: Parent folder ID (None = root folders). + + Returns: + List of folder dicts. + """ + from modules.datamodels.datamodelFileFolder import FileFolder + recordFilter = {"_createdBy": self.user.id if self.user else ""} + if parentId is not None: + recordFilter["parentId"] = parentId + else: + recordFilter["parentId"] = None + return self.interfaceDbComponent.db.getRecordset(FileFolder, recordFilter=recordFilter) + + def createFolder(self, name: str, parentId: str = None) -> Dict[str, Any]: + """Create a new file folder.""" + from modules.datamodels.datamodelFileFolder import FileFolder + folder = FileFolder(name=name, parentId=parentId) + return self.interfaceDbComponent.db.recordCreate(FileFolder, folder) + + # ---- DataSource CRUD ---- + + def createDataSource( + self, connectionId: str, sourceType: str, path: str, label: str, + featureInstanceId: str = None + ) -> Dict[str, Any]: + """Create a new external data source reference.""" + from modules.datamodels.datamodelDataSource import DataSource + ds = DataSource( + connectionId=connectionId, + sourceType=sourceType, + path=path, + label=label, + featureInstanceId=featureInstanceId or self._context.feature_instance_id or "", + mandateId=self._context.mandate_id or "", + userId=self.user.id if self.user else "", + ) + return self.interfaceDbComponent.db.recordCreate(DataSource, ds) + + def listDataSources(self, featureInstanceId: str = None) -> List[Dict[str, Any]]: + """List data sources, optionally filtered by feature instance.""" + from modules.datamodels.datamodelDataSource import DataSource + recordFilter = {} + if featureInstanceId: + recordFilter["featureInstanceId"] = featureInstanceId + return self.interfaceDbComponent.db.getRecordset(DataSource, recordFilter=recordFilter) + + def getDataSource(self, dataSourceId: str) -> Optional[Dict[str, Any]]: + """Get a single data source by ID.""" + from modules.datamodels.datamodelDataSource import DataSource + results = self.interfaceDbComponent.db.getRecordset(DataSource, recordFilter={"id": dataSourceId}) + return results[0] if results else None + + def deleteDataSource(self, dataSourceId: str) -> bool: + """Delete a data source.""" + from modules.datamodels.datamodelDataSource import DataSource + try: + self.interfaceDbComponent.db.recordDelete(DataSource, dataSourceId) + return True + except Exception as e: + logger.error(f"Failed to delete DataSource {dataSourceId}: {e}") + return False + + def getUserConnections(self) -> List[Dict[str, Any]]: + """Get all UserConnections for the current user.""" + try: + if self.interfaceDbApp and self.user: + connections = self.interfaceDbApp.getUserConnections(self.user.id) + return [c.model_dump() if hasattr(c, "model_dump") else c for c in (connections or [])] + except Exception as e: + logger.error(f"Error getting user connections: {e}") + return [] + def _diagnoseDocumentAccess(self, document: ChatDocument) -> Dict[str, Any]: """ Diagnose document access issues and provide recovery information. @@ -688,35 +824,6 @@ class ChatService: workflow.logs.append(chatLog) return chatLog - def storeWorkflowStat(self, workflow: Any, aiResponse: Any, process: str) -> ChatStat: - """Persist workflow-level ChatStat from AiCallResponse and append to workflow stats list. - - Billing is handled at the AI call source (interfaceAiObjects._callWithModel) - via billingCallback - not here. This method only handles workflow stats. - """ - try: - statData = { - "workflowId": workflow.id, - "process": process, - "engine": aiResponse.modelName, - "priceCHF": aiResponse.priceCHF, - "processingTime": aiResponse.processingTime, - "bytesSent": aiResponse.bytesSent, - "bytesReceived": aiResponse.bytesReceived, - "errorCount": aiResponse.errorCount - } - - stat = self.interfaceDbChat.createStat(statData) - - if not hasattr(workflow, 'stats') or workflow.stats is None: - workflow.stats = [] - workflow.stats.append(stat) - - return stat - except Exception as e: - logger.error(f"Failed to store workflow stat: {e}") - raise - def updateMessage(self, messageId: str, messageData: Dict[str, Any]): """Update message by delegating to the chat interface""" try: diff --git a/modules/serviceCenter/services/serviceExtraction/chunking/chunkerStructure.py b/modules/serviceCenter/services/serviceExtraction/chunking/chunkerStructure.py index f4d23a72..fa65e19c 100644 --- a/modules/serviceCenter/services/serviceExtraction/chunking/chunkerStructure.py +++ b/modules/serviceCenter/services/serviceExtraction/chunking/chunkerStructure.py @@ -2,90 +2,147 @@ # All rights reserved. from typing import Any, Dict, List import json +import logging from modules.datamodels.datamodelExtraction import ContentPart from ..subRegistry import Chunker +logger = logging.getLogger(__name__) + class StructureChunker(Chunker): def chunk(self, part: ContentPart, options: Dict[str, Any]) -> list[Dict[str, Any]]: maxBytes = int(options.get("structureChunkSize", 40000)) data = part.data or "" - # best-effort: try JSON list/object bucketing; else fallback to line-based chunks: List[Dict[str, Any]] = [] + try: obj = json.loads(data) - def emit(bucket: Any): - text = json.dumps(bucket, ensure_ascii=False) - chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) - if isinstance(obj, list): - bucket: list[Any] = [] - size = 0 - for item in obj: - text = json.dumps(item, ensure_ascii=False) - s = len(text.encode('utf-8')) - if size + s > maxBytes and bucket: - emit(bucket) - bucket = [item] - size = s - else: - bucket.append(item) - size += s - if bucket: - emit(bucket) - else: - # JSON object (dict) - check if it fits - text = json.dumps(obj, ensure_ascii=False) - textSize = len(text.encode('utf-8')) - if textSize <= maxBytes: - emit(obj) - else: - # Object too large - try to split by keys if possible - # For large objects, we need to chunk by character boundaries - # since we can't split JSON objects arbitrarily - if isinstance(obj, dict) and len(obj) > 1: - # Try to split object into multiple chunks by keys - # This preserves JSON structure better than line-based chunking - currentChunk: Dict[str, Any] = {} - currentSize = 2 # Start with "{}" overhead - for key, value in obj.items(): - itemText = json.dumps({key: value}, ensure_ascii=False) - itemSize = len(itemText.encode('utf-8')) - # Account for comma and spacing between items - if currentChunk: - itemSize += 2 # ", " separator - - if currentSize + itemSize > maxBytes and currentChunk: - # Current chunk is full, emit it - emit(currentChunk) - currentChunk = {key: value} - currentSize = len(itemText.encode('utf-8')) - else: - currentChunk[key] = value - currentSize += itemSize - - # Emit remaining chunk - if currentChunk: - emit(currentChunk) - else: - # Single large value or can't split - fallback to line chunking - raise ValueError("too large") - except Exception: - current: List[str] = [] - size = 0 - for line in data.split('\n'): - s = len(line.encode('utf-8')) + 1 - if size + s > maxBytes and current: - text = '\n'.join(current) - chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) - current = [line] - size = s - else: - current.append(line) - size += s - if current: - text = '\n'.join(current) - chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) + self._chunkValue(obj, maxBytes, chunks) + except (json.JSONDecodeError, ValueError): + self._chunkByLines(data, maxBytes, chunks) + return chunks + def _chunkValue(self, obj: Any, maxBytes: int, chunks: List[Dict[str, Any]]): + """Recursively chunk a JSON value (list or dict) into pieces <= maxBytes.""" + text = json.dumps(obj, ensure_ascii=False) + if len(text.encode('utf-8')) <= maxBytes: + self._emit(obj, chunks) + return + + if isinstance(obj, list): + self._chunkList(obj, maxBytes, chunks) + elif isinstance(obj, dict): + self._chunkDict(obj, maxBytes, chunks) + else: + self._chunkByLines(text, maxBytes, chunks) + + def _chunkList(self, items: list, maxBytes: int, chunks: List[Dict[str, Any]]): + """Split a JSON array into sub-arrays that each fit within maxBytes.""" + bucket: list = [] + bucketSize = 2 # "[]" overhead + + for item in items: + itemText = json.dumps(item, ensure_ascii=False) + itemSize = len(itemText.encode('utf-8')) + separator = 2 if bucket else 0 # ", " + + if bucketSize + itemSize + separator > maxBytes and bucket: + self._emit(bucket, chunks) + bucket = [] + bucketSize = 2 + separator = 0 + + if itemSize + 2 > maxBytes: + if bucket: + self._emit(bucket, chunks) + bucket = [] + bucketSize = 2 + self._chunkValue(item, maxBytes, chunks) + else: + bucket.append(item) + bucketSize += itemSize + separator + + if bucket: + self._emit(bucket, chunks) + + def _chunkDict(self, obj: dict, maxBytes: int, chunks: List[Dict[str, Any]]): + """Split a JSON object by keys. If a single key's value exceeds maxBytes, recurse into it.""" + if len(obj) <= 1: + key, value = next(iter(obj.items())) + if isinstance(value, (list, dict)): + self._chunkSingleKeyValue(key, value, maxBytes, chunks) + else: + text = json.dumps(obj, ensure_ascii=False) + self._chunkByLines(text, maxBytes, chunks) + return + + currentChunk: Dict[str, Any] = {} + currentSize = 2 # "{}" overhead + + for key, value in obj.items(): + itemText = json.dumps({key: value}, ensure_ascii=False) + itemSize = len(itemText.encode('utf-8')) + separator = 2 if currentChunk else 0 + + if currentSize + itemSize + separator > maxBytes and currentChunk: + self._emit(currentChunk, chunks) + currentChunk = {} + currentSize = 2 + separator = 0 + + if itemSize + 2 > maxBytes: + if currentChunk: + self._emit(currentChunk, chunks) + currentChunk = {} + currentSize = 2 + if isinstance(value, (list, dict)): + self._chunkSingleKeyValue(key, value, maxBytes, chunks) + else: + self._chunkByLines(itemText, maxBytes, chunks) + else: + currentChunk[key] = value + currentSize += itemSize + separator + + if currentChunk: + self._emit(currentChunk, chunks) + + def _chunkSingleKeyValue(self, key: str, value: Any, maxBytes: int, chunks: List[Dict[str, Any]]): + """Handle a single dict key whose value is too large. Wraps sub-chunks back in {key: subChunk}.""" + subChunks: List[Dict[str, Any]] = [] + self._chunkValue(value, maxBytes, subChunks) + + for sub in subChunks: + subData = json.loads(sub["data"]) + wrapped = {key: subData} + wrappedText = json.dumps(wrapped, ensure_ascii=False) + wrappedSize = len(wrappedText.encode('utf-8')) + if wrappedSize <= maxBytes: + self._emit(wrapped, chunks) + else: + self._chunkByLines(wrappedText, maxBytes, chunks) + + def _emit(self, bucket: Any, chunks: List[Dict[str, Any]]): + text = json.dumps(bucket, ensure_ascii=False) + chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) + + def _chunkByLines(self, data: str, maxBytes: int, chunks: List[Dict[str, Any]]): + """Line-based fallback for content that cannot be split structurally.""" + current: List[str] = [] + size = 0 + for line in data.split('\n'): + s = len(line.encode('utf-8')) + 1 + if size + s > maxBytes and current: + text = '\n'.join(current) + chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) + current = [line] + size = s + else: + current.append(line) + size += s + if current: + text = '\n'.join(current) + chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": len(chunks)}) + diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorAudio.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorAudio.py new file mode 100644 index 00000000..a1f06f99 --- /dev/null +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorAudio.py @@ -0,0 +1,175 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Audio extractor for common audio formats. + +Extracts metadata (duration, bitrate, sample rate, channels) and produces +an `audiostream` ContentPart. For files under 10 MB the base64 audio data +is included; larger files only get metadata. + +Optional dependency: mutagen (for rich metadata). +""" + +from typing import Any, Dict, List +import base64 +import logging +import struct + +from modules.datamodels.datamodelExtraction import ContentPart +from ..subUtils import makeId +from ..subRegistry import Extractor + +logger = logging.getLogger(__name__) + +_AUDIO_MIME_TYPES = [ + "audio/mpeg", + "audio/mp3", + "audio/wav", + "audio/x-wav", + "audio/ogg", + "audio/flac", + "audio/x-flac", + "audio/mp4", + "audio/x-m4a", + "audio/aac", + "audio/webm", +] +_AUDIO_EXTENSIONS = [".mp3", ".wav", ".ogg", ".flac", ".m4a", ".aac", ".wma", ".webm"] + +_MAX_INLINE_SIZE = 10 * 1024 * 1024 # 10 MB + + +class AudioExtractor(Extractor): + """Extractor for audio files. + + Produces: + - 1 text ContentPart with metadata summary + - 1 audiostream ContentPart (base64 data included only if < 10 MB) + """ + + def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: + if mimeType in _AUDIO_MIME_TYPES: + return True + lower = (fileName or "").lower() + return any(lower.endswith(ext) for ext in _AUDIO_EXTENSIONS) + + def getSupportedExtensions(self) -> list[str]: + return list(_AUDIO_EXTENSIONS) + + def getSupportedMimeTypes(self) -> list[str]: + return list(_AUDIO_MIME_TYPES) + + def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: + fileName = context.get("fileName", "audio") + mimeType = context.get("mimeType") or "audio/mpeg" + fileSize = len(fileBytes) + + rootId = makeId() + parts: List[ContentPart] = [] + + meta = _extractMetadata(fileBytes, fileName) + meta["size"] = fileSize + meta["fileName"] = fileName + meta["mimeType"] = mimeType + + metaLines = [f"Audio file: {fileName}"] + if meta.get("duration"): + mins = int(meta["duration"] // 60) + secs = int(meta["duration"] % 60) + metaLines.append(f"Duration: {mins}:{secs:02d}") + if meta.get("bitrate"): + metaLines.append(f"Bitrate: {meta['bitrate']} kbps") + if meta.get("sampleRate"): + metaLines.append(f"Sample rate: {meta['sampleRate']} Hz") + if meta.get("channels"): + metaLines.append(f"Channels: {meta['channels']}") + if meta.get("title") or meta.get("artist") or meta.get("album"): + metaLines.append(f"Title: {meta.get('title', 'N/A')}") + metaLines.append(f"Artist: {meta.get('artist', 'N/A')}") + metaLines.append(f"Album: {meta.get('album', 'N/A')}") + metaLines.append(f"Size: {fileSize:,} bytes") + + parts.append(ContentPart( + id=rootId, parentId=None, label="metadata", + typeGroup="text", mimeType="text/plain", + data="\n".join(metaLines), metadata=meta, + )) + + audioData = "" + if fileSize <= _MAX_INLINE_SIZE: + audioData = base64.b64encode(fileBytes).decode("utf-8") + + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="audiostream", + typeGroup="audiostream", mimeType=mimeType, + data=audioData, metadata={"size": fileSize, "inlined": fileSize <= _MAX_INLINE_SIZE}, + )) + + return parts + + +def _extractMetadata(fileBytes: bytes, fileName: str) -> Dict[str, Any]: + """Extract audio metadata using mutagen (optional) with stdlib fallback.""" + meta: Dict[str, Any] = {} + + try: + import mutagen + import io + audio = mutagen.File(io.BytesIO(fileBytes)) + if audio is not None: + if audio.info: + meta["duration"] = getattr(audio.info, "length", None) + meta["bitrate"] = getattr(audio.info, "bitrate", None) + if meta["bitrate"]: + meta["bitrate"] = meta["bitrate"] // 1000 + meta["sampleRate"] = getattr(audio.info, "sample_rate", None) + meta["channels"] = getattr(audio.info, "channels", None) + + tags = audio.tags + if tags: + meta["title"] = _getTag(tags, ["TIT2", "title", "\xa9nam"]) + meta["artist"] = _getTag(tags, ["TPE1", "artist", "\xa9ART"]) + meta["album"] = _getTag(tags, ["TALB", "album", "\xa9alb"]) + + return {k: v for k, v in meta.items() if v is not None} + except ImportError: + logger.debug("mutagen not installed -- using basic metadata extraction") + except Exception as e: + logger.debug(f"mutagen metadata extraction failed: {e}") + + lower = fileName.lower() + if lower.endswith(".wav"): + meta.update(_parseWavHeader(fileBytes)) + + return {k: v for k, v in meta.items() if v is not None} + + +def _getTag(tags, keys: list) -> Any: + """Try multiple tag keys and return the first found value.""" + for key in keys: + val = tags.get(key) + if val is not None: + return str(val) if not isinstance(val, str) else val + return None + + +def _parseWavHeader(fileBytes: bytes) -> Dict[str, Any]: + """Minimal WAV header parser for basic metadata.""" + meta: Dict[str, Any] = {} + if len(fileBytes) < 44: + return meta + try: + if fileBytes[:4] != b"RIFF" or fileBytes[8:12] != b"WAVE": + return meta + channels = struct.unpack_from(" str: + """Detect MIME type from file name.""" + guessed, _ = mimetypes.guess_type(fileName) + return guessed or "application/octet-stream" + + +def _isSymlink(info) -> bool: + """Check if a tar member is a symlink.""" + if hasattr(info, "issym") and callable(info.issym): + return info.issym() or info.islnk() + return False + + +class ContainerExtractor(Extractor): + """Extractor for archive containers (ZIP, TAR, GZ, 7Z). + + Recursively resolves nested containers and produces a flat list of + ContentPart entries -- one per contained file -- with containerPath metadata. + """ + + def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: + if mimeType in _CONTAINER_MIME_TYPES: + return True + lower = (fileName or "").lower() + return any(lower.endswith(ext) for ext in _CONTAINER_EXTENSIONS) + + def getSupportedExtensions(self) -> list[str]: + return list(_CONTAINER_EXTENSIONS) + + def getSupportedMimeTypes(self) -> list[str]: + return list(_CONTAINER_MIME_TYPES) + + def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: + """Extract by recursively unpacking the container.""" + fileName = context.get("fileName", "archive") + mimeType = context.get("mimeType", "application/octet-stream") + + rootId = makeId() + parts: List[ContentPart] = [ + ContentPart( + id=rootId, + parentId=None, + label=fileName, + typeGroup="container", + mimeType=mimeType, + data="", + metadata={"size": len(fileBytes), "containerType": "archive"}, + ) + ] + + state = {"totalSize": 0, "fileCount": 0} + try: + childParts = _resolveContainerRecursive( + fileBytes, mimeType, fileName, rootId, "", 0, state + ) + parts.extend(childParts) + except ContainerLimitError as e: + logger.warning(f"Container limit reached for {fileName}: {e}") + parts.append(ContentPart( + id=makeId(), + parentId=rootId, + label="limit_exceeded", + typeGroup="text", + mimeType="text/plain", + data=str(e), + metadata={"warning": "Container extraction limit exceeded"}, + )) + + return parts + + +def _resolveContainerRecursive( + containerBytes: bytes, + containerMime: str, + containerName: str, + parentId: str, + containerPath: str, + depth: int, + state: Dict[str, int], +) -> List[ContentPart]: + """Recursively unpack containers. No AI calls.""" + if depth > MAX_DEPTH: + raise ContainerLimitError(f"Max nesting depth {MAX_DEPTH} exceeded") + + parts: List[ContentPart] = [] + + if containerMime in ("application/zip", "application/x-zip-compressed") or containerName.lower().endswith(".zip"): + parts.extend(_extractZip(containerBytes, parentId, containerPath, depth, state)) + elif containerMime in ("application/x-tar",) or containerName.lower().endswith(".tar"): + parts.extend(_extractTar(containerBytes, parentId, containerPath, depth, state, compressed=False)) + elif containerMime in ("application/gzip", "application/x-gzip") or containerName.lower().endswith((".gz", ".tgz", ".tar.gz")): + parts.extend(_extractTar(containerBytes, parentId, containerPath, depth, state, compressed=True)) + elif containerName.lower().endswith(".7z"): + parts.extend(_extract7z(containerBytes, parentId, containerPath, depth, state)) + else: + logger.warning(f"Unknown container format: {containerMime} ({containerName})") + + return parts + + +def _addFilePart( + data: bytes, + fileName: str, + parentId: str, + containerPath: str, + state: Dict[str, int], +) -> List[ContentPart]: + """Extract a file via its type-specific Extractor and return ContentParts.""" + state["totalSize"] += len(data) + state["fileCount"] += 1 + + if state["totalSize"] > MAX_TOTAL_EXTRACTED_SIZE: + raise ContainerLimitError(f"Total extracted size exceeds {MAX_TOTAL_EXTRACTED_SIZE // (1024 * 1024)} MB") + if state["fileCount"] > MAX_FILE_COUNT: + raise ContainerLimitError(f"File count exceeds {MAX_FILE_COUNT}") + + entryPath = f"{containerPath}/{fileName}" if containerPath else fileName + detectedMime = _detectMimeType(fileName) + + from ..subRegistry import ExtractorRegistry + registry = ExtractorRegistry() + extractor = registry.resolve(detectedMime, fileName) + + if extractor and not isinstance(extractor, ContainerExtractor): + try: + childParts = extractor.extract(data, {"fileName": fileName, "mimeType": detectedMime}) + for part in childParts: + part.parentId = parentId + if not part.metadata: + part.metadata = {} + part.metadata["containerPath"] = entryPath + return childParts + except Exception as e: + logger.warning(f"Type-extractor failed for {fileName} in container: {e}") + + import base64 + encodedData = base64.b64encode(data).decode("utf-8") if data else "" + + return [ContentPart( + id=makeId(), + parentId=parentId, + label=fileName, + typeGroup="binary", + mimeType=detectedMime, + data=encodedData, + metadata={ + "size": len(data), + "containerPath": entryPath, + "contextRef": ContentContextRef( + containerPath=entryPath, + location="file", + ).model_dump(), + }, + )] + + +def _isNestedContainer(fileName: str, mimeType: str) -> bool: + lower = fileName.lower() + return any(lower.endswith(ext) for ext in _CONTAINER_EXTENSIONS) or mimeType in _CONTAINER_MIME_TYPES + + +def _extractZip( + data: bytes, parentId: str, containerPath: str, depth: int, state: Dict[str, int] +) -> List[ContentPart]: + parts: List[ContentPart] = [] + try: + with zipfile.ZipFile(io.BytesIO(data)) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + if info.file_size == 0: + continue + + entryPath = f"{containerPath}/{info.filename}" if containerPath else info.filename + entryMime = _detectMimeType(info.filename) + entryData = zf.read(info.filename) + + if _isNestedContainer(info.filename, entryMime): + nestedId = makeId() + parts.append(ContentPart( + id=nestedId, + parentId=parentId, + label=info.filename, + typeGroup="container", + mimeType=entryMime, + data="", + metadata={"size": len(entryData), "containerPath": entryPath}, + )) + nested = _resolveContainerRecursive( + entryData, entryMime, info.filename, nestedId, entryPath, depth + 1, state + ) + parts.extend(nested) + else: + parts.extend(_addFilePart(entryData, info.filename, parentId, containerPath, state)) + except zipfile.BadZipFile as e: + logger.error(f"Invalid ZIP file: {e}") + parts.append(ContentPart( + id=makeId(), parentId=parentId, label="error", + typeGroup="text", mimeType="text/plain", + data=f"Invalid ZIP archive: {e}", metadata={"error": True}, + )) + return parts + + +def _extractTar( + data: bytes, parentId: str, containerPath: str, depth: int, state: Dict[str, int], + compressed: bool = False, +) -> List[ContentPart]: + parts: List[ContentPart] = [] + mode = "r:gz" if compressed else "r:" + try: + with tarfile.open(fileobj=io.BytesIO(data), mode=mode) as tf: + for member in tf.getmembers(): + if member.isdir(): + continue + if _isSymlink(member): + logger.warning(f"Skipping symlink in TAR: {member.name}") + continue + if member.size == 0: + continue + + entryPath = f"{containerPath}/{member.name}" if containerPath else member.name + entryMime = _detectMimeType(member.name) + fobj = tf.extractfile(member) + if fobj is None: + continue + entryData = fobj.read() + + if _isNestedContainer(member.name, entryMime): + nestedId = makeId() + parts.append(ContentPart( + id=nestedId, parentId=parentId, label=member.name, + typeGroup="container", mimeType=entryMime, data="", + metadata={"size": len(entryData), "containerPath": entryPath}, + )) + nested = _resolveContainerRecursive( + entryData, entryMime, member.name, nestedId, entryPath, depth + 1, state + ) + parts.extend(nested) + else: + parts.extend(_addFilePart(entryData, member.name, parentId, containerPath, state)) + except tarfile.TarError as e: + logger.error(f"Invalid TAR file: {e}") + parts.append(ContentPart( + id=makeId(), parentId=parentId, label="error", + typeGroup="text", mimeType="text/plain", + data=f"Invalid TAR archive: {e}", metadata={"error": True}, + )) + return parts + + +def _extract7z( + data: bytes, parentId: str, containerPath: str, depth: int, state: Dict[str, int] +) -> List[ContentPart]: + """Extract 7z archive. Requires py7zr (optional dependency).""" + parts: List[ContentPart] = [] + try: + import py7zr + with py7zr.SevenZipFile(io.BytesIO(data), mode="r") as szf: + allFiles = szf.readall() + for fileName, bio in allFiles.items(): + entryData = bio.read() if hasattr(bio, "read") else bytes(bio) + if not entryData: + continue + + entryPath = f"{containerPath}/{fileName}" if containerPath else fileName + entryMime = _detectMimeType(fileName) + + if _isNestedContainer(fileName, entryMime): + nestedId = makeId() + parts.append(ContentPart( + id=nestedId, parentId=parentId, label=fileName, + typeGroup="container", mimeType=entryMime, data="", + metadata={"size": len(entryData), "containerPath": entryPath}, + )) + nested = _resolveContainerRecursive( + entryData, entryMime, fileName, nestedId, entryPath, depth + 1, state + ) + parts.extend(nested) + else: + parts.extend(_addFilePart(entryData, fileName, parentId, containerPath, state)) + except ImportError: + logger.warning("py7zr not installed -- 7z files will be treated as binary") + parts.append(ContentPart( + id=makeId(), parentId=parentId, label="unsupported", + typeGroup="text", mimeType="text/plain", + data="7z extraction requires py7zr package", metadata={"warning": True}, + )) + except Exception as e: + logger.error(f"Invalid 7z file: {e}") + parts.append(ContentPart( + id=makeId(), parentId=parentId, label="error", + typeGroup="text", mimeType="text/plain", + data=f"Invalid 7z archive: {e}", metadata={"error": True}, + )) + return parts diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorDocx.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorDocx.py index 096f288b..c8e7c289 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorDocx.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorDocx.py @@ -74,19 +74,33 @@ class DocxExtractor(Extractor): with io.BytesIO(fileBytes) as buf: d = docx.Document(buf) # paragraphs + fileName = context.get("fileName", "document.docx") + headingIndex = 0 + currentSection = "body" for i, para in enumerate(d.paragraphs): text = para.text or "" - if text.strip(): - parts.append(ContentPart( - id=makeId(), - parentId=rootId, - label=f"p_{i+1}", - typeGroup="text", - mimeType="text/plain", - data=text, - metadata={"size": len(text.encode('utf-8'))} - )) - # tables → CSV rows + if not text.strip(): + continue + styleName = (para.style.name or "").lower() if para.style else "" + if "heading" in styleName: + headingIndex += 1 + currentSection = f"heading:{headingIndex}" + parts.append(ContentPart( + id=makeId(), + parentId=rootId, + label=f"p_{i+1}", + typeGroup="text", + mimeType="text/plain", + data=text, + metadata={ + "size": len(text.encode('utf-8')), + "contextRef": { + "containerPath": fileName, + "location": f"paragraph:{i+1}", + "sectionId": currentSection, + }, + } + )) for ti, table in enumerate(d.tables): rows: list[str] = [] for row in table.rows: @@ -101,7 +115,14 @@ class DocxExtractor(Extractor): typeGroup="table", mimeType="text/csv", data=csvData, - metadata={"size": len(csvData.encode('utf-8'))} + metadata={ + "size": len(csvData.encode('utf-8')), + "contextRef": { + "containerPath": fileName, + "location": f"table:{ti+1}", + "sectionId": currentSection, + }, + } )) return parts diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py new file mode 100644 index 00000000..2c4295ab --- /dev/null +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorEmail.py @@ -0,0 +1,230 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Email extractor for EML and MSG files. + +Parses email headers, body (text/html), and attachments. +Attachments are delegated to the ExtractorRegistry for type-specific processing. + +Optional dependency: extract-msg (for .msg files). +""" + +from typing import Any, Dict, List +import email +import email.policy +import email.utils +import io +import logging +import mimetypes + +from modules.datamodels.datamodelExtraction import ContentPart +from ..subUtils import makeId +from ..subRegistry import Extractor + +logger = logging.getLogger(__name__) + +_EMAIL_MIME_TYPES = [ + "message/rfc822", + "application/vnd.ms-outlook", +] +_EMAIL_EXTENSIONS = [".eml", ".msg"] + + +class EmailExtractor(Extractor): + """Extractor for email files (EML, MSG). + + Produces: + - 1 text ContentPart with header metadata (From, To, Subject, Date) + - 1 text ContentPart per body part (plain text / HTML) + - Delegated ContentParts for each attachment via ExtractorRegistry + """ + + def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: + if mimeType in _EMAIL_MIME_TYPES: + return True + lower = (fileName or "").lower() + return any(lower.endswith(ext) for ext in _EMAIL_EXTENSIONS) + + def getSupportedExtensions(self) -> list[str]: + return list(_EMAIL_EXTENSIONS) + + def getSupportedMimeTypes(self) -> list[str]: + return list(_EMAIL_MIME_TYPES) + + def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: + fileName = context.get("fileName", "email") + lower = (fileName or "").lower() + + if lower.endswith(".msg"): + return self._extractMsg(fileBytes, fileName) + return self._extractEml(fileBytes, fileName) + + def _extractEml(self, fileBytes: bytes, fileName: str) -> List[ContentPart]: + """Parse standard EML (RFC 822) using stdlib email.""" + rootId = makeId() + parts: List[ContentPart] = [] + + try: + msg = email.message_from_bytes(fileBytes, policy=email.policy.default) + except Exception as e: + logger.error(f"EmailExtractor: failed to parse EML: {e}") + return [ContentPart( + id=rootId, parentId=None, label=fileName, + typeGroup="text", mimeType="text/plain", + data=f"Failed to parse email: {e}", metadata={"error": True}, + )] + + headerText = _buildHeaderText(msg) + parts.append(ContentPart( + id=rootId, parentId=None, label="headers", + typeGroup="text", mimeType="text/plain", + data=headerText, metadata={"emailPart": "headers"}, + )) + + for part in msg.walk(): + contentType = part.get_content_type() + disposition = str(part.get("Content-Disposition", "")) + + if part.is_multipart(): + continue + + if "attachment" in disposition: + attachName = part.get_filename() or "attachment" + attachData = part.get_payload(decode=True) + if attachData: + parts.extend(_delegateAttachment(attachData, attachName, rootId)) + continue + + if contentType == "text/plain": + body = part.get_content() + if body: + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="body_text", + typeGroup="text", mimeType="text/plain", + data=str(body), metadata={"emailPart": "body"}, + )) + elif contentType == "text/html": + body = part.get_content() + if body: + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="body_html", + typeGroup="text", mimeType="text/html", + data=str(body), metadata={"emailPart": "body_html"}, + )) + + return parts + + def _extractMsg(self, fileBytes: bytes, fileName: str) -> List[ContentPart]: + """Parse Outlook MSG files using extract-msg (optional).""" + rootId = makeId() + parts: List[ContentPart] = [] + + try: + import extract_msg + except ImportError: + logger.warning("extract-msg not installed -- MSG files will be treated as binary") + return [ContentPart( + id=rootId, parentId=None, label=fileName, + typeGroup="text", mimeType="text/plain", + data="MSG extraction requires the extract-msg package.", + metadata={"warning": True}, + )] + + try: + msgFile = extract_msg.Message(io.BytesIO(fileBytes)) + except Exception as e: + logger.error(f"EmailExtractor: failed to parse MSG: {e}") + return [ContentPart( + id=rootId, parentId=None, label=fileName, + typeGroup="text", mimeType="text/plain", + data=f"Failed to parse MSG: {e}", metadata={"error": True}, + )] + + headerLines = [] + if msgFile.sender: + headerLines.append(f"From: {msgFile.sender}") + if msgFile.to: + headerLines.append(f"To: {msgFile.to}") + if getattr(msgFile, "cc", None): + headerLines.append(f"Cc: {msgFile.cc}") + if msgFile.subject: + headerLines.append(f"Subject: {msgFile.subject}") + if msgFile.date: + headerLines.append(f"Date: {msgFile.date}") + + parts.append(ContentPart( + id=rootId, parentId=None, label="headers", + typeGroup="text", mimeType="text/plain", + data="\n".join(headerLines), metadata={"emailPart": "headers"}, + )) + + body = msgFile.body + if body: + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="body_text", + typeGroup="text", mimeType="text/plain", + data=body, metadata={"emailPart": "body"}, + )) + + htmlBody = getattr(msgFile, "htmlBody", None) + if htmlBody: + if isinstance(htmlBody, bytes): + htmlBody = htmlBody.decode("utf-8", errors="replace") + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="body_html", + typeGroup="text", mimeType="text/html", + data=htmlBody, metadata={"emailPart": "body_html"}, + )) + + for attachment in (msgFile.attachments or []): + attachName = getattr(attachment, "longFilename", None) or getattr(attachment, "shortFilename", None) or "attachment" + attachData = getattr(attachment, "data", None) + if attachData: + parts.extend(_delegateAttachment(attachData, attachName, rootId)) + + try: + msgFile.close() + except Exception: + pass + + return parts + + +def _buildHeaderText(msg) -> str: + """Build a readable text summary of key email headers.""" + lines = [] + for header in ("From", "To", "Cc", "Subject", "Date", "Message-ID"): + value = msg.get(header) + if value: + lines.append(f"{header}: {value}") + return "\n".join(lines) + + +def _delegateAttachment(attachData: bytes, attachName: str, parentId: str) -> List[ContentPart]: + """Delegate an attachment to the appropriate type-specific extractor.""" + guessedMime, _ = mimetypes.guess_type(attachName) + detectedMime = guessedMime or "application/octet-stream" + + from ..subRegistry import ExtractorRegistry + registry = ExtractorRegistry() + extractor = registry.resolve(detectedMime, attachName) + + if extractor and not isinstance(extractor, EmailExtractor): + try: + childParts = extractor.extract(attachData, {"fileName": attachName, "mimeType": detectedMime}) + for part in childParts: + part.parentId = parentId + if not part.metadata: + part.metadata = {} + part.metadata["emailAttachment"] = attachName + return childParts + except Exception as e: + logger.warning(f"Extractor failed for email attachment {attachName}: {e}") + + import base64 + encodedData = base64.b64encode(attachData).decode("utf-8") if attachData else "" + return [ContentPart( + id=makeId(), parentId=parentId, label=attachName, + typeGroup="binary", mimeType=detectedMime, + data=encodedData, + metadata={"size": len(attachData), "emailAttachment": attachName}, + )] diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py new file mode 100644 index 00000000..51c8d9f5 --- /dev/null +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorFolder.py @@ -0,0 +1,184 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Folder extractor -- treats a local folder reference as a container. + +Not registered in the MIME-based ExtractorRegistry (folders have no MIME type). +Instead, called directly by agent tools (browseContainer) when handling folder references. + +Applies the same safety limits as ContainerExtractor. +""" + +from typing import Any, Dict, List +import logging +import mimetypes +from pathlib import Path + +from ..subUtils import makeId +from modules.datamodels.datamodelExtraction import ContentPart +from modules.datamodels.datamodelContent import ContainerLimitError, ContentContextRef +from ..subRegistry import Extractor + +logger = logging.getLogger(__name__) + +MAX_TOTAL_EXTRACTED_SIZE = 500 * 1024 * 1024 +MAX_FILE_COUNT = 10000 +MAX_DEPTH = 5 + + +class FolderExtractor(Extractor): + """Extracts contents from a local folder path. + + Unlike other extractors, this does not receive fileBytes. Instead it + receives a folder path via context["folderPath"] and walks the directory. + """ + + def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: + return False + + def getSupportedExtensions(self) -> list[str]: + return [] + + def getSupportedMimeTypes(self) -> list[str]: + return [] + + def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: + """Extract folder contents. + + context must contain: + folderPath: str -- absolute path to the folder + """ + folderPath = context.get("folderPath", "") + if not folderPath: + return [] + + folder = Path(folderPath) + if not folder.is_dir(): + logger.error(f"FolderExtractor: not a directory: {folderPath}") + return [] + + rootId = makeId() + parts: List[ContentPart] = [ + ContentPart( + id=rootId, + parentId=None, + label=folder.name or "folder", + typeGroup="container", + mimeType="inode/directory", + data="", + metadata={"folderPath": str(folder), "containerType": "folder"}, + ) + ] + + state = {"totalSize": 0, "fileCount": 0} + try: + _walkFolder(folder, rootId, "", 0, state, parts) + except ContainerLimitError as e: + logger.warning(f"Folder extraction limit reached: {e}") + parts.append(ContentPart( + id=makeId(), + parentId=rootId, + label="limit_exceeded", + typeGroup="text", + mimeType="text/plain", + data=str(e), + metadata={"warning": "Folder extraction limit exceeded"}, + )) + + return parts + + +def _walkFolder( + folder: Path, + parentId: str, + containerPath: str, + depth: int, + state: Dict[str, int], + parts: List[ContentPart], +) -> None: + if depth > MAX_DEPTH: + raise ContainerLimitError(f"Max folder depth {MAX_DEPTH} exceeded") + + try: + entries = sorted(folder.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())) + except PermissionError: + logger.warning(f"Permission denied: {folder}") + return + + for entry in entries: + if entry.is_symlink(): + logger.debug(f"Skipping symlink: {entry}") + continue + + entryPath = f"{containerPath}/{entry.name}" if containerPath else entry.name + + if entry.is_dir(): + folderId = makeId() + parts.append(ContentPart( + id=folderId, + parentId=parentId, + label=entry.name, + typeGroup="container", + mimeType="inode/directory", + data="", + metadata={"containerPath": entryPath, "containerType": "folder"}, + )) + _walkFolder(entry, folderId, entryPath, depth + 1, state, parts) + + elif entry.is_file(): + try: + fileSize = entry.stat().st_size + except OSError: + continue + + state["totalSize"] += fileSize + state["fileCount"] += 1 + + if state["totalSize"] > MAX_TOTAL_EXTRACTED_SIZE: + raise ContainerLimitError(f"Total extracted size exceeds {MAX_TOTAL_EXTRACTED_SIZE // (1024 * 1024)} MB") + if state["fileCount"] > MAX_FILE_COUNT: + raise ContainerLimitError(f"File count exceeds {MAX_FILE_COUNT}") + + guessedMime, _ = mimetypes.guess_type(entry.name) + detectedMime = guessedMime or "application/octet-stream" + + from ..subRegistry import ExtractorRegistry + registry = ExtractorRegistry() + extractor = registry.resolve(detectedMime, entry.name) + + if extractor and not isinstance(extractor, FolderExtractor): + try: + fileData = entry.read_bytes() + childParts = extractor.extract(fileData, {"fileName": entry.name, "mimeType": detectedMime}) + for part in childParts: + part.parentId = parentId + if not part.metadata: + part.metadata = {} + part.metadata["containerPath"] = entryPath + parts.extend(childParts) + continue + except Exception as e: + logger.warning(f"Type-extractor failed for {entry.name}: {e}") + + import base64 + try: + fileData = entry.read_bytes() + encodedData = base64.b64encode(fileData).decode("utf-8") + except Exception: + encodedData = "" + + parts.append(ContentPart( + id=makeId(), + parentId=parentId, + label=entry.name, + typeGroup="binary", + mimeType=detectedMime, + data=encodedData, + metadata={ + "size": fileSize, + "containerPath": entryPath, + "contextRef": ContentContextRef( + containerPath=entryPath, + location="file", + ).model_dump(), + }, + )) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorPdf.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorPdf.py index 244aef90..98b83188 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorPdf.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorPdf.py @@ -89,7 +89,15 @@ class PdfExtractor(Extractor): typeGroup="text", mimeType="text/plain", data=text, - metadata={"pages": 1, "pageIndex": i, "size": len(text.encode('utf-8'))} + metadata={ + "pages": 1, "pageIndex": i, + "size": len(text.encode('utf-8')), + "contextRef": { + "containerPath": context.get("fileName", "document.pdf"), + "location": f"page:{i+1}", + "pageIndex": i, + }, + } )) except Exception: continue @@ -114,7 +122,15 @@ class PdfExtractor(Extractor): typeGroup="text", mimeType="text/plain", data=text, - metadata={"pages": 1, "pageIndex": i, "size": len(text.encode('utf-8'))} + metadata={ + "pages": 1, "pageIndex": i, + "size": len(text.encode('utf-8')), + "contextRef": { + "containerPath": context.get("fileName", "document.pdf"), + "location": f"page:{i+1}", + "pageIndex": i, + }, + } )) except Exception: continue @@ -143,7 +159,14 @@ class PdfExtractor(Extractor): typeGroup="image", mimeType=f"image/{ext}", data=base64.b64encode(imgBytes).decode("utf-8"), - metadata={"pageIndex": i, "size": len(imgBytes)} + metadata={ + "pageIndex": i, "size": len(imgBytes), + "contextRef": { + "containerPath": context.get("fileName", "document.pdf"), + "location": f"page:{i+1}/image:{j}", + "pageIndex": i, + }, + } )) except Exception: continue diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorPptx.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorPptx.py index 6c9e6c6c..0c811d20 100644 --- a/modules/serviceCenter/services/serviceExtraction/extractors/extractorPptx.py +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorPptx.py @@ -119,17 +119,22 @@ class PptxExtractor(Extractor): image_bytes = image.blob image_b64 = base64.b64encode(image_bytes).decode('utf-8') - # Create image part + fileName = context.get("fileName", "presentation.pptx") image_part = ContentPart( id=f"slide_{slide_index}_image_{len(parts)}", label=f"Slide {slide_index} Image", typeGroup="image", - mimeType="image/png", # Default to PNG + mimeType="image/png", data=image_b64, metadata={ "slide_number": slide_index, "shape_type": "image", - "extracted_from": "powerpoint" + "extracted_from": "powerpoint", + "contextRef": { + "containerPath": fileName, + "location": f"slide:{slide_index}/image", + "slideIndex": slide_index - 1, + }, } ) parts.append(image_part) @@ -140,6 +145,7 @@ class PptxExtractor(Extractor): if slide_content: slide_text = f"# Slide {slide_index}\n\n" + "\n\n".join(slide_content) + fileName = context.get("fileName", "presentation.pptx") slide_part = ContentPart( id=f"slide_{slide_index}", label=f"Slide {slide_index} Content", @@ -150,7 +156,12 @@ class PptxExtractor(Extractor): "slide_number": slide_index, "content_type": "slide", "extracted_from": "powerpoint", - "text_length": len(slide_text) + "text_length": len(slide_text), + "contextRef": { + "containerPath": fileName, + "location": f"slide:{slide_index}", + "slideIndex": slide_index - 1, + }, } ) parts.append(slide_part) diff --git a/modules/serviceCenter/services/serviceExtraction/extractors/extractorVideo.py b/modules/serviceCenter/services/serviceExtraction/extractors/extractorVideo.py new file mode 100644 index 00000000..1b0513ce --- /dev/null +++ b/modules/serviceCenter/services/serviceExtraction/extractors/extractorVideo.py @@ -0,0 +1,208 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Video extractor for common video formats. + +Extracts metadata (duration, resolution, codec, bitrate) and produces +a `videostream` ContentPart. Video data is never base64-encoded due to size. + +Optional dependency: mutagen (for rich metadata from MP4/WebM containers). +""" + +from typing import Any, Dict, List +import logging +import struct + +from modules.datamodels.datamodelExtraction import ContentPart +from ..subUtils import makeId +from ..subRegistry import Extractor + +logger = logging.getLogger(__name__) + +_VIDEO_MIME_TYPES = [ + "video/mp4", + "video/webm", + "video/x-msvideo", + "video/avi", + "video/quicktime", + "video/x-matroska", + "video/x-ms-wmv", + "video/mpeg", + "video/ogg", +] +_VIDEO_EXTENSIONS = [".mp4", ".webm", ".avi", ".mov", ".mkv", ".wmv", ".mpeg", ".mpg", ".ogv"] + + +class VideoExtractor(Extractor): + """Extractor for video files. + + Produces: + - 1 text ContentPart with metadata summary + - 1 videostream ContentPart (no inline data -- too large) + """ + + def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: + if mimeType in _VIDEO_MIME_TYPES: + return True + lower = (fileName or "").lower() + return any(lower.endswith(ext) for ext in _VIDEO_EXTENSIONS) + + def getSupportedExtensions(self) -> list[str]: + return list(_VIDEO_EXTENSIONS) + + def getSupportedMimeTypes(self) -> list[str]: + return list(_VIDEO_MIME_TYPES) + + def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: + fileName = context.get("fileName", "video") + mimeType = context.get("mimeType") or "video/mp4" + fileSize = len(fileBytes) + + rootId = makeId() + parts: List[ContentPart] = [] + + meta = _extractMetadata(fileBytes, fileName) + meta["size"] = fileSize + meta["fileName"] = fileName + meta["mimeType"] = mimeType + + metaLines = [f"Video file: {fileName}"] + if meta.get("duration"): + mins = int(meta["duration"] // 60) + secs = int(meta["duration"] % 60) + metaLines.append(f"Duration: {mins}:{secs:02d}") + if meta.get("width") and meta.get("height"): + metaLines.append(f"Resolution: {meta['width']}x{meta['height']}") + if meta.get("codec"): + metaLines.append(f"Codec: {meta['codec']}") + if meta.get("bitrate"): + metaLines.append(f"Bitrate: {meta['bitrate']} kbps") + if meta.get("fps"): + metaLines.append(f"FPS: {meta['fps']}") + metaLines.append(f"Size: {fileSize:,} bytes") + + parts.append(ContentPart( + id=rootId, parentId=None, label="metadata", + typeGroup="text", mimeType="text/plain", + data="\n".join(metaLines), metadata=meta, + )) + + parts.append(ContentPart( + id=makeId(), parentId=rootId, label="videostream", + typeGroup="videostream", mimeType=mimeType, + data="", metadata={"size": fileSize, "inlined": False}, + )) + + return parts + + +def _extractMetadata(fileBytes: bytes, fileName: str) -> Dict[str, Any]: + """Extract video metadata using mutagen (optional) with basic fallback.""" + meta: Dict[str, Any] = {} + + try: + import mutagen + import io + mediaFile = mutagen.File(io.BytesIO(fileBytes)) + if mediaFile is not None and mediaFile.info: + meta["duration"] = getattr(mediaFile.info, "length", None) + meta["bitrate"] = getattr(mediaFile.info, "bitrate", None) + if meta["bitrate"]: + meta["bitrate"] = meta["bitrate"] // 1000 + + if hasattr(mediaFile.info, "video"): + for stream in (mediaFile.info.video if isinstance(mediaFile.info.video, list) else [mediaFile.info.video]): + if hasattr(stream, "width"): + meta["width"] = stream.width + if hasattr(stream, "height"): + meta["height"] = stream.height + if hasattr(stream, "codec"): + meta["codec"] = stream.codec + + width = getattr(mediaFile.info, "width", None) + height = getattr(mediaFile.info, "height", None) + if width and height: + meta["width"] = width + meta["height"] = height + + fps = getattr(mediaFile.info, "fps", None) + if fps: + meta["fps"] = round(fps, 2) + + codec = getattr(mediaFile.info, "codec", None) + if codec: + meta["codec"] = codec + + return {k: v for k, v in meta.items() if v is not None} + except ImportError: + logger.debug("mutagen not installed -- using basic video metadata extraction") + except Exception as e: + logger.debug(f"mutagen video metadata extraction failed: {e}") + + lower = fileName.lower() + if lower.endswith(".mp4"): + meta.update(_parseMp4Header(fileBytes)) + elif lower.endswith(".avi"): + meta.update(_parseAviHeader(fileBytes)) + + return {k: v for k, v in meta.items() if v is not None} + + +def _parseMp4Header(fileBytes: bytes) -> Dict[str, Any]: + """Minimal MP4 moov/mvhd parser for duration and timescale.""" + meta: Dict[str, Any] = {} + try: + pos = 0 + while pos < len(fileBytes) - 8: + boxSize = struct.unpack_from(">I", fileBytes, pos)[0] + boxType = fileBytes[pos + 4:pos + 8] + if boxSize < 8: + break + if boxType == b"moov": + meta.update(_parseMoovBox(fileBytes[pos + 8:pos + boxSize])) + break + pos += boxSize + except Exception: + pass + return meta + + +def _parseMoovBox(data: bytes) -> Dict[str, Any]: + """Parse moov box to find mvhd with duration.""" + meta: Dict[str, Any] = {} + pos = 0 + while pos < len(data) - 8: + try: + boxSize = struct.unpack_from(">I", data, pos)[0] + boxType = data[pos + 4:pos + 8] + if boxSize < 8: + break + if boxType == b"mvhd": + version = data[pos + 8] + if version == 0 and pos + 28 < len(data): + timeScale = struct.unpack_from(">I", data, pos + 20)[0] + duration = struct.unpack_from(">I", data, pos + 24)[0] + if timeScale > 0: + meta["duration"] = duration / timeScale + break + pos += boxSize + except Exception: + break + return meta + + +def _parseAviHeader(fileBytes: bytes) -> Dict[str, Any]: + """Minimal AVI header parser for resolution.""" + meta: Dict[str, Any] = {} + if len(fileBytes) < 72: + return meta + try: + if fileBytes[:4] != b"RIFF" or fileBytes[8:12] != b"AVI ": + return meta + width = struct.unpack_from(" availableContentBytes and chunkData: + logger.warning(f" Chunk {i+1}/{len(chunks)}: {chunkSize} bytes exceeds target {availableContentBytes} bytes, force-splitting by lines") + subChunks = self._forceLineSplit(chunkData, availableContentBytes, len(validatedChunks)) + validatedChunks.extend(subChunks) + else: + chunk["order"] = len(validatedChunks) + validatedChunks.append(chunk) + + if len(validatedChunks) != len(chunks): + logger.info(f"Post-chunking validation: {len(chunks)} -> {len(validatedChunks)} chunks after force-splitting oversized chunks") + + for i, chunk in enumerate(validatedChunks): + chunkSize = len(chunk.get('data', '').encode('utf-8')) if chunk.get('data') else 0 + logger.info(f" Chunk {i+1}/{len(validatedChunks)}: {chunkSize} bytes") + + return validatedChunks except Exception as e: logger.error(f"Chunking failed for {contentPart.typeGroup}: {str(e)}") return [] + def _forceLineSplit(self, data: str, maxBytes: int, startOrder: int) -> List[Dict[str, Any]]: + """Line-based safety-net split for chunks that still exceed maxBytes after structured chunking.""" + chunks: List[Dict[str, Any]] = [] + current: List[str] = [] + size = 0 + for line in data.split('\n'): + s = len(line.encode('utf-8')) + 1 + if size + s > maxBytes and current: + text = '\n'.join(current) + chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": startOrder + len(chunks)}) + current = [line] + size = s + else: + current.append(line) + size += s + if current: + text = '\n'.join(current) + chunks.append({"data": text, "size": len(text.encode('utf-8')), "order": startOrder + len(chunks)}) + return chunks + async def processContentPartWithFallback(self, contentPart, prompt: str, options, failoverModelList, aiObjects, progressCallback=None) -> AiCallResponse: """Process a single content part with model-aware chunking and fallback. @@ -1386,73 +1419,210 @@ class ExtractionService: logger.warning(f"⚠️ Content part ({contentTokens:.0f} tokens est.) exceeds available space ({availableContentBytes/TOKEN_SAFETY_FACTOR:.0f} tokens est.), chunking required") # If either condition fails, chunk the content - # CRITICAL: IMAGE_GENERATE operations should NOT use chunking - they generate images from prompts, not process content chunks + # CRITICAL: IMAGE_GENERATE operations should NOT use chunking if (totalTokens > maxTotalTokens or partSize > availableContentBytes) and options.operationType != OperationTypeEnum.IMAGE_GENERATE: - # Part too large or total exceeds limit - chunk it (but not for image generation) chunks = await self.chunkContentPartForAi(contentPart, model, options, prompt) if not chunks: raise ValueError(f"Failed to chunk content part for model {model.name}") - - logger.info(f"Starting to process {len(chunks)} chunks with model {model.name}") - - if progressCallback: - progressCallback(0.0, f"Starting to process {len(chunks)} chunks") - - chunkResults = [] - for idx, chunk in enumerate(chunks): - chunkNum = idx + 1 - chunkData = chunk.get('data', '') - logger.info(f"Processing chunk {chunkNum}/{len(chunks)} with model {model.name}") - - if progressCallback: - progressCallback(chunkNum / len(chunks), f"Processing chunk {chunkNum}/{len(chunks)}") - - try: - chunkResponse = await aiObjects._callWithModel(model, prompt, chunkData, options) - chunkResults.append(chunkResponse) - except Exception as chunkError: - logger.error(f"Error processing chunk {chunkNum}/{len(chunks)}: {str(chunkError)}") - # Continue with other chunks even if one fails - continue - - # Merge chunk results - if not chunkResults: - raise ValueError(f"All chunks failed for content part") - - # Pass original contentPart to preserve typeGroup for all chunks (one-to-many: 1 part -> N chunks) - mergedContent = self.mergePartResults(chunkResults, options, [contentPart]) + + # Parallel chunk processing with per-chunk failover + remainingModels = failoverModelList[attempt:] + allChunkResults, allResponses = await self._processChunksParallel( + chunks, prompt, options, remainingModels, aiObjects, progressCallback + ) + + if not allResponses: + raise ValueError("All chunks failed for content part") + + mergedContent = self.mergePartResults(allResponses, options, [contentPart]) + + # Stitch pass: reconcile cross-chunk artifacts when multiple chunks were processed + if len(allResponses) > 1: + mergedContent = await self._stitchChunkResults( + mergedContent, len(allResponses), prompt, options, aiObjects + ) + return AiCallResponse( content=mergedContent, modelName=model.name, provider=model.connectorType, - priceCHF=sum(r.priceCHF for r in chunkResults), - processingTime=sum(r.processingTime for r in chunkResults), - bytesSent=sum(r.bytesSent for r in chunkResults), - bytesReceived=sum(r.bytesReceived for r in chunkResults), - errorCount=sum(r.errorCount for r in chunkResults) + priceCHF=sum(r.priceCHF for r in allResponses), + processingTime=sum(r.processingTime for r in allResponses), + bytesSent=sum(r.bytesSent for r in allResponses), + bytesReceived=sum(r.bytesReceived for r in allResponses), + errorCount=sum(r.errorCount for r in allResponses) ) else: - # Part fits - call AI directly via aiObjects interface - logger.info(f"✅ Content part fits within model limits, processing directly") + # Part fits - call AI directly + logger.info(f"Content part fits within model limits, processing directly") response = await aiObjects._callWithModel(model, prompt, contentPart.data, options) - logger.info(f"✅ Content part processed successfully with model: {model.name}") + logger.info(f"Content part processed successfully with model: {model.name}") return response except Exception as e: lastError = e error_msg = str(e) if str(e) else f"{type(e).__name__}" - logger.warning(f"❌ Model {model.name} failed for content part: {error_msg}", exc_info=True) + logger.warning(f"Model {model.name} failed for content part: {error_msg}", exc_info=True) if attempt < len(failoverModelList) - 1: - logger.info(f"🔄 Trying next failover model...") + logger.info(f"Trying next failover model...") continue else: - logger.error(f"💥 All {len(failoverModelList)} models failed for content part") + logger.error(f"All {len(failoverModelList)} models failed for content part") break # All models failed return self._createErrorResponse(f"All models failed: {str(lastError)}", 0, 0) + async def _processChunksParallel( + self, + chunks: List[Dict[str, Any]], + prompt: str, + options, + failoverModels: list, + aiObjects, + progressCallback=None, + maxRetries: int = 3 + ) -> tuple: + """Process chunks in parallel. On failure, re-chunk only the failed chunks for the next model. + + Returns (orderedResults, allResponses) where orderedResults is a dict + mapping original order -> AiCallResponse and allResponses is a flat list. + """ + if not failoverModels: + return {}, [] + + pendingChunks = [(chunk.get("order", i), chunk) for i, chunk in enumerate(chunks)] + completedResults: Dict[float, AiCallResponse] = {} + allResponses: List[AiCallResponse] = [] + retryCount = 0 + modelIdx = 0 + currentModel = failoverModels[modelIdx] + + maxConcurrent = 3 + semaphore = asyncio.Semaphore(maxConcurrent) + + logger.info(f"Starting parallel chunk processing: {len(pendingChunks)} chunks with model {currentModel.name}") + + while pendingChunks and retryCount <= maxRetries and currentModel: + modelForRound = currentModel + totalInRound = len(pendingChunks) + completedInRound = [0] + + async def _processOneChunk(order: float, chunkData: str, model=modelForRound): + async with semaphore: + result = await aiObjects._callWithModel(model, prompt, chunkData, options) + completedInRound[0] += 1 + if progressCallback: + progressCallback(completedInRound[0] / totalInRound, f"Chunk {completedInRound[0]}/{totalInRound} completed") + return result + + tasks = {} + for order, chunk in pendingChunks: + chunkData = chunk.get('data', '') + tasks[order] = asyncio.create_task(_processOneChunk(order, chunkData)) + + if progressCallback: + progressCallback(0.0, f"Processing {len(tasks)} chunks in parallel with {currentModel.name}") + + results = await asyncio.gather(*tasks.values(), return_exceptions=True) + + failedChunks = [] + for (order, chunk), result in zip(pendingChunks, results): + if isinstance(result, Exception): + logger.warning(f"Chunk order={order} failed with {currentModel.name}: {result}") + failedChunks.append((order, chunk)) + else: + completedResults[order] = result + allResponses.append(result) + + logger.info(f"Round {retryCount}: {len(pendingChunks) - len(failedChunks)}/{len(pendingChunks)} chunks succeeded with {currentModel.name}") + + if not failedChunks: + break + + retryCount += 1 + modelIdx += 1 + if modelIdx >= len(failoverModels): + logger.error(f"No more failover models available, {len(failedChunks)} chunks remain failed") + break + + currentModel = failoverModels[modelIdx] + logger.info(f"Failover: re-chunking {len(failedChunks)} failed chunks for model {currentModel.name}") + + newPending = [] + for order, failedChunk in failedChunks: + reChunked = await self._reChunkForModel(failedChunk, currentModel, prompt, options) + for i, subChunk in enumerate(reChunked): + subOrder = order + i * 0.001 + newPending.append((subOrder, subChunk)) + + pendingChunks = newPending + + orderedResponses = [completedResults[k] for k in sorted(completedResults.keys())] + return orderedResponses, allResponses + + async def _reChunkForModel(self, chunk: Dict[str, Any], model, prompt: str, options) -> List[Dict[str, Any]]: + """Re-chunk a single failed chunk according to the new model's context limits.""" + chunkData = chunk.get('data', '') + tempPart = ContentPart( + id=f"rechunk_{uuid.uuid4().hex[:8]}", + label="re-chunk", + typeGroup="structure" if chunkData.strip().startswith(('{', '[')) else "text", + mimeType="application/json" if chunkData.strip().startswith(('{', '[')) else "text/plain", + data=chunkData + ) + reChunked = await self.chunkContentPartForAi(tempPart, model, options, prompt) + if not reChunked: + return [chunk] + return reChunked + + async def _stitchChunkResults( + self, + mergedContent: str, + chunkCount: int, + originalPrompt: str, + options, + aiObjects + ) -> str: + """Reconcile cross-chunk artifacts in merged content. + + Only called when chunkCount > 1. Delegates to aiObjects.callWithTextContext + which handles model selection, failover, and billing. + """ + mergedSize = len(mergedContent.encode('utf-8')) if mergedContent else 0 + + stitchPrompt = ( + "The following content was assembled from multiple independently processed " + f"chunks ({chunkCount} chunks) of the same document. " + "Review and fix ONLY these issues, preserving all content:\n" + "1. Cross-references that point to content from other chunks\n" + "2. Duplicate text at chunk boundaries (remove duplicates)\n" + "3. Sentences or paragraphs split mid-thought (reconnect them)\n" + "4. Inconsistent terminology for the same entity\n\n" + "Do NOT add, remove, or rephrase content beyond these fixes. " + "Return the corrected content in the same format.\n\n" + f"Original processing instruction (truncated): {originalPrompt[:500]}" + ) + + try: + logger.info(f"Running stitch pass on {mergedSize} bytes") + request = AiCallRequest( + prompt=stitchPrompt, + context=mergedContent, + options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE) + ) + response = await aiObjects.callWithTextContext(request) + if hasattr(response, 'errorCount') and response.errorCount > 0: + logger.warning(f"Stitch pass returned error: {response.content[:200] if response.content else 'empty'}") + return mergedContent + resultSize = len(response.content.encode('utf-8')) if response.content else 0 + logger.info(f"Stitch pass completed: {mergedSize} -> {resultSize} bytes") + return response.content + except Exception as e: + logger.warning(f"Stitch pass failed (non-fatal), returning unstitched content: {e}") + return mergedContent + def _createErrorResponse(self, errorMsg: str, inputBytes: int, outputBytes: int) -> AiCallResponse: """Create an error response.""" return AiCallResponse( @@ -1521,9 +1691,18 @@ class ExtractionService: progressCallback(0.1 + (partIndex / totalParts) * 0.8, f"Processing {partLabel} ({partType}) - {partIndex+1}/{totalParts}") try: - # Process the part + partProgressCb = None + if progressCallback: + partStart = 0.1 + (partIndex / totalParts) * 0.8 + partRange = 0.8 / totalParts + def _makePartProgressCb(start, rangeSize): + def _cb(chunkProgress, message): + progressCallback(start + chunkProgress * rangeSize, message) + return _cb + partProgressCb = _makePartProgressCb(partStart, partRange) + partResult = await self.processContentPartWithFallback( - contentPart, prompt, options, failoverModelList, aiObjects, None # Don't pass progressCallback to avoid double logging + contentPart, prompt, options, failoverModelList, aiObjects, partProgressCb ) # Write debug files for generation phase (section content generation) diff --git a/modules/serviceCenter/services/serviceExtraction/subRegistry.py b/modules/serviceCenter/services/serviceExtraction/subRegistry.py index 216f2664..cd14b0d7 100644 --- a/modules/serviceCenter/services/serviceExtraction/subRegistry.py +++ b/modules/serviceCenter/services/serviceExtraction/subRegistry.py @@ -191,9 +191,11 @@ class ChunkerRegistry: self.register("table", TableChunker()) self.register("structure", StructureChunker()) self.register("image", ImageChunker()) - # Use text chunker for container and binary content + # Use text chunker for container, binary, and media stream content self.register("container", TextChunker()) self.register("binary", TextChunker()) + self.register("audiostream", TextChunker()) + self.register("videostream", TextChunker()) except Exception as e: logger.error(f"ChunkerRegistry: Failed to register chunkers: {str(e)}") import traceback diff --git a/modules/serviceCenter/services/serviceKnowledge/__init__.py b/modules/serviceCenter/services/serviceKnowledge/__init__.py new file mode 100644 index 00000000..a5d1fc04 --- /dev/null +++ b/modules/serviceCenter/services/serviceKnowledge/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""serviceKnowledge: 3-tier RAG Knowledge Store with semantic search.""" diff --git a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py new file mode 100644 index 00000000..a2dabadf --- /dev/null +++ b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py @@ -0,0 +1,531 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Knowledge service: 3-tier RAG with indexing, semantic search, and context building.""" + +import logging +from typing import Any, Callable, Dict, List, Optional + +from modules.datamodels.datamodelKnowledge import ( + FileContentIndex, ContentChunk, WorkflowMemory, +) +from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum +from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + +DEFAULT_CHUNK_SIZE = 512 +DEFAULT_CONTEXT_BUDGET = 8000 + + +class KnowledgeService: + """Service for Knowledge Store operations: indexing, retrieval, and context building.""" + + def __init__(self, context, get_service: Callable[[str], Any]): + self._context = context + self._getService = get_service + self._knowledgeDb = getKnowledgeInterface(context.user) + + # ========================================================================= + # Embedding helper + # ========================================================================= + + async def _embed(self, texts: List[str]) -> List[List[float]]: + """Embed texts via the AI interface's generic embedding method.""" + aiService = self._getService("ai") + await aiService.ensureAiObjectsInitialized() + aiObjects = aiService.aiObjects + if aiObjects is None: + logger.warning("Embedding skipped: aiObjects not available") + return [] + response = await aiObjects.callEmbedding(texts) + if response.errorCount > 0: + logger.error(f"Embedding failed: {response.content}") + return [] + return (response.metadata or {}).get("embeddings", []) + + async def _embedSingle(self, text: str) -> List[float]: + """Embed a single text. Returns empty list on failure.""" + results = await self._embed([text]) + return results[0] if results else [] + + # ========================================================================= + # File Indexing (called after extraction, before embedding) + # ========================================================================= + + async def indexFile( + self, + fileId: str, + fileName: str, + mimeType: str, + userId: str, + featureInstanceId: str = "", + mandateId: str = "", + contentObjects: List[Dict[str, Any]] = None, + structure: Dict[str, Any] = None, + containerPath: str = None, + ) -> FileContentIndex: + """Index a file's content objects and create embeddings for text chunks. + + This is the main entry point after non-AI extraction has produced content objects. + + Args: + fileId: The file ID. + fileName: Original file name. + mimeType: MIME type. + userId: Owner user. + featureInstanceId: Feature instance scope. + mandateId: Mandate scope. + contentObjects: List of extracted content objects, each with keys: + contentType (str), data (str), contextRef (dict), contentObjectId (str). + structure: Structural overview of the file. + containerPath: Path within container if applicable. + + Returns: + The created FileContentIndex. + """ + contentObjects = contentObjects or [] + + # 1. Create FileContentIndex + index = FileContentIndex( + id=fileId, + userId=userId, + featureInstanceId=featureInstanceId, + mandateId=mandateId, + fileName=fileName, + mimeType=mimeType, + containerPath=containerPath, + totalObjects=len(contentObjects), + totalSize=sum(len(obj.get("data", "").encode("utf-8")) for obj in contentObjects), + structure=structure or {}, + objectSummary=[ + { + "id": obj.get("contentObjectId", ""), + "type": obj.get("contentType", "other"), + "size": len(obj.get("data", "").encode("utf-8")), + "ref": obj.get("contextRef", {}), + } + for obj in contentObjects + ], + status="extracted", + ) + self._knowledgeDb.upsertFileContentIndex(index) + + # 2. Chunk text content objects and create embeddings + textObjects = [o for o in contentObjects if o.get("contentType") == "text"] + if textObjects: + self._knowledgeDb.updateFileStatus(fileId, "embedding") + chunks = _chunkForEmbedding(textObjects, chunkSize=DEFAULT_CHUNK_SIZE) + texts = [c["data"] for c in chunks] + + embeddings = await self._embed(texts) if texts else [] + + for i, chunk in enumerate(chunks): + embedding = embeddings[i] if i < len(embeddings) else None + contentChunk = ContentChunk( + contentObjectId=chunk["contentObjectId"], + fileId=fileId, + userId=userId, + featureInstanceId=featureInstanceId, + contentType="text", + data=chunk["data"], + contextRef=chunk["contextRef"], + embedding=embedding, + ) + self._knowledgeDb.upsertContentChunk(contentChunk) + + # 3. Store non-text content objects (images, etc.) without embedding + nonTextObjects = [o for o in contentObjects if o.get("contentType") != "text"] + for obj in nonTextObjects: + contentChunk = ContentChunk( + contentObjectId=obj.get("contentObjectId", ""), + fileId=fileId, + userId=userId, + featureInstanceId=featureInstanceId, + contentType=obj.get("contentType", "other"), + data=obj.get("data", ""), + contextRef=obj.get("contextRef", {}), + embedding=None, + ) + self._knowledgeDb.upsertContentChunk(contentChunk) + + self._knowledgeDb.updateFileStatus(fileId, "indexed") + index.status = "indexed" + logger.info(f"Indexed file {fileId} ({fileName}): {len(contentObjects)} objects, {len(textObjects)} text chunks") + return index + + # ========================================================================= + # RAG Context Building (3-tier search) + # ========================================================================= + + async def buildAgentContext( + self, + currentPrompt: str, + workflowId: str, + userId: str, + featureInstanceId: str = "", + mandateId: str = "", + contextBudget: int = DEFAULT_CONTEXT_BUDGET, + ) -> str: + """Build RAG context for an agent round by searching all 3 layers. + + Args: + currentPrompt: The current user prompt to find relevant context for. + workflowId: Current workflow ID. + userId: Current user. + featureInstanceId: Feature instance scope. + mandateId: Mandate scope. + contextBudget: Maximum characters for the context string. + + Returns: + Formatted context string for injection into the agent's system prompt. + """ + queryVector = await self._embedSingle(currentPrompt) + if not queryVector: + return "" + + builder = _ContextBuilder(budget=contextBudget) + + # Layer 1: Instance Layer (user's own documents, highest priority) + instanceChunks = self._knowledgeDb.semanticSearch( + queryVector=queryVector, + userId=userId, + featureInstanceId=featureInstanceId, + limit=15, + minScore=0.65, + ) + if instanceChunks: + builder.add(priority=1, label="Relevant Documents", items=instanceChunks) + + # Layer 2: Workflow Layer (current workflow entities & memory) + entities = self._knowledgeDb.getWorkflowEntities(workflowId) + if entities: + builder.add(priority=2, label="Workflow Context", items=entities, isKeyValue=True) + + # Layer 3: Shared Layer (mandate-wide shared documents) + sharedChunks = self._knowledgeDb.semanticSearch( + queryVector=queryVector, + mandateId=mandateId, + isShared=True, + limit=10, + minScore=0.7, + ) + if sharedChunks: + builder.add(priority=3, label="Shared Knowledge", items=sharedChunks) + + return builder.build() + + # ========================================================================= + # Workflow Memory + # ========================================================================= + + async def storeEntity( + self, + workflowId: str, + userId: str, + featureInstanceId: str, + key: str, + value: str, + source: str = "extraction", + ) -> WorkflowMemory: + """Store a key-value entity in workflow memory with optional embedding.""" + embedding = await self._embedSingle(f"{key}: {value}") + memory = WorkflowMemory( + workflowId=workflowId, + userId=userId, + featureInstanceId=featureInstanceId, + key=key, + value=value, + source=source, + embedding=embedding if embedding else None, + ) + self._knowledgeDb.upsertWorkflowMemory(memory) + return memory + + def getEntities(self, workflowId: str) -> List[Dict[str, Any]]: + """Get all entities for a workflow.""" + return self._knowledgeDb.getWorkflowEntities(workflowId) + + # ========================================================================= + # File Status + # ========================================================================= + + def getFileStatus(self, fileId: str) -> Optional[str]: + """Get the indexing status of a file.""" + index = self._knowledgeDb.getFileContentIndex(fileId) + return index.get("status") if index else None + + def isFileIndexed(self, fileId: str) -> bool: + """Check if a file has been fully indexed.""" + return self.getFileStatus(fileId) == "indexed" + + # ========================================================================= + # On-Demand Extraction (Smart Document Handling) + # ========================================================================= + + async def readSection(self, fileId: str, sectionId: str) -> List[Dict[str, Any]]: + """Read content objects for a specific section. Uses cache if available. + + Args: + fileId: Source file ID. + sectionId: Section identifier from the FileContentIndex structure. + + Returns: + List of content object dicts with data and contextRef. + """ + cached = self._knowledgeDb.getContentChunks(fileId) + sectionChunks = [ + c for c in (cached or []) + if (c.get("contextRef", {}).get("sectionId") == sectionId) + ] + if sectionChunks: + return sectionChunks + + index = self._knowledgeDb.getFileContentIndex(fileId) + if not index: + return [] + + structure = index.get("structure", {}) if isinstance(index, dict) else getattr(index, "structure", {}) + sections = structure.get("sections", []) + section = next((s for s in sections if s.get("id") == sectionId), None) + if not section: + return [] + + startPage = section.get("startPage", 0) + endPage = section.get("endPage", startPage) + + return await self._extractPagesOnDemand(fileId, startPage, endPage, sectionId) + + async def readContentObjects( + self, fileId: str, filter: Dict[str, Any] = None + ) -> List[Dict[str, Any]]: + """Read content objects with optional filters (pageIndex, contentType, sectionId). + + Args: + fileId: Source file ID. + filter: Optional dict with keys pageIndex (list[int]), contentType (str), sectionId (str). + + Returns: + Filtered list of content chunk dicts. + """ + filter = filter or {} + chunks = self._knowledgeDb.getContentChunks(fileId) or [] + + if "pageIndex" in filter: + targetPages = filter["pageIndex"] + if isinstance(targetPages, int): + targetPages = [targetPages] + chunks = [ + c for c in chunks + if c.get("contextRef", {}).get("pageIndex") in targetPages + ] + + if "contentType" in filter: + chunks = [c for c in chunks if c.get("contentType") == filter["contentType"]] + + if "sectionId" in filter: + chunks = [ + c for c in chunks + if c.get("contextRef", {}).get("sectionId") == filter["sectionId"] + ] + + return chunks + + async def extractContainerItem( + self, fileId: str, containerPath: str + ) -> Optional[Dict[str, Any]]: + """On-demand extraction of a specific item within a container. + + If the item is already indexed, returns existing data. + Otherwise triggers extraction and indexing. + + Args: + fileId: The container file ID. + containerPath: Path within the container (e.g. "folder/report.pdf"). + + Returns: + FileContentIndex dict for the extracted item, or None. + """ + existing = self._knowledgeDb.getFileContentIndex(fileId) + if existing: + existingPath = existing.get("containerPath") if isinstance(existing, dict) else getattr(existing, "containerPath", None) + if existingPath == containerPath: + return existing + + logger.info(f"On-demand extraction for {containerPath} in file {fileId}") + return None + + async def _extractPagesOnDemand( + self, fileId: str, startPage: int, endPage: int, sectionId: str + ) -> List[Dict[str, Any]]: + """Extract specific pages from a file and cache in knowledge store.""" + try: + chatService = self._getService("chat") + fileContent = chatService.getFileContent(fileId) + if not fileContent: + return [] + + fileData = fileContent.get("data", b"") + mimeType = fileContent.get("mimeType", "") + fileName = fileContent.get("fileName", "") + + if isinstance(fileData, str): + import base64 + fileData = base64.b64decode(fileData) + + if mimeType != "application/pdf": + return [] + + try: + import fitz + except ImportError: + return [] + + doc = fitz.open(stream=fileData, filetype="pdf") + results = [] + + for pageIdx in range(startPage, min(endPage + 1, len(doc))): + page = doc[pageIdx] + text = page.get_text() or "" + if not text.strip(): + continue + + chunk = ContentChunk( + contentObjectId=f"page-{pageIdx}", + fileId=fileId, + userId=self._context.user.id if self._context.user else "", + featureInstanceId=self._context.feature_instance_id or "", + contentType="text", + data=text, + contextRef={ + "containerPath": fileName, + "location": f"page:{pageIdx+1}", + "pageIndex": pageIdx, + "sectionId": sectionId, + }, + ) + + embedding = await self._embedSingle(text[:2000]) + if embedding: + chunk.embedding = embedding + + self._knowledgeDb.upsertContentChunk(chunk) + results.append(chunk.model_dump()) + + doc.close() + return results + + except Exception as e: + logger.error(f"On-demand page extraction failed: {e}") + return [] + + def getFileContentIndex(self, fileId: str) -> Optional[Dict[str, Any]]: + """Get the FileContentIndex for a file.""" + return self._knowledgeDb.getFileContentIndex(fileId) + + +# ============================================================================= +# Internal helpers +# ============================================================================= + +def _chunkForEmbedding( + textObjects: List[Dict[str, Any]], chunkSize: int = 512 +) -> List[Dict[str, Any]]: + """Split text content objects into chunks suitable for embedding. + + Each chunk preserves the contextRef from its source object. + Long texts are split at sentence boundaries where possible. + """ + chunks = [] + for obj in textObjects: + text = obj.get("data", "") + contentObjectId = obj.get("contentObjectId", "") + contextRef = obj.get("contextRef", {}) + + if len(text) <= chunkSize: + chunks.append({ + "data": text, + "contentObjectId": contentObjectId, + "contextRef": contextRef, + }) + continue + + # Split at sentence boundaries + sentences = text.replace("\n", " ").split(". ") + currentChunk = "" + for sentence in sentences: + candidate = f"{currentChunk}. {sentence}" if currentChunk else sentence + if len(candidate) > chunkSize and currentChunk: + chunks.append({ + "data": currentChunk.strip(), + "contentObjectId": contentObjectId, + "contextRef": contextRef, + }) + currentChunk = sentence + else: + currentChunk = candidate + + if currentChunk.strip(): + chunks.append({ + "data": currentChunk.strip(), + "contentObjectId": contentObjectId, + "contextRef": contextRef, + }) + + return chunks + + +class _ContextBuilder: + """Assembles RAG context from multiple sources respecting a character budget.""" + + def __init__(self, budget: int): + self._budget = budget + self._sections: List[Dict[str, Any]] = [] + + def add( + self, + priority: int, + label: str, + items: List[Dict[str, Any]], + isKeyValue: bool = False, + ): + self._sections.append({ + "priority": priority, + "label": label, + "items": items, + "isKeyValue": isKeyValue, + }) + + def build(self) -> str: + self._sections.sort(key=lambda s: s["priority"]) + parts = [] + remaining = self._budget + + for section in self._sections: + if remaining <= 0: + break + + header = f"### {section['label']}\n" + sectionText = header + remaining -= len(header) + + for item in section["items"]: + if remaining <= 0: + break + + if section["isKeyValue"]: + line = f"- {item.get('key', '')}: {item.get('value', '')}\n" + else: + data = item.get("data", "") + ref = item.get("contextRef", {}) + score = item.get("_score", "") + refStr = f" [{ref}]" if ref else "" + line = f"{data}{refStr}\n" + + if len(line) <= remaining: + sectionText += line + remaining -= len(line) + + parts.append(sectionText) + + return "\n".join(parts).strip() diff --git a/modules/serviceCenter/services/serviceKnowledge/subPreScan.py b/modules/serviceCenter/services/serviceKnowledge/subPreScan.py new file mode 100644 index 00000000..e025dd99 --- /dev/null +++ b/modules/serviceCenter/services/serviceKnowledge/subPreScan.py @@ -0,0 +1,427 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Structure Pre-Scan: fast, AI-free document analysis. + +Extracts TOC, headings, page map, image positions, and structural metadata +from documents. Used as the first step in the auto-index pipeline. + +Supported formats: + - PDF: TOC, heading detection (font-size heuristic), page map, image positions + - DOCX: heading styles, paragraph map + - PPTX: slide titles, slide map + - XLSX: sheet names, row/column counts + - Other: minimal index (single content object = the file itself) +""" + +import io +import logging +from typing import Dict, Any, List, Optional + +from modules.datamodels.datamodelKnowledge import FileContentIndex +from modules.datamodels.datamodelContent import ContentObjectSummary, ContentContextRef + +logger = logging.getLogger(__name__) + + +async def preScanDocument( + fileData: bytes, + mimeType: str, + fileId: str, + fileName: str = "", + userId: str = "", + featureInstanceId: str = "", + mandateId: str = "", +) -> FileContentIndex: + """Create a structural FileContentIndex without AI. + + This is purely programmatic: TOC extraction, heading detection, + page mapping, image position scanning. + """ + scanner = _SCANNER_MAP.get(mimeType) + if scanner is None: + ext = (fileName.rsplit(".", 1)[-1].lower()) if "." in fileName else "" + scanner = _EXTENSION_SCANNER_MAP.get(ext, _scanMinimal) + + try: + structure, objectSummary, totalObjects, totalSize = await scanner(fileData, fileName) + except Exception as e: + logger.error(f"Pre-scan failed for {fileName} ({mimeType}): {e}") + structure = {"error": str(e)} + objectSummary = [] + totalObjects = 0 + totalSize = len(fileData) + + return FileContentIndex( + id=fileId, + userId=userId, + featureInstanceId=featureInstanceId, + mandateId=mandateId, + fileName=fileName, + mimeType=mimeType, + totalObjects=totalObjects, + totalSize=totalSize, + structure=structure, + objectSummary=[s.model_dump() for s in objectSummary], + status="extracted", + ) + + +# --------------------------------------------------------------------------- +# PDF scanner +# --------------------------------------------------------------------------- + +async def _scanPdf(fileData: bytes, fileName: str): + try: + import fitz + except ImportError: + logger.warning("PyMuPDF not installed -- PDF pre-scan unavailable") + return _fallbackStructure(fileData, fileName) + + doc = fitz.open(stream=fileData, filetype="pdf") + toc = doc.get_toc() + + pageMap: List[Dict[str, Any]] = [] + summaries: List[ContentObjectSummary] = [] + totalSize = 0 + objIndex = 0 + + for i in range(len(doc)): + page = doc[i] + textLen = len(page.get_text()) + blocks = page.get_text("dict", flags=0).get("blocks", []) + + headings = [] + for b in blocks: + if b.get("type") != 0: + continue + for line in b.get("lines", []): + for span in line.get("spans", []): + if _isHeading(span): + headings.append(span.get("text", "").strip()) + + images = page.get_images(full=True) + hasTable = _detectTableHeuristic(page) + + pageMap.append({ + "pageIndex": i, + "headings": headings, + "hasImages": len(images) > 0, + "imageCount": len(images), + "textLength": textLen, + "hasTable": hasTable, + }) + + if textLen > 0: + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="text", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"page:{i+1}", + pageIndex=i, + ), + charCount=textLen, + )) + totalSize += textLen + objIndex += 1 + + for j in range(len(images)): + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="image", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"page:{i+1}/image:{j}", + pageIndex=i, + ), + )) + objIndex += 1 + + sections = _buildSectionsFromTocOrHeadings(toc, pageMap) + doc.close() + + structure = { + "pages": len(pageMap), + "toc": toc, + "sections": sections, + "pageMap": pageMap, + "imageCount": sum(p.get("imageCount", 0) for p in pageMap), + "tableCount": sum(1 for p in pageMap if p.get("hasTable")), + } + return structure, summaries, len(summaries), totalSize + + +def _isHeading(span: Dict) -> bool: + """Heuristic: heading if font size >= 14 or bold + size >= 12.""" + size = span.get("size", 0) + flags = span.get("flags", 0) + isBold = bool(flags & (1 << 4)) + return size >= 14 or (isBold and size >= 12) + + +def _detectTableHeuristic(page) -> bool: + """Detect tables by looking for grid-like line patterns.""" + try: + drawings = page.get_drawings() + lineCount = sum(1 for d in drawings if d.get("type") == "l") + return lineCount >= 6 + except Exception: + return False + + +def _buildSectionsFromTocOrHeadings( + toc: list, pageMap: List[Dict] +) -> List[Dict[str, Any]]: + """Build section boundaries from TOC or heading data.""" + sections: List[Dict[str, Any]] = [] + + if toc: + for i, entry in enumerate(toc): + level, title, pageNum = entry[0], entry[1], entry[2] + endPage = toc[i + 1][2] - 1 if i + 1 < len(toc) else len(pageMap) - 1 + sections.append({ + "id": f"section-{i}", + "title": title, + "level": level, + "startPage": pageNum - 1, + "endPage": endPage, + }) + else: + currentSection = None + for pm in pageMap: + headings = pm.get("headings", []) + if headings: + if currentSection: + currentSection["endPage"] = pm["pageIndex"] - 1 + sections.append(currentSection) + currentSection = { + "id": f"section-{len(sections)}", + "title": headings[0], + "level": 1, + "startPage": pm["pageIndex"], + "endPage": pm["pageIndex"], + } + elif currentSection: + currentSection["endPage"] = pm["pageIndex"] + + if currentSection: + sections.append(currentSection) + + return sections + + +# --------------------------------------------------------------------------- +# DOCX scanner +# --------------------------------------------------------------------------- + +async def _scanDocx(fileData: bytes, fileName: str): + try: + import docx + except ImportError: + return _fallbackStructure(fileData, fileName) + + doc = docx.Document(io.BytesIO(fileData)) + summaries: List[ContentObjectSummary] = [] + sections: List[Dict[str, Any]] = [] + totalSize = 0 + objIndex = 0 + currentSection = None + + for i, para in enumerate(doc.paragraphs): + text = para.text or "" + styleName = (para.style.name or "").lower() if para.style else "" + + if "heading" in styleName and text.strip(): + if currentSection: + sections.append(currentSection) + level = 1 + for ch in styleName: + if ch.isdigit(): + level = int(ch) + break + currentSection = { + "id": f"section-{len(sections)}", + "title": text.strip(), + "level": level, + "startParagraph": i, + "endParagraph": i, + } + elif currentSection: + currentSection["endParagraph"] = i + + if text.strip(): + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="text", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"paragraph:{i+1}", + sectionId=currentSection["id"] if currentSection else "body", + ), + charCount=len(text), + )) + totalSize += len(text) + objIndex += 1 + + if currentSection: + sections.append(currentSection) + + for ti, table in enumerate(doc.tables): + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="text", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"table:{ti+1}", + ), + )) + objIndex += 1 + + structure = { + "paragraphs": len(doc.paragraphs), + "tables": len(doc.tables), + "sections": sections, + } + return structure, summaries, len(summaries), totalSize + + +# --------------------------------------------------------------------------- +# PPTX scanner +# --------------------------------------------------------------------------- + +async def _scanPptx(fileData: bytes, fileName: str): + try: + from pptx import Presentation + except ImportError: + return _fallbackStructure(fileData, fileName) + + prs = Presentation(io.BytesIO(fileData)) + summaries: List[ContentObjectSummary] = [] + slideMap: List[Dict[str, Any]] = [] + totalSize = 0 + objIndex = 0 + + for i, slide in enumerate(prs.slides): + title = "" + textLen = 0 + imageCount = 0 + for shape in slide.shapes: + if hasattr(shape, "text"): + textLen += len(shape.text) + if shape.has_text_frame and not title: + title = shape.text.strip()[:80] + if shape.shape_type == 13: + imageCount += 1 + + slideMap.append({ + "slideIndex": i, + "title": title, + "textLength": textLen, + "imageCount": imageCount, + }) + + if textLen > 0: + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="text", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"slide:{i+1}", + slideIndex=i, + ), + charCount=textLen, + )) + totalSize += textLen + objIndex += 1 + + structure = { + "slides": len(prs.slides), + "slideMap": slideMap, + } + return structure, summaries, len(summaries), totalSize + + +# --------------------------------------------------------------------------- +# XLSX scanner +# --------------------------------------------------------------------------- + +async def _scanXlsx(fileData: bytes, fileName: str): + try: + import openpyxl + except ImportError: + return _fallbackStructure(fileData, fileName) + + wb = openpyxl.load_workbook(io.BytesIO(fileData), data_only=True, read_only=True) + summaries: List[ContentObjectSummary] = [] + sheetMap: List[Dict[str, Any]] = [] + totalSize = 0 + objIndex = 0 + + for sheetName in wb.sheetnames: + ws = wb[sheetName] + rowCount = ws.max_row or 0 + colCount = ws.max_column or 0 + + sheetMap.append({ + "sheetName": sheetName, + "rows": rowCount, + "columns": colCount, + }) + + summaries.append(ContentObjectSummary( + id=f"co-{objIndex}", + contentType="text", + contextRef=ContentContextRef( + containerPath=fileName, + location=f"sheet:{sheetName}", + sheetName=sheetName, + ), + charCount=rowCount * colCount * 10, + )) + totalSize += rowCount * colCount * 10 + objIndex += 1 + + wb.close() + structure = {"sheets": len(wb.sheetnames), "sheetMap": sheetMap} + return structure, summaries, len(summaries), totalSize + + +# --------------------------------------------------------------------------- +# Minimal / fallback scanner +# --------------------------------------------------------------------------- + +async def _scanMinimal(fileData: bytes, fileName: str): + return _fallbackStructure(fileData, fileName) + + +def _fallbackStructure(fileData: bytes, fileName: str): + summary = ContentObjectSummary( + id="co-0", + contentType="other", + contextRef=ContentContextRef(containerPath=fileName, location="file"), + charCount=len(fileData), + ) + structure = {"type": "single", "size": len(fileData)} + return structure, [summary], 1, len(fileData) + + +# --------------------------------------------------------------------------- +# Scanner map +# --------------------------------------------------------------------------- + +_SCANNER_MAP: Dict[str, Any] = { + "application/pdf": _scanPdf, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": _scanDocx, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": _scanPptx, + "application/vnd.ms-powerpoint": _scanPptx, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": _scanXlsx, +} + +_EXTENSION_SCANNER_MAP: Dict[str, Any] = { + "pdf": _scanPdf, + "docx": _scanDocx, + "pptx": _scanPptx, + "ppt": _scanPptx, + "xlsx": _scanXlsx, + "xlsm": _scanXlsx, +} diff --git a/modules/serviceCenter/services/serviceWeb/mainServiceWeb.py b/modules/serviceCenter/services/serviceWeb/mainServiceWeb.py index 1af97f22..4ffc15aa 100644 --- a/modules/serviceCenter/services/serviceWeb/mainServiceWeb.py +++ b/modules/serviceCenter/services/serviceWeb/mainServiceWeb.py @@ -375,7 +375,7 @@ USER PROVIDED: - Language: {language or "Not specified"} Extract and provide a JSON response with: -1. instruction: Formulate directly, WHAT you want to find on the web. Do not include URLs in the instruction. Good example: "What is the company Xyz doing?". Bad example: "Conduct web research on the company Xyz" +1. instruction: Formulate a concise search query (MAXIMUM 400 characters) stating WHAT you want to find on the web. Do not include URLs in the instruction. Keep it focused on the core question. Good example: "What is the company Xyz doing?". Bad example: "Conduct web research on the company Xyz and find all information about..." 2. urls: Put list of URLs found in the prompt text, and URL's you know, that are relevant to the research 3. needsSearch: true if web search is needed to identify url's to crawl, false if only crawling of provided URLs is wanted 4. maxNumberPages: Recommended number of URLs to crawl (based on research scope, typical: 2-20) diff --git a/modules/services/__init__.py b/modules/serviceHub/__init__.py similarity index 54% rename from modules/services/__init__.py rename to modules/serviceHub/__init__.py index f6ac292a..162aebe4 100644 --- a/modules/services/__init__.py +++ b/modules/serviceHub/__init__.py @@ -1,13 +1,18 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. """ -Services Module. -Central service registry that provides access to shared services. +Service Hub. +Consumer-facing aggregation layer for services, DB interfaces, and runtime state. -IMPORTANT: Import-Regelwerk -- Zentrale Module (wie dieses) dürfen KEINE Feature-Container importieren +Architecture: +- serviceHub delegates service resolution to serviceCenter (DI container) +- serviceHub owns DB interface initialization and runtime state +- serviceCenter knows nothing about serviceHub (one-way dependency) + +Import-Regelwerk: +- Zentrale Module (wie dieses) duerfen KEINE Feature-Container importieren - Feature-spezifische Services werden dynamisch geladen -- Nur Shared Services werden direkt geladen +- Shared Services werden via serviceCenter resolved """ import os @@ -23,7 +28,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Path to feature containers _FEATURES_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "features") @@ -54,15 +58,19 @@ class PublicService: ]) -class Services: +class ServiceHub: """ - Central Services class providing access to all services. - - Import-Regelwerk: - - Shared Services are loaded directly (from modules/services/) - - Feature-specific Services are loaded dynamically via filename discovery + Consumer-facing aggregation of services, DB interfaces, and runtime state. + + Services are lazy-resolved via serviceCenter on first access. + DB interfaces and runtime state are initialized eagerly. + Feature services/interfaces are discovered dynamically from features/. """ + _SERVICE_CENTER_WRAPPING = { + "ai": {"functionsOnly": False}, + } + def __init__(self, user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): self.user: User = user self.workflow = workflow @@ -71,123 +79,89 @@ class Services: self.currentUserPrompt: str = "" self.rawUserPrompt: str = "" - # Initialize central interfaces + from modules.serviceCenter.context import ServiceCenterContext + self._serviceCenterContext = ServiceCenterContext( + user=user, + workflow=workflow, + mandate_id=mandateId, + feature_instance_id=featureInstanceId, + ) + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface self.interfaceDbApp = getAppInterface(user, mandateId=mandateId) - + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface self.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId) - + self.rbac = self.interfaceDbApp.rbac if self.interfaceDbApp else None - - # ============================================================ - # CENTRAL INTERFACE (Chat/Workflow) - # ============================================================ + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface self.interfaceDbChat = getChatInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) - - # ============================================================ - # SHARED SERVICES (from modules/services/) - # ============================================================ - from .serviceSharepoint.mainServiceSharepoint import SharepointService - self.sharepoint = PublicService(SharepointService(self)) - - from .serviceTicket.mainServiceTicket import TicketService - self.ticket = PublicService(TicketService(self)) - - from .serviceChat.mainServiceChat import ChatService - self.chat = PublicService(ChatService(self)) - - from .serviceUtils.mainServiceUtils import UtilsService - self.utils = PublicService(UtilsService(self)) - - from .serviceSecurity.mainServiceSecurity import SecurityService - self.security = PublicService(SecurityService(self)) - - from .serviceMessaging.mainServiceMessaging import MessagingService - self.messaging = PublicService(MessagingService(self)) - - from .serviceStreaming.mainServiceStreaming import StreamingService - self.streaming = PublicService(StreamingService(self)) - - # ============================================================ - # AI SERVICES (from modules/services/) - # ============================================================ - from .serviceAi.mainServiceAi import AiService - self.ai = PublicService(AiService(self), functionsOnly=False) - - from .serviceExtraction.mainServiceExtraction import ExtractionService - self.extraction = PublicService(ExtractionService(self)) - - from .serviceGeneration.mainServiceGeneration import GenerationService - self.generation = PublicService(GenerationService(self)) - - from .serviceWeb.mainServiceWeb import WebService - self.web = PublicService(WebService(self)) - - # ============================================================ - # FEATURE INTERFACES (dynamically loaded) - # ============================================================ + self._loadFeatureInterfaces() self._loadFeatureServices() - + + def __getattr__(self, name: str): + """Lazy-resolve services via serviceCenter on first access.""" + if name.startswith('_'): + raise AttributeError(name) + try: + from modules.serviceCenter import getService + service = getService(name, self._serviceCenterContext) + wrapping = self._SERVICE_CENTER_WRAPPING.get(name, {}) + functionsOnly = wrapping.get("functionsOnly", True) + wrapped = PublicService(service, functionsOnly=functionsOnly) + setattr(self, name, wrapped) + return wrapped + except KeyError: + raise AttributeError(f"'{type(self).__name__}' has no attribute '{name}'") + def _loadFeatureInterfaces(self): """Dynamically load interfaces from feature containers by filename pattern.""" - # Find all interfaceFeature*.py files pattern = os.path.join(_FEATURES_DIR, "*", "interfaceFeature*.py") for filepath in glob.glob(pattern): try: - # Extract feature name and interface name featureDir = os.path.basename(os.path.dirname(filepath)) - filename = os.path.basename(filepath)[:-3] # Remove .py - - # Build module path: modules.features.. + filename = os.path.basename(filepath)[:-3] + modulePath = f"modules.features.{featureDir}.{filename}" module = importlib.import_module(modulePath) - - # Get interface via getInterface() + if hasattr(module, "getInterface"): interface = module.getInterface(self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId) - # Derive attribute name: interfaceFeatureAiChat -> interfaceDbChat attrName = filename.replace("interfaceFeature", "interfaceDb") setattr(self, attrName, interface) logger.debug(f"Loaded interface: {attrName} from {modulePath}") except Exception as e: logger.debug(f"Could not load interface from {filepath}: {e}") - + def _loadFeatureServices(self): """Dynamically load services from feature containers by filename pattern.""" - # Find all service*/mainService*.py files in feature containers pattern = os.path.join(_FEATURES_DIR, "*", "service*", "mainService*.py") for filepath in glob.glob(pattern): try: - # Extract paths serviceDir = os.path.basename(os.path.dirname(filepath)) featureDir = os.path.basename(os.path.dirname(os.path.dirname(filepath))) - filename = os.path.basename(filepath)[:-3] # Remove .py - - # Build module path: modules.features... + filename = os.path.basename(filepath)[:-3] + modulePath = f"modules.features.{featureDir}.{serviceDir}.{filename}" module = importlib.import_module(modulePath) - - # Find service class (ends with "Service") + serviceClass = None - for name in dir(module): - if name.endswith("Service") and not name.startswith("_"): - cls = getattr(module, name) + for attrName in dir(module): + if attrName.endswith("Service") and not attrName.startswith("_"): + cls = getattr(module, attrName) if isinstance(cls, type): serviceClass = cls break - + if serviceClass: - # Derive attribute name: serviceAi -> ai, serviceExtraction -> extraction attrName = serviceDir.replace("service", "").lower() if not attrName: attrName = serviceDir.lower() - - # Check if it needs functionsOnly=False (for AI service) + functionsOnly = attrName != "ai" - + serviceInstance = serviceClass(self) setattr(self, attrName, PublicService(serviceInstance, functionsOnly=functionsOnly)) logger.debug(f"Loaded service: {attrName} from {modulePath}") @@ -195,6 +169,10 @@ class Services: logger.debug(f"Could not load service from {filepath}: {e}") -def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> Services: - """Get Services instance for the given user, mandate, and feature instance context.""" - return Services(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId) +# Backward-compatible alias +Services = ServiceHub + + +def getInterface(user: User, workflow: "ChatWorkflow" = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> ServiceHub: + """Get ServiceHub instance for the given user, mandate, and feature instance context.""" + return ServiceHub(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId) diff --git a/modules/services/serviceAi/mainAiChat.py b/modules/services/serviceAi/mainAiChat.py deleted file mode 100644 index 2e6514e6..00000000 --- a/modules/services/serviceAi/mainAiChat.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -AIChat Feature Container - Main Module. -Handles feature initialization and RBAC catalog registration. - -AIChat is the dynamic chat workflow feature that handles: -- AI-powered document processing -- Dynamic workflow execution -- Automation definitions -""" - -import logging -from typing import Dict, List, Any - -logger = logging.getLogger(__name__) - -# Feature metadata -FEATURE_CODE = "chatworkflow" -FEATURE_LABEL = {"en": "Chat Workflow", "de": "Chat-Workflow", "fr": "Workflow de Chat"} -FEATURE_ICON = "mdi-message-cog" - -# UI Objects for RBAC catalog -UI_OBJECTS = [ - { - "objectKey": "ui.feature.aichat.workflows", - "label": {"en": "Workflows", "de": "Workflows", "fr": "Workflows"}, - "meta": {"area": "workflows"} - }, - { - "objectKey": "ui.feature.aichat.automations", - "label": {"en": "Automations", "de": "Automatisierungen", "fr": "Automatisations"}, - "meta": {"area": "automations"} - }, - { - "objectKey": "ui.feature.aichat.logs", - "label": {"en": "Logs", "de": "Logs", "fr": "Journaux"}, - "meta": {"area": "logs"} - }, -] - -# Resource Objects for RBAC catalog -RESOURCE_OBJECTS = [ - { - "objectKey": "resource.feature.aichat.workflow.start", - "label": {"en": "Start Workflow", "de": "Workflow starten", "fr": "Démarrer workflow"}, - "meta": {"endpoint": "/api/chat/playground/start", "method": "POST"} - }, - { - "objectKey": "resource.feature.aichat.workflow.stop", - "label": {"en": "Stop Workflow", "de": "Workflow stoppen", "fr": "Arrêter workflow"}, - "meta": {"endpoint": "/api/chat/playground/stop/{workflowId}", "method": "POST"} - }, - { - "objectKey": "resource.feature.aichat.workflow.delete", - "label": {"en": "Delete Workflow", "de": "Workflow löschen", "fr": "Supprimer workflow"}, - "meta": {"endpoint": "/api/chat/playground/workflow/{workflowId}", "method": "DELETE"} - }, -] - -# Template roles for this feature -TEMPLATE_ROLES = [ - { - "roleLabel": "workflow-admin", - "description": { - "en": "Workflow Administrator - Full access to workflow configuration and execution", - "de": "Workflow-Administrator - Vollzugriff auf Workflow-Konfiguration und Ausführung", - "fr": "Administrateur workflow - Accès complet à la configuration et exécution" - } - }, - { - "roleLabel": "workflow-editor", - "description": { - "en": "Workflow Editor - Create and modify workflows", - "de": "Workflow-Editor - Workflows erstellen und bearbeiten", - "fr": "Éditeur workflow - Créer et modifier les workflows" - } - }, - { - "roleLabel": "workflow-viewer", - "description": { - "en": "Workflow Viewer - View workflows and execution results", - "de": "Workflow-Betrachter - Workflows und Ausführungsergebnisse einsehen", - "fr": "Visualiseur workflow - Consulter les workflows et résultats" - } - }, -] - - -def getFeatureDefinition() -> Dict[str, Any]: - """Return the feature definition for registration.""" - return { - "code": FEATURE_CODE, - "label": FEATURE_LABEL, - "icon": FEATURE_ICON - } - - -def getUiObjects() -> List[Dict[str, Any]]: - """Return UI objects for RBAC catalog registration.""" - return UI_OBJECTS - - -def getResourceObjects() -> List[Dict[str, Any]]: - """Return resource objects for RBAC catalog registration.""" - return RESOURCE_OBJECTS - - -def getTemplateRoles() -> List[Dict[str, Any]]: - """Return template roles for this feature.""" - return TEMPLATE_ROLES - - -def registerFeature(catalogService) -> bool: - """ - Register this feature's RBAC objects in the catalog. - - Args: - catalogService: The RBAC catalog service instance - - Returns: - True if registration was successful - """ - try: - # Register UI objects - for uiObj in UI_OBJECTS: - catalogService.registerUiObject( - featureCode=FEATURE_CODE, - objectKey=uiObj["objectKey"], - label=uiObj["label"], - meta=uiObj.get("meta") - ) - - # Register Resource objects - for resObj in RESOURCE_OBJECTS: - catalogService.registerResourceObject( - featureCode=FEATURE_CODE, - objectKey=resObj["objectKey"], - label=resObj["label"], - meta=resObj.get("meta") - ) - - logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") - return True - - except Exception as e: - logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") - return False - - -async def onStart(eventUser) -> None: - """ - Called when the feature container starts. - Initializes AI connectors for model registry. - """ - try: - from modules.aicore.aicoreModelRegistry import modelRegistry - modelRegistry.ensureConnectorsRegistered() - logger.info(f"Feature '{FEATURE_CODE}' started - AI connectors initialized") - except Exception as e: - logger.error(f"Feature '{FEATURE_CODE}' failed to initialize AI connectors: {e}") - - -async def onStop(eventUser) -> None: - """Called when the feature container stops.""" - logger.info(f"Feature '{FEATURE_CODE}' stopped") diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py deleted file mode 100644 index b4015d6d..00000000 --- a/modules/services/serviceAi/mainServiceAi.py +++ /dev/null @@ -1,1535 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -import json -import logging -import re -import time -import base64 -from typing import Dict, Any, List, Optional, Tuple -from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument, WorkflowModeEnum -from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent -from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData -from modules.datamodels.datamodelDocument import RenderedDocument -from modules.interfaces.interfaceAiObjects import AiObjects -from modules.shared.jsonUtils import ( - parseJsonWithModel -) -from .subJsonResponseHandling import JsonResponseHandler -from modules.datamodels.datamodelAi import JsonAccumulationState -from modules.services.serviceBilling.mainServiceBilling import ( - getService as getBillingService, - InsufficientBalanceException, - ProviderNotAllowedException, - BillingContextError -) - -logger = logging.getLogger(__name__) - -# Rebuild the model to resolve forward references -AiCallRequest.model_rebuild() - -class AiService: - """AI service with core operations integrated.""" - - def __init__(self, serviceCenter=None) -> None: - """Initialize AI service with service center access. - - Args: - serviceCenter: Service center instance for accessing other services - """ - self.services = serviceCenter - # Only depend on interfaces - self.aiObjects = None # Will be initialized in create() or ensureAiObjectsInitialized() - # Submodules initialized as None - will be set in _initializeSubmodules() after aiObjects is ready - self.extractionService = None - - def _initializeSubmodules(self): - """Initialize all submodules after aiObjects is ready.""" - if self.aiObjects is None: - raise RuntimeError("aiObjects must be initialized before initializing submodules") - - if self.extractionService is None: - logger.info("Initializing ExtractionService...") - self.extractionService = ExtractionService(self.services) - - # Initialize new submodules - from .subResponseParsing import ResponseParser - from .subDocumentIntents import DocumentIntentAnalyzer - from .subContentExtraction import ContentExtractor - from .subStructureGeneration import StructureGenerator - from .subStructureFilling import StructureFiller - from .subAiCallLooping import AiCallLooper - - if not hasattr(self, 'responseParser'): - logger.info("Initializing ResponseParser...") - self.responseParser = ResponseParser(self.services) - - if not hasattr(self, 'intentAnalyzer'): - logger.info("Initializing DocumentIntentAnalyzer...") - self.intentAnalyzer = DocumentIntentAnalyzer(self.services, self) - - if not hasattr(self, 'contentExtractor'): - logger.info("Initializing ContentExtractor...") - self.contentExtractor = ContentExtractor(self.services, self, self.intentAnalyzer) - - if not hasattr(self, 'structureGenerator'): - logger.info("Initializing StructureGenerator...") - self.structureGenerator = StructureGenerator(self.services, self) - - if not hasattr(self, 'structureFiller'): - logger.info("Initializing StructureFiller...") - self.structureFiller = StructureFiller(self.services, self) - - if not hasattr(self, 'aiCallLooper'): - logger.info("Initializing AiCallLooper...") - self.aiCallLooper = AiCallLooper(self.services, self, self.responseParser) - - async def callAi(self, request: AiCallRequest, progressCallback=None): - """Router: handles content parts via extractionService, text context via interface. - - FAIL-SAFE BILLING at the source: - 1. Pre-flight check: validates billing context is complete (RAISES if not) - 2. Balance & provider check before AI call - 3. billingCallback on aiObjects: records one billing transaction per model call - with exact provider + model name (set before AI call, invoked by _callWithModel) - """ - # SPEECH_TEAMS: Dedicated pipeline, bypasses standard model selection - if request.options and request.options.operationType == OperationTypeEnum.SPEECH_TEAMS: - return await self._handleSpeechTeams(request) - - # FAIL-SAFE: Pre-flight billing validation (like 0 CHF credit card check) - self._preflightBillingCheck() - - # Balance & provider permission checks - await self._checkBillingBeforeAiCall() - - # Calculate effective allowedProviders: RBAC ∩ Workflow - effectiveProviders = self._calculateEffectiveProviders() - if effectiveProviders and request.options: - request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders}) - logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}") - - # Set billing callback on aiObjects BEFORE the AI call - # This callback is invoked by _callWithModel() after EVERY individual model call - # For parallel content parts (e.g., 200 MB doc), each model call creates its own transaction - self.aiObjects.billingCallback = self._createBillingCallback() - - try: - if hasattr(request, 'contentParts') and request.contentParts: - response = await self.extractionService.processContentPartsWithAi( - request, self.aiObjects, progressCallback - ) - else: - response = await self.aiObjects.callWithTextContext(request) - finally: - # Clear callback after call completes - self.aiObjects.billingCallback = None - - # Store workflow stats for analytics - self._storeAiCallStats(response, request) - - return response - - # ========================================================================= - # SPEECH_TEAMS: Dedicated handler for Teams Meeting AI analysis - # Bypasses standard model selection. Uses a fixed fast model. - # ========================================================================= - - async def _handleSpeechTeams(self, request: AiCallRequest): - """ - Dedicated handler for SPEECH_TEAMS operation type. - Bypasses standard model selection and uses a fixed fast model optimized - for low-latency meeting transcript analysis. - - The handler: - 1. Selects a fixed fast model (no model selector) - 2. Builds a specialized system prompt for meeting analysis - 3. Calls the model with structured JSON output - 4. Returns a SpeechTeamsResponse wrapped in AiCallResponse - - Args: - request: AiCallRequest with: - - prompt: User-configured system prompt (from FeatureInstance.config.aiSystemPrompt) - - context: Accumulated transcript segments to analyze - - options.metadata: Optional dict with "botName" key - - Returns: - AiCallResponse with content as JSON string (SpeechTeamsResponse format) - """ - from modules.datamodels.datamodelAi import AiCallResponse, AiModelCall, AiCallOptions, PriorityEnum - - startTime = time.time() - - # Billing pre-flight (SPEECH_TEAMS also needs billing) - self._preflightBillingCheck() - await self._checkBillingBeforeAiCall() - - # 1. Select a fixed fast model (bypass model selector) - model = self._getSpeechTeamsModel() - if not model: - return AiCallResponse( - content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": "No suitable model available for SPEECH_TEAMS", "detectedIntent": "none"}), - modelName="error", - provider="unknown", - priceCHF=0.0, - processingTime=0.0, - bytesSent=0, - bytesReceived=0, - errorCount=1 - ) - - # 2. Build specialized system prompt - metadata = {} - if hasattr(request.options, 'allowedProviders') and request.options.allowedProviders: - # Reuse allowedProviders field as metadata carrier if set (backward compat) - pass - botName = metadata.get("botName", "AI Assistant") - - # Extract botName from context if embedded as header - contextText = request.context or "" - if contextText.startswith("BOT_NAME:"): - lines = contextText.split("\n", 1) - botName = lines[0].replace("BOT_NAME:", "").strip() - contextText = lines[1] if len(lines) > 1 else "" - - userSystemPrompt = request.prompt or "" - systemPrompt = self._buildSpeechTeamsSystemPrompt(userSystemPrompt, botName) - - # 3. Build messages - messages = [ - {"role": "system", "content": systemPrompt}, - {"role": "user", "content": contextText} - ] - - # 4. Call model directly (no failover loop -- single fast model) - modelCall = AiModelCall( - messages=messages, - model=model, - options=AiCallOptions( - operationType=OperationTypeEnum.SPEECH_TEAMS, - priority=PriorityEnum.SPEED, - temperature=0.3, - resultFormat="json" - ) - ) - - # Set billing callback - self.aiObjects.billingCallback = self._createBillingCallback() - - try: - inputBytes = len((systemPrompt + contextText).encode('utf-8')) - modelResponse = await model.functionCall(modelCall) - - if not modelResponse.success: - raise ValueError(f"SPEECH_TEAMS model call failed: {modelResponse.error}") - - content = modelResponse.content - outputBytes = len(content.encode('utf-8')) - processingTime = time.time() - startTime - priceCHF = model.calculatepriceCHF(processingTime, inputBytes, outputBytes) - - response = AiCallResponse( - content=content, - modelName=model.name, - provider=model.connectorType, - priceCHF=priceCHF, - processingTime=processingTime, - bytesSent=inputBytes, - bytesReceived=outputBytes, - errorCount=0 - ) - - # Record billing - if self.aiObjects.billingCallback: - try: - self.aiObjects.billingCallback(response) - except Exception as e: - logger.error(f"BILLING: Failed to record billing for SPEECH_TEAMS: {e}") - - # Store stats - self._storeAiCallStats(response, request) - - logger.info(f"SPEECH_TEAMS call completed: model={model.name}, time={processingTime:.2f}s, cost={priceCHF:.4f} CHF") - return response - - except Exception as e: - processingTime = time.time() - startTime - logger.error(f"SPEECH_TEAMS call failed: {e}") - return AiCallResponse( - content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": f"Error: {str(e)}", "detectedIntent": "none"}), - modelName=model.name if model else "error", - provider=model.connectorType if model else "unknown", - priceCHF=0.0, - processingTime=processingTime, - bytesSent=0, - bytesReceived=0, - errorCount=1 - ) - finally: - self.aiObjects.billingCallback = None - - def _getSpeechTeamsModel(self): - """ - Get the fixed fast model for SPEECH_TEAMS. - Prioritizes: GPT-4o-mini > Claude Haiku > any fast model with DATA_ANALYSE capability. - Returns the AiModel instance or None. - """ - from modules.aicore.aicoreModelRegistry import modelRegistry - - availableModels = modelRegistry.getAvailableModels() - if not availableModels: - logger.error("SPEECH_TEAMS: No models available in registry") - return None - - # Priority list of preferred models for SPEECH_TEAMS (fast + cheap) - _preferredModelNames = [ - "gpt-4o-mini", # OpenAI: fast, cheap, good at JSON - "claude-3-5-haiku", # Anthropic: fast, cheap - "gpt-4o", # OpenAI: fallback to quality model - "claude-sonnet-4-5", # Anthropic: fallback - ] - - # Try preferred models in order - for preferredName in _preferredModelNames: - for model in availableModels: - if preferredName in model.name.lower() and model.functionCall and model.isAvailable: - logger.info(f"SPEECH_TEAMS: Selected preferred model '{model.name}' ({model.displayName})") - return model - - # Fallback: pick fastest available model with DATA_ANALYSE capability - _dataAnalyseModels = [] - for model in availableModels: - if not model.functionCall or not model.isAvailable: - continue - for opRating in model.operationTypes: - if opRating.operationType == OperationTypeEnum.DATA_ANALYSE: - _dataAnalyseModels.append((model, opRating.rating)) - break - - if _dataAnalyseModels: - # Sort by speed rating (descending) then cost (ascending) - _dataAnalyseModels.sort(key=lambda x: (-x[0].speedRating, x[0].costPer1kTokensInput)) - bestModel = _dataAnalyseModels[0][0] - logger.info(f"SPEECH_TEAMS: Selected fallback model '{bestModel.name}' (speed={bestModel.speedRating})") - return bestModel - - # Last resort: first available model - for model in availableModels: - if model.functionCall and model.isAvailable: - logger.warning(f"SPEECH_TEAMS: Using last-resort model '{model.name}'") - return model - - return None - - def _buildSpeechTeamsSystemPrompt(self, userSystemPrompt: str, botName: str) -> str: - """ - Build the specialized system prompt for SPEECH_TEAMS meeting analysis. - Combines a fixed base prompt with user-configurable instructions. - """ - # Extract first name for examples (e.g. "Nyla" from "Nyla Larsson") - botFirstName = botName.split()[0] if " " in botName else botName - - basePrompt = f"""Du bist "{botName}", ein AI-Teilnehmer in einem Microsoft Teams Meeting. -Analysiere das folgende Transkript und entscheide, ob du antworten sollst. - -SPRACHE: Das Transkript kann in verschiedenen Sprachen sein. Antworte immer in der Sprache des letzten Sprechers der dich angesprochen hat. Wenn jemand sagt "let's talk German" oder "sprich deutsch", wechsle die Sprache entsprechend. - -WICHTIG - SPRACHERKENNUNG: Das Transkript stammt aus einer automatischen Spracherkennung (Live Captions). -Dein Name "{botFirstName}" kann VERZERRT transkribiert werden, z.B. als aehnlich klingende Varianten -(z.B. "{botFirstName}" koennte als "Naila", "Neela", "Nila", "Sheila" etc. erscheinen). -Wenn ein Wort im Transkript PHONETISCH AEHNLICH zu "{botFirstName}" klingt und im Kontext einer Anrede steht, bist du gemeint. - -WANN ANTWORTEN: - -REGEL 1 (HOECHSTE PRIORITAET - NUR wenn direkt angesprochen): -Antworte NUR wenn dein Name "{botFirstName}" (oder phonetisch aehnliche Varianten durch Spracherkennung) DIREKT im aktuellsten Transkript-Segment vorkommt. -Beispiele wo du antworten MUSST: "{botFirstName}, was denkst du?", "Hey {botFirstName}", "{botFirstName} please introduce yourself" -Beispiele wo du NICHT antworten darfst: Jemand spricht ueber ein Thema ohne dich zu adressieren. - -REGEL 2 (NUR bei direkter Frage an dich): -Wenn jemand eine Frage DIREKT AN DICH stellt (mit deinem Namen), beantworte sie. -Antworte NICHT auf allgemeine Fragen in der Runde, die nicht an dich gerichtet sind. - -REGEL 3 (NICHT ANTWORTEN - sehr wichtig): -- Wenn Teilnehmer miteinander sprechen ohne dich zu adressieren: NICHT antworten -- Wenn die Konversation nicht an dich gerichtet ist: NICHT antworten -- Wenn du bereits auf dieselbe Frage geantwortet hast: NICHT nochmal antworten -- Wenn du nicht sicher bist ob du gemeint bist: NICHT antworten -- Im Zweifel: shouldRespond = false - -ANTWORT-STIL (wenn du antwortest): -- Direkt und konkret antworten, KEINE Floskeln -- NICHT mit "Hallo [Name]" anfangen wenn du bereits begruessst hast -- NICHT "Ich bin {botName} und ich bin hier um zu helfen" wiederholen -- NICHT frueheres wiederholen das du schon gesagt hast -- Max 1-2 Saetze, praezise auf den Punkt -- Sieh dir an was du (markiert als [YOU]) bereits gesagt hast und wiederhole es NICHT -- KEINE reinen Absichtssaetze wie "Ich werde ...", "Ich kann ...", "Gerne ...". - Liefere direkt den eigentlichen Inhalt in der gleichen Antwort. - -WENN DER USER DICH BITTET ETWAS VORZULESEN / ZUSAMMENZUFASSEN: -- Gib IMMER sofort die Zusammenfassung aus (nicht nur ankündigen). -- Falls Vorlesen gewuenscht ist, setze zusaetzlich ein "readAloud"-Kommando mit dem Text. - -KANAL-AUSWAHL (Voice vs Chat) - Je nach Anfrage unterschiedlich antworten: -- Du kannst pro Anfrage festlegen, ob deine Antwort per Voice (TTS), per Chat, oder beides erfolgt. -- Wenn jemand sagt "schreib das in den Chat", "schreib die Zusammenfassung in den Chat", "poste das im Chat": - - responseChannels: ["voice", "chat"] - - responseTextForVoice: Kurze Bestaetigung (z.B. "Ich schreibe die Zusammenfassung jetzt in den Chat") - - responseTextForChat: Der eigentliche Inhalt (z.B. die vollstaendige Zusammenfassung) -- Wenn jemand sagt "sag mir das", "lies das vor", "sprich das aus": - - responseChannels: ["voice"] oder ["voice","chat"] je nach Kontext - - responseTextForVoice: Der zu sprechende Text -- Wenn jemand sagt "nur im Chat", "schreib nur": responseChannels: ["chat"] -- Wenn keine Kanal-Praeferenz erkennbar: responseChannels weglassen (Config entscheidet), responseText verwenden. - -STOP-ERKENNUNG: -Wenn jemand dich bittet aufzuhoeren, still zu sein, zu stoppen, oder nicht mehr zu reden -(in JEDER Sprache, z.B. "{botFirstName} stop", "{botFirstName} sei still", "{botFirstName} halt", "{botFirstName} be quiet", -"{botFirstName} shut up", "{botFirstName} arrete", etc.), dann setze detectedIntent auf "stop" und -shouldRespond auf false. Du musst NICHT antworten wenn jemand dich stoppt.""" - - # Append user-configured instructions if provided - if userSystemPrompt and userSystemPrompt.strip(): - basePrompt += f"\n\nZUSAETZLICHE ANWEISUNGEN:\n{userSystemPrompt.strip()}" - - basePrompt += f""" - -KOMMANDOS: Du kannst optionale Aktions-Kommandos ausfuehren lassen. -Verfuegbare Kommandos (im "commands" Array): -- {{"action": "toggleTranscript", "params": {{"enable": true/false}}}} — Transkription ein-/ausschalten -- {{"action": "sendChat", "params": {{"text": "Nachricht"}}}} — Zusaetzliche Nachricht in den Chat schreiben (unabhaengig von responseText) -- {{"action": "readAloud", "params": {{"text": "Text zum Vorlesen"}}}} — Einen bestimmten Text vorlesen (unabhaengig von responseText) -- {{"action": "changeLanguage", "params": {{"language": "en-US"}}}} — Kommunikationssprache aendern (z.B. "de-DE", "en-US", "fr-FR") -- {{"action": "sendMail", "params": {{"recipient": "email@example.com", "subject": "Betreff", "message": "Inhalt"}}}} — Email senden (z.B. Meeting-Zusammenfassung, Notizen) -- {{"action": "storeDocument", "params": {{"sitePath": "/sites/team", "folderPath": "/Shared Documents", "fileName": "datei.txt", "content": "Inhalt"}}}} — Dokument in SharePoint ablegen - -Verwende Kommandos NUR wenn explizit darum gebeten wird (z.B. "schalte die Transkription ein", "schreib das in den Chat", "lies das vor", "sprich Englisch", "schick eine Mail", "speichere das Dokument"). -WICHTIG: Wenn du gebeten wirst eine Email zu senden, verwende IMMER den sendMail-Befehl. Schreibe NICHT nur eine Chat-Nachricht die behauptet, die Mail sei gesendet. - -WICHTIG: Antworte IMMER als valides JSON in exakt diesem Format: -{{ - "shouldRespond": true/false, - "responseText": "Deine Antwort hier" oder null (Standard fuer beide Kanäle), - "responseTextForVoice": optional - Text nur fuer TTS/Voice (z.B. kurze Bestaetigung), - "responseTextForChat": optional - Text nur fuer Chat (z.B. lange Zusammenfassung), - "responseChannels": optional - ["voice"], ["chat"] oder ["voice","chat"] je nach User-Anfrage, - "reasoning": "Kurze Begruendung deiner Entscheidung", - "detectedIntent": "addressed" | "question" | "proactive" | "stop" | "none", - "commands": [] oder null -}} - -detectedIntent-Werte: -- "addressed": {botName} wurde direkt angesprochen -- "question": Eine allgemeine Frage wurde gestellt -- "proactive": Du hast einen wertvollen proaktiven Beitrag -- "stop": Der User bittet {botName} aufzuhoeren/still zu sein (in jeder Sprache) -- "none": Kein Handlungsbedarf""" - - return basePrompt - - def _preflightBillingCheck(self) -> None: - """ - Pre-flight billing validation - like a 0 CHF credit card authorization check. - - Validates that ALL required billing context is present and that a billing - transaction CAN be recorded. This dry-run check catches missing context - BEFORE an expensive AI call starts. - - FAIL-SAFE: This method RAISES if billing context is incomplete. - An AI call without billing context MUST NOT proceed. - - Raises: - BillingContextError: If billing context is incomplete or invalid - """ - if not self.services: - raise BillingContextError("No service context available - cannot bill AI call") - - user = getattr(self.services, 'user', None) - if not user: - raise BillingContextError("No user context - cannot bill AI call") - - mandateId = getattr(self.services, 'mandateId', None) - if not mandateId: - raise BillingContextError( - f"No mandateId in service context for user {user.id} - cannot bill AI call. " - "Every AI call MUST have a mandate context for billing." - ) - - # Validate billing service can be created - featureInstanceId = getattr(self.services, 'featureInstanceId', None) - featureCode = getattr(self.services, 'featureCode', None) - - try: - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) - except Exception as e: - raise BillingContextError( - f"Cannot create billing service for user {user.id}, mandate {mandateId}: {e}" - ) - - # Dry-run: verify billing service can check balance (DB accessible) - try: - billingService.checkBalance(0.0) - except Exception as e: - raise BillingContextError( - f"Billing system not accessible for mandate {mandateId}: {e}" - ) - - logger.debug( - f"Pre-flight billing check PASSED: user={user.id}, mandate={mandateId}, " - f"feature={featureCode or 'none'}, instance={featureInstanceId or 'none'}" - ) - - async def _checkBillingBeforeAiCall(self) -> None: - """ - Check billing status before making an AI call. - - FAIL-SAFE: Context validation is done in _preflightBillingCheck() which is - called first. This method handles balance and provider permission checks. - - Verifies: - 1. User has sufficient balance (for prepay models) - 2. Provider is allowed for the user (via RBAC) - - Raises: - InsufficientBalanceException: If balance is insufficient - ProviderNotAllowedException: If provider is not allowed - BillingContextError: If billing check fails unexpectedly - """ - # Context is already validated by _preflightBillingCheck() - user = self.services.user - mandateId = self.services.mandateId - featureInstanceId = getattr(self.services, 'featureInstanceId', None) - featureCode = getattr(self.services, 'featureCode', None) - - try: - # Get billing service - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) - - # Check balance (estimate typical AI call cost) - estimatedCost = 0.01 # ~1 cent CHF minimum - balanceCheck = billingService.checkBalance(estimatedCost) - - if not balanceCheck.allowed: - balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" - logger.warning( - f"Billing check failed for user {user.id}: " - f"Balance {balance_str} CHF, " - f"Reason: {balanceCheck.reason}" - ) - raise InsufficientBalanceException( - currentBalance=balanceCheck.currentBalance or 0.0, - requiredAmount=estimatedCost, - message=f"Ungenugendes Guthaben. Aktuell: CHF {balance_str}" - ) - - balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" - logger.debug(f"Billing check passed: Balance {balance_str} CHF") - - # Check if at least one provider is allowed (RBAC check) - rbacAllowedProviders = billingService.getallowedProviders() - if not rbacAllowedProviders: - logger.warning(f"No AI providers allowed for user {user.id} in mandate {mandateId}") - raise ProviderNotAllowedException( - provider="any", - message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator." - ) - - # Check automation-level allowedProviders restriction - automationAllowedProviders = getattr(self.services, 'allowedProviders', None) - if automationAllowedProviders: - effectiveProviders = [p for p in automationAllowedProviders if p in rbacAllowedProviders] - if not effectiveProviders: - logger.warning(f"No providers available after automation restriction. " - f"Automation allows: {automationAllowedProviders}, " - f"RBAC allows: {rbacAllowedProviders}") - raise ProviderNotAllowedException( - provider="any", - message="Die konfigurierten AI-Provider dieser Automation sind fuer Ihre Rolle nicht freigegeben." - ) - logger.debug(f"Automation provider check passed: {effectiveProviders}") - - # Check if preferred providers (from UI multiselect) are allowed - preferredProviders = getattr(self.services, 'preferredProviders', None) - if preferredProviders: - for provider in preferredProviders: - if provider not in rbacAllowedProviders: - logger.warning(f"Preferred provider {provider} not allowed for user {user.id}") - raise ProviderNotAllowedException( - provider=provider, - message=f"Der gewaehlte Provider '{provider}' ist fuer Ihre Rolle nicht freigegeben." - ) - logger.debug(f"All preferred providers are allowed: {preferredProviders}") - - logger.debug(f"Provider check passed: {len(rbacAllowedProviders)} providers allowed") - - except InsufficientBalanceException: - raise - except ProviderNotAllowedException: - raise - except BillingContextError: - raise - except Exception as e: - # FAIL-SAFE: Don't silently swallow errors - log at ERROR level - logger.error(f"BILLING FAIL-SAFE: Billing check failed with unexpected error: {e}") - raise BillingContextError(f"Billing check failed: {e}") - - def _createBillingCallback(self): - """ - Create a billing callback for interfaceAiObjects._callWithModel(). - - Returns a function that records one billing transaction per individual model call. - Each transaction contains the exact provider name AND model name. - - For a 200 MB document processed with N parallel AI calls (possibly different models), - this creates N separate billing transactions - one per model call. - """ - user = self.services.user - mandateId = self.services.mandateId - featureInstanceId = getattr(self.services, 'featureInstanceId', None) - featureCode = getattr(self.services, 'featureCode', None) - - # Get workflow ID if available - workflowId = None - workflow = getattr(self.services, 'workflow', None) - if workflow and hasattr(workflow, 'id'): - workflowId = workflow.id - - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) - - def _billingCallback(response) -> None: - """Record billing for a single AI model call.""" - if not response or getattr(response, 'errorCount', 0) > 0: - return - - priceCHF = getattr(response, 'priceCHF', 0.0) - if not priceCHF or priceCHF <= 0: - return - - provider = getattr(response, 'provider', None) or 'unknown' - modelName = getattr(response, 'modelName', None) or 'unknown' - - try: - billingService.recordUsage( - priceCHF=priceCHF, - workflowId=workflowId, - aicoreProvider=provider, - aicoreModel=modelName, - description=f"AI: {modelName}" - ) - logger.debug( - f"Billed model call: {priceCHF:.4f} CHF, " - f"provider={provider}, model={modelName}, mandate={mandateId}" - ) - except Exception as e: - logger.error( - f"BILLING: Failed to record transaction! " - f"Cost={priceCHF:.4f} CHF, user={user.id}, mandate={mandateId}, " - f"provider={provider}, model={modelName}, error={e}" - ) - - return _billingCallback - - def _calculateEffectiveProviders(self) -> Optional[List[str]]: - """ - Calculate effective allowed providers: RBAC ∩ Workflow. - - RBAC is master - only RBAC-permitted providers can ever be used. - If workflow specifies allowedProviders, intersect with RBAC. - If no workflow providers, use all RBAC-permitted providers. - - Returns: - List of effective allowed providers, or None if no filtering needed - """ - try: - user = getattr(self.services, 'user', None) - mandateId = getattr(self.services, 'mandateId', None) - - if not user or not mandateId: - return None - - # Get RBAC-permitted providers (master list) - # Note: getBillingService is imported at module level from mainServiceBilling - featureInstanceId = getattr(self.services, 'featureInstanceId', None) - featureCode = getattr(self.services, 'featureCode', None) - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) - rbacProviders = billingService.getallowedProviders() - - if not rbacProviders: - logger.warning("No RBAC-permitted providers found") - return None - - # Get workflow-specified providers (optional filter) - workflowProviders = getattr(self.services, 'allowedProviders', None) - - if workflowProviders: - # Intersect: only providers that are both RBAC-permitted AND workflow-allowed - effectiveProviders = [p for p in workflowProviders if p in rbacProviders] - logger.debug(f"Provider filter: RBAC={rbacProviders}, Workflow={workflowProviders}, Effective={effectiveProviders}") - else: - # No workflow filter - use all RBAC-permitted providers - effectiveProviders = rbacProviders - logger.debug(f"Provider filter: RBAC={rbacProviders} (no workflow filter)") - - return effectiveProviders if effectiveProviders else None - - except Exception as e: - logger.warning(f"Error calculating effective providers: {e}") - return None - - def _storeAiCallStats(self, response, request: AiCallRequest) -> None: - """Store workflow stats after an AI call. - - This method stores the AI call statistics (cost, processing time, bytes) - to the workflow stats collection for tracking and billing purposes. - - Args: - response: AiCallResponse with cost/timing data - request: Original AiCallRequest for context - """ - try: - # Skip if no workflow context - workflow = getattr(self.services, 'workflow', None) - if not workflow or not hasattr(workflow, 'id') or not workflow.id: - logger.debug("No workflow context - skipping stats storage") - return - - # Skip if response is an error - if not response or getattr(response, 'errorCount', 0) > 0: - logger.debug("Error response - skipping stats storage") - return - - # Determine process name from operation type - opType = getattr(request.options, 'operationType', 'unknown') if request.options else 'unknown' - process = f"ai.call.{opType}" - - # Store the stat - self.services.chat.storeWorkflowStat(workflow, response, process) - logger.debug(f"Stored AI call stat: {process}, cost={getattr(response, 'priceCHF', 0):.4f} CHF") - - except Exception as e: - # Log but don't fail - stats storage is not critical - logger.debug(f"Could not store AI call stat: {str(e)}") - - async def ensureAiObjectsInitialized(self): - """Ensure aiObjects is initialized and submodules are ready.""" - if self.aiObjects is None: - logger.info("Lazy initializing AiObjects...") - self.aiObjects = await AiObjects.create() - logger.info("AiObjects initialization completed") - # Initialize submodules after aiObjects is ready - self._initializeSubmodules() - - @classmethod - async def create(cls, serviceCenter=None) -> "AiService": - """Create AiService instance with all connectors and submodules initialized.""" - logger.info("AiService.create() called") - instance = cls(serviceCenter) - logger.info("AiService created, about to call AiObjects.create()...") - instance.aiObjects = await AiObjects.create() - logger.info("AiObjects.create() completed") - # Initialize all submodules after aiObjects is ready - instance._initializeSubmodules() - logger.info("AiService submodules initialized") - return instance - - # Helper methods - - def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str: - """ - Build full prompt by replacing placeholders with their content. - Uses the new {{KEY:placeholder}} format. - - Args: - prompt: The base prompt template - placeholders: Dictionary of placeholder key-value pairs - - Returns: - Prompt with placeholders replaced - """ - if not placeholders: - return prompt - - full_prompt = prompt - for placeholder, content in placeholders.items(): - # Skip if content is None or empty - if content is None: - continue - # Replace {{KEY:placeholder}} - full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", str(content)) - - return full_prompt - - async def _analyzePromptAndCreateOptions(self, prompt: str) -> AiCallOptions: - """Analyze prompt to determine appropriate AiCallOptions parameters.""" - try: - # Get dynamic enum values from Pydantic models - operationTypes = [e.value for e in OperationTypeEnum] - priorities = [e.value for e in PriorityEnum] - processingModes = [e.value for e in ProcessingModeEnum] - - # Create analysis prompt for AI to determine operation type and parameters - analysisPrompt = f""" -You are an AI operation analyzer. Analyze the following prompt and determine the most appropriate operation type and parameters. - -PROMPT TO ANALYZE: -{self.services.utils.sanitizePromptContent(prompt, 'userinput')} - -Based on the prompt content, determine: -1. operationType: Choose the most appropriate from: {', '.join(operationTypes)} -2. priority: Choose from: {', '.join(priorities)} -3. processingMode: Choose from: {', '.join(processingModes)} -4. compressPrompt: true/false (true for story-like prompts, false for structured prompts with JSON/schemas) -5. compressContext: true/false (true to summarize context, false to process fully) - -Respond with ONLY a JSON object in this exact format: -{{ - "operationType": "dataAnalyse", - "priority": "balanced", - "processingMode": "basic", - "compressPrompt": true, - "compressContext": true -}} -""" - - # Use AI to analyze the prompt - request = AiCallRequest( - prompt=analysisPrompt, - options=AiCallOptions( - operationType=OperationTypeEnum.DATA_ANALYSE, - priority=PriorityEnum.SPEED, - processingMode=ProcessingModeEnum.BASIC, - compressPrompt=True, - compressContext=False - ) - ) - - response = await self.callAi(request) - - # Parse AI response using structured parsing with AiCallOptions model - try: - # Use parseJsonWithModel to parse response into AiCallOptions (handles enum conversion automatically) - analysis = parseJsonWithModel(response.content, AiCallOptions) - return analysis - except Exception as e: - logger.warning(f"Failed to parse AI analysis response: {e}") - - except Exception as e: - logger.warning(f"Prompt analysis failed: {e}") - - # Fallback to default options - return AiCallOptions( - operationType=OperationTypeEnum.DATA_ANALYSE, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.BASIC - ) - - async def callAiWithLooping( - self, - prompt: str, - options: AiCallOptions, - debugPrefix: str = "ai_call", - promptBuilder: Optional[callable] = None, - promptArgs: Optional[Dict[str, Any]] = None, - operationId: Optional[str] = None, - userPrompt: Optional[str] = None, - contentParts: Optional[List[ContentPart]] = None, # ARCHITECTURE: Support ContentParts for large content - useCaseId: Optional[str] = None # REQUIRED: Explicit use case ID for generic looping system - ) -> str: - """Public method: Delegate to AiCallLooper for AI calls with looping support.""" - return await self.aiCallLooper.callAiWithLooping( - prompt, options, debugPrefix, promptBuilder, promptArgs, operationId, userPrompt, contentParts, useCaseId - ) - - # JSON merging logic moved to subJsonResponseHandling.py - - def _extractSectionsFromResponse( - self, - result: str, - iteration: int, - debugPrefix: str, - allSections: List[Dict[str, Any]] = None, - accumulationState: Optional[JsonAccumulationState] = None - ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]: - """Delegate to ResponseParser.""" - return self.responseParser.extractSectionsFromResponse( - result, iteration, debugPrefix, allSections, accumulationState - ) - - def _shouldContinueGeneration( - self, - allSections: List[Dict[str, Any]], - iteration: int, - wasJsonComplete: bool, - rawResponse: str = None - ) -> bool: - """Delegate to ResponseParser.""" - return self.responseParser.shouldContinueGeneration( - allSections, iteration, wasJsonComplete, rawResponse - ) - - def _extractDocumentMetadata( - self, - parsedResult: Dict[str, Any] - ) -> Optional[Dict[str, Any]]: - """Delegate to ResponseParser.""" - return self.responseParser.extractDocumentMetadata(parsedResult) - - def _buildFinalResultFromSections( - self, - allSections: List[Dict[str, Any]], - documentMetadata: Optional[Dict[str, Any]] = None - ) -> str: - """Delegate to ResponseParser.""" - return self.responseParser.buildFinalResultFromSections(allSections, documentMetadata) - - # Public API Methods - - # Planning AI Call - async def callAiPlanning( - self, - prompt: str, - placeholders: Optional[List[PromptPlaceholder]] = None, - debugType: Optional[str] = None - ) -> str: - """ - Planning AI call for task planning, action planning, action selection, etc. - Always uses static parameters optimized for planning tasks. - - Args: - prompt: The planning prompt - placeholders: Optional list of placeholder replacements - debugType: Optional debug file type identifier (e.g., 'taskplan', 'dynamic', 'intentanalysis') - If not provided, defaults to 'plan' - - Returns: - Planning JSON response - """ - await self.ensureAiObjectsInitialized() - - # Planning calls always use static parameters - options = AiCallOptions( - operationType=OperationTypeEnum.PLAN, - priority=PriorityEnum.QUALITY, - processingMode=ProcessingModeEnum.DETAILED, - compressPrompt=False, - compressContext=False - ) - - # Build full prompt with placeholders - if placeholders: - placeholdersDict = {p.label: p.content for p in placeholders} - fullPrompt = self._buildPromptWithPlaceholders(prompt, placeholdersDict) - else: - fullPrompt = prompt - - # Root-cause fix: planning must return raw single-shot JSON, not section-based output - request = AiCallRequest( - prompt=fullPrompt, - context="", - options=options - ) - - # Debug: persist prompt/response for analysis with context-specific naming - debugPrefix = debugType if debugType else "plan" - self.services.utils.writeDebugFile(fullPrompt, f"{debugPrefix}_prompt") - response = await self.callAi(request) # Use callAi to ensure stats are stored - result = response.content or "" - self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") - return result - - # Helper methods for callAiContent refactoring - - async def _handleImageGeneration( - self, - prompt: str, - options: AiCallOptions, - title: Optional[str], - parentOperationId: Optional[str] - ) -> AiResponse: - """Handle IMAGE_GENERATE operation type using image generation path.""" - from modules.services.serviceGeneration.paths.imagePath import ImageGenerationPath - - imagePath = ImageGenerationPath(self.services) - - # Extract format from options - format = options.resultFormat or "png" - - return await imagePath.generateImages( - userPrompt=prompt, - format=format, - title=title, - parentOperationId=parentOperationId - ) - - async def _handleWebOperation( - self, - prompt: str, - options: AiCallOptions, - opType: OperationTypeEnum, - aiOperationId: str - ) -> AiResponse: - """Handle WEB_SEARCH_DATA and WEB_CRAWL operation types.""" - self.services.chat.progressLogUpdate(aiOperationId, 0.4, f"Calling AI for {opType.name}") - - request = AiCallRequest( - prompt=prompt, # Raw JSON prompt - connector will parse it - context="", - options=options - ) - - response = await self.callAi(request) - - if not response.content: - errorMsg = f"No content returned from {opType.name}: {response.content}" - logger.error(f"Error in {opType.name}: {errorMsg}") - self.services.chat.progressLogFinish(aiOperationId, False) - raise ValueError(errorMsg) - - metadata = AiResponseMetadata( - operationType=opType.value - ) - - # Note: Stats are now stored centrally in callAi() - no need to duplicate here - - self.services.chat.progressLogUpdate(aiOperationId, 0.9, f"{opType.name} completed") - self.services.chat.progressLogFinish(aiOperationId, True) - - # Preserve metadata from response if available (e.g., results_with_content from Tavily) - # Check if response has metadata attribute (AiCallResponse from callAi) - if hasattr(response, 'metadata') and response.metadata: - # If metadata is a dict, store it in additionalData - if isinstance(response.metadata, dict): - if not metadata.additionalData: - metadata.additionalData = {} - metadata.additionalData.update(response.metadata) - # If metadata is an object with attributes, extract them - elif hasattr(response.metadata, '__dict__'): - if not metadata.additionalData: - metadata.additionalData = {} - for key, value in response.metadata.__dict__.items(): - if not key.startswith('_'): - metadata.additionalData[key] = value - - return AiResponse( - content=response.content, - metadata=metadata - ) - - def _getIntentForDocument( - self, - docId: str, - intents: Optional[List[DocumentIntent]] - ) -> Optional[DocumentIntent]: - """Find DocumentIntent for given documentId.""" - if not intents: - return None - for intent in intents: - if intent.documentId == docId: - return intent - return None - - async def clarifyDocumentIntents( - self, - documents: List[ChatDocument], - userPrompt: str, - actionParameters: Dict[str, Any], - parentOperationId: str - ) -> List[DocumentIntent]: - """Public method: Delegate to DocumentIntentAnalyzer.""" - return await self.intentAnalyzer.clarifyDocumentIntents( - documents, userPrompt, actionParameters, parentOperationId - ) - - async def extractAndPrepareContent( - self, - documents: List[ChatDocument], - documentIntents: List[DocumentIntent], - parentOperationId: str - ) -> List[ContentPart]: - """Public method: Delegate to ContentExtractor.""" - return await self.contentExtractor.extractAndPrepareContent( - documents, documentIntents, parentOperationId, self._getIntentForDocument - ) - - async def generateStructure( - self, - userPrompt: str, - contentParts: List[ContentPart], - outputFormat: Optional[str] = None, - parentOperationId: str = None - ) -> Dict[str, Any]: - """Public method: Delegate to StructureGenerator.""" - return await self.structureGenerator.generateStructure( - userPrompt, contentParts, outputFormat, parentOperationId - ) - - async def fillStructure( - self, - structure: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - parentOperationId: str - ) -> Dict[str, Any]: - """Public method: Delegate to StructureFiller.""" - return await self.structureFiller.fillStructure( - structure, contentParts, userPrompt, parentOperationId - ) - - async def renderResult( - self, - filledStructure: Dict[str, Any], - outputFormat: str, - language: str, - title: str, - userPrompt: str, - parentOperationId: str - ) -> List[RenderedDocument]: - """ - Phase 5E: Rendert gefüllte Struktur zum Ziel-Format. - Jedes Dokument wird einzeln gerendert, jeder Renderer kann 1..n Dokumente zurückgeben. - - Render filled structure to documents. - Per-document format and language are extracted from structure (validated in State 3). - The outputFormat and language parameters are only used as global fallbacks. - Multiple documents can have different formats and languages. - - Args: - filledStructure: Gefüllte Struktur mit elements - outputFormat: Ziel-Format (pdf, docx, html, etc.) - Global fallback - language: Language (global fallback) - Per-document language extracted from structure - title: Dokument-Titel - userPrompt: User-Anfrage - parentOperationId: Parent Operation-ID für ChatLog-Hierarchie - - Returns: - List of RenderedDocument objects. - Jedes RenderedDocument repräsentiert ein gerendertes Dokument (Hauptdokument oder unterstützende Datei) - """ - # Language comes from structure (per-document), validated in State 3 - # This parameter is only used as global fallback if structure validation fails - # Use validated currentUserLanguage as fallback (always valid) - if not language: - language = self._getUserLanguage() if hasattr(self, '_getUserLanguage') else (self.services.currentUserLanguage if hasattr(self.services, 'currentUserLanguage') else 'en') - # Erstelle Operation-ID für Rendering - renderOperationId = f"{parentOperationId}_rendering" - - # Starte ChatLog mit Parent-Referenz - self.services.chat.progressLogStart( - renderOperationId, - "Content Rendering", - "Rendering", - f"Rendering to {outputFormat} format", - parentOperationId=parentOperationId - ) - - try: - from modules.services.serviceGeneration.mainServiceGeneration import GenerationService - - generationService = GenerationService(self.services) - - # renderReport verarbeitet jetzt jedes Dokument einzeln - # und gibt Liste von (documentData, mimeType, filename) zurück - renderedDocuments = await generationService.renderReport( - filledStructure, - outputFormat, - language, # Pass language (global fallback, per-document extracted in renderReport) - title, - userPrompt, - self, - parentOperationId=renderOperationId # Parent-Referenz für ChatLog-Hierarchie - ) - - # ChatLog abschließen - self.services.chat.progressLogFinish(renderOperationId, True) - - return renderedDocuments - - except Exception as e: - self.services.chat.progressLogFinish(renderOperationId, False) - logger.error(f"Error in _renderResult: {str(e)}") - raise - - def _shouldSkipContentPart( - self, - part: ContentPart - ) -> bool: - """Check if ContentPart should be skipped (already structured JSON).""" - if part.typeGroup == "structure" and part.mimeType == "application/json": - if part.metadata.get("skipExtraction", False): - logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (skipExtraction=True)") - return True - try: - if isinstance(part.data, str): - jsonData = json.loads(part.data) - if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): - logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (contains documents/sections)") - return True - except Exception: - pass # Not JSON, continue processing - return False - - async def callAiContent( - self, - prompt: str, - options: AiCallOptions, - contentParts: Optional[List[ContentPart]] = None, - documentList: Optional[Any] = None, # DocumentReferenceList - documentIntents: Optional[List[DocumentIntent]] = None, - outputFormat: Optional[str] = None, - title: Optional[str] = None, - parentOperationId: Optional[str] = None, - generationIntent: Optional[str] = None # NEW: Explicit intent from action (skips detection) - ) -> AiResponse: - """ - Unified AI content generation with explicit intent requirement. - - All AI-Actions (ai.process, ai.generateDocument, etc.) route through here. - They differ only in parameters, not in logic. - - Args: - prompt: The main prompt for the AI call - options: AI call configuration options (REQUIRED - operationType must be set) - contentParts: Optional list of already-extracted content parts (preferred) - documentList: Optional DocumentReferenceList (wird zu ChatDocuments konvertiert) - documentIntents: Optional list of DocumentIntent objects (wird erstellt wenn nicht vorhanden) - outputFormat: Optional output format for document generation (e.g., 'pdf', 'docx', 'xlsx') - title: Optional title for generated documents - parentOperationId: Optional parent operation ID for hierarchical logging - generationIntent: REQUIRED explicit intent ("document" | "code" | "image") from action. - NO auto-detection - actions must explicitly specify intent. - - Returns: - AiResponse with content, metadata, and optional documents - """ - await self.ensureAiObjectsInitialized() - - # Erstelle Operation-ID - workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" - aiOperationId = f"ai_content_{workflowId}_{int(time.time())}" - - # Starte Progress-Tracking mit Parent-Referenz - formatDisplay = outputFormat if outputFormat else "auto-determined" - self.services.chat.progressLogStart( - aiOperationId, - "AI content processing", - "Content Processing", - f"Format: {formatDisplay}", - parentOperationId=parentOperationId - ) - - try: - # outputFormat is optional - if None, formats determined from prompt by AI - # No default fallback here - let AI service handle it - - opType = getattr(options, "operationType", None) - if not opType: - options.operationType = OperationTypeEnum.DATA_GENERATE - opType = OperationTypeEnum.DATA_GENERATE - - # Route zu Operation-spezifischen Handlern - if opType == OperationTypeEnum.IMAGE_GENERATE: - # Image generation - route to image path - return await self._handleImageGeneration(prompt, options, title, parentOperationId) - - if opType == OperationTypeEnum.WEB_SEARCH_DATA or opType == OperationTypeEnum.WEB_CRAWL: - return await self._handleWebOperation(prompt, options, opType, aiOperationId) - - # Data generation - REQUIRES explicit generationIntent - if opType == OperationTypeEnum.DATA_GENERATE: - if not generationIntent: - errorMsg = ( - "generationIntent is required for DATA_GENERATE operation. " - "Actions must explicitly specify 'document' or 'code' intent. " - "No auto-detection - use qualified actions (ai.generateDocument, ai.generateCode)." - ) - logger.error(errorMsg) - self.services.chat.progressLogFinish(aiOperationId, False) - raise ValueError(errorMsg) - - # Route based on explicit intent (no auto-detection, no fallback) - if generationIntent == "code": - # Route to code generation path - return await self._handleCodeGeneration( - prompt=prompt, - options=options, - contentParts=contentParts, - outputFormat=outputFormat, - title=title, - parentOperationId=parentOperationId - ) - else: - # Route to document generation path (existing behavior) - return await self._handleDocumentGeneration( - prompt=prompt, - options=options, - documentList=documentList, - documentIntents=documentIntents, - contentParts=contentParts, - outputFormat=outputFormat, - title=title, - parentOperationId=parentOperationId - ) - - # DATA_EXTRACT: Extract content from documents and process with AI (no structure generation) - if opType == OperationTypeEnum.DATA_EXTRACT: - return await self._handleDataExtraction( - prompt=prompt, - options=options, - documentList=documentList, - documentIntents=documentIntents, - contentParts=contentParts, - outputFormat=outputFormat, - title=title, - parentOperationId=parentOperationId - ) - - # Other operation types (DATA_ANALYSE, etc.) - not supported - errorMsg = f"Unsupported operation type: {opType}. Supported types: IMAGE_GENERATE, DATA_GENERATE, DATA_EXTRACT" - logger.error(errorMsg) - self.services.chat.progressLogFinish(aiOperationId, False) - raise ValueError(errorMsg) - - except Exception as e: - logger.error(f"Error in callAiContent: {str(e)}") - self.services.chat.progressLogFinish(aiOperationId, False) - raise - - async def _handleDataExtraction( - self, - prompt: str, - options: AiCallOptions, - documentList: Optional[Any], - documentIntents: Optional[List[DocumentIntent]], - contentParts: Optional[List[ContentPart]], - outputFormat: str, - title: str, - parentOperationId: Optional[str] - ) -> AiResponse: - """ - Handle DATA_EXTRACT: Extract content from documents, then process with AI. - - - AUTOMATION mode: No intent analysis. The passed prompt is used as extractionPrompt - for every document and for the final AI call (exact prompt preserved). - - DYNAMIC mode: Intent analysis (clarifyDocumentIntents) runs first; extraction and - processing use the intents and AI-derived extractionPrompt. - """ - import time - - # Create operation ID - workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" - extractOperationId = f"data_extract_{workflowId}_{int(time.time())}" - - # Start progress tracking - self.services.chat.progressLogStart( - extractOperationId, - "Data Extraction", - "Extraction", - f"Format: {outputFormat}", - parentOperationId=parentOperationId - ) - - try: - # Step 1: Get documents from documentList - documents = [] - if documentList: - documents = self.services.chat.getChatDocumentsFromDocumentList(documentList) - - # Filter: Remove original documents if already covered by pre-extracted JSONs - # (to prevent duplicate ContentParts - pre-extracted JSONs contain already extracted ContentParts) - if documents: - # Step 1: Identify all original document IDs covered by pre-extracted JSONs - originalDocIdsCoveredByPreExtracted = set() - for doc in documents: - preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc) - if preExtracted: - originalDocId = preExtracted["originalDocument"]["id"] - originalDocIdsCoveredByPreExtracted.add(originalDocId) - logger.debug(f"Found pre-extracted JSON {doc.id} covering original document {originalDocId}") - - # Step 2: Filter documents - remove originals covered by pre-extracted JSONs - filteredDocuments = [] - for doc in documents: - preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc) - if preExtracted: - filteredDocuments.append(doc) # Keep pre-extracted JSON - elif doc.id in originalDocIdsCoveredByPreExtracted: - logger.info(f"Skipping original document {doc.id} ({doc.fileName}) - already covered by pre-extracted JSON") - else: - filteredDocuments.append(doc) # Keep regular document - - documents = filteredDocuments # Use filtered list - - # Step 2: Document intents – AUTOMATION uses exact prompt; DYNAMIC uses intent analysis - if not documentIntents and documents: - workflowMode = getattr(self.services.workflow, "workflowMode", None) if self.services.workflow else None - if workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION: - # Automation: no intent AI call – use the given prompt as extractionPrompt for every document - documentIntents = [ - DocumentIntent( - documentId=doc.id, - intents=["extract"], - extractionPrompt=prompt, - reasoning="Automation mode: use exact prompt from action", - ) - for doc in documents - ] - logger.debug("DATA_EXTRACT in AUTOMATION mode: using exact prompt for all documents (no intent analysis)") - else: - documentIntents = await self.clarifyDocumentIntents( - documents, - prompt, - {"outputFormat": outputFormat}, - extractOperationId - ) - - # Step 3: Extract and prepare content (NO AI - pure extraction) - REQUIRED for all documents - if documents: - preparedContentParts = await self.extractAndPrepareContent( - documents, - documentIntents or [], - extractOperationId - ) - - # Merge with provided contentParts (if any) - if contentParts: - for part in contentParts: - if part.metadata.get("skipExtraction", False): - part.metadata.setdefault("contentFormat", "extracted") - part.metadata.setdefault("isPreExtracted", True) - preparedContentParts.extend(contentParts) - - contentParts = preparedContentParts - - # Step 4: Process contentParts with AI via ExtractionService - # Always use processContentPartsWithAi – it handles text vs image parts correctly: - # - Text parts → text models (with chunking if needed) - # - Image parts → Vision AI (proper image_url content blocks) - # No manual contentText concatenation or token estimation needed. - if not contentParts: - raise ValueError("No content extracted from documents") - - # Filter out empty content parts (e.g. PDF container with 0 bytes) that would - # produce garbage AI responses and pollute the merged result. - nonEmptyParts = [p for p in contentParts if p.data and len(p.data.strip()) > 0] - if not nonEmptyParts: - raise ValueError("No non-empty content parts to process") - - self.services.utils.writeDebugFile(prompt, "data_extract_prompt") - extractionService = self.services.extraction - aiRequest = AiCallRequest( - prompt=prompt, - context="", - options=options, - contentParts=nonEmptyParts, - ) - aiResponse = await extractionService.processContentPartsWithAi( - aiRequest, self.aiObjects - ) - _respText = aiResponse.content if isinstance(aiResponse.content, str) else (aiResponse.content.decode("utf-8", errors="replace") if aiResponse.content else "") - self.services.utils.writeDebugFile(_respText, "data_extract_response") - - # Create response document - resultDocument = DocumentData( - documentName=f"{title or 'extracted_data'}.{outputFormat}", - documentData=aiResponse.content.encode('utf-8') if isinstance(aiResponse.content, str) else aiResponse.content, - mimeType=f"text/{outputFormat}" if outputFormat in ["txt", "json", "csv"] else "application/octet-stream" - ) - - metadata = AiResponseMetadata( - title=title or "Extracted Data", - operationType=OperationTypeEnum.DATA_EXTRACT.value - ) - - self.services.chat.progressLogFinish(extractOperationId, True) - - return AiResponse( - content=aiResponse.content if isinstance(aiResponse.content, str) else aiResponse.content.decode('utf-8', errors='replace'), - metadata=metadata, - documents=[resultDocument] - ) - - except Exception as e: - logger.error(f"Error in data extraction: {str(e)}") - self.services.chat.progressLogFinish(extractOperationId, False) - raise - - async def _handleCodeGeneration( - self, - prompt: str, - options: AiCallOptions, - contentParts: Optional[List[ContentPart]], - outputFormat: str, - title: str, - parentOperationId: Optional[str] - ) -> AiResponse: - """Handle code generation using code generation path.""" - from modules.services.serviceGeneration.paths.codePath import CodeGenerationPath - - codePath = CodeGenerationPath(self.services) - return await codePath.generateCode( - userPrompt=prompt, - outputFormat=outputFormat, - contentParts=contentParts, - title=title or "Generated Code", - parentOperationId=parentOperationId - ) - - async def _handleDocumentGeneration( - self, - prompt: str, - options: AiCallOptions, - documentList: Optional[Any], - documentIntents: Optional[List[DocumentIntent]], - contentParts: Optional[List[ContentPart]], - outputFormat: str, - title: str, - parentOperationId: Optional[str] - ) -> AiResponse: - """Handle document generation using document generation path.""" - from modules.services.serviceGeneration.paths.documentPath import DocumentGenerationPath - - # Set compression options for document generation - options.compressPrompt = False - options.compressContext = False - - documentPath = DocumentGenerationPath(self.services) - return await documentPath.generateDocument( - userPrompt=prompt, - documentList=documentList, - documentIntents=documentIntents, - contentParts=contentParts, - outputFormat=outputFormat, - title=title or "Generated Document", - parentOperationId=parentOperationId - ) - - - def _determineDocumentName( - self, - filledStructure: Dict[str, Any], - outputFormat: str, - title: Optional[str] - ) -> str: - """Bestimme Dokument-Namen aus Struktur oder Titel.""" - # Versuche aus Struktur zu extrahieren - if isinstance(filledStructure, dict) and "documents" in filledStructure: - docs = filledStructure["documents"] - if isinstance(docs, list) and len(docs) > 0: - firstDoc = docs[0] - if isinstance(firstDoc, dict) and firstDoc.get("filename"): - return firstDoc["filename"] - - # Fallback zu Titel - if title: - sanitized = re.sub(r"[^a-zA-Z0-9._-]", "_", title) - sanitized = re.sub(r"_+", "_", sanitized).strip("_") - if sanitized: - if not sanitized.lower().endswith(f".{outputFormat}"): - return f"{sanitized}.{outputFormat}" - return sanitized - - return f"generated.{outputFormat}" - diff --git a/modules/services/serviceAi/merge_1.txt b/modules/services/serviceAi/merge_1.txt deleted file mode 100644 index 29a8102d..00000000 --- a/modules/services/serviceAi/merge_1.txt +++ /dev/null @@ -1,513 +0,0 @@ -================================================================================ -JSON MERGE OPERATION #1 -================================================================================ -Timestamp: 2026-01-06T22:24:33.405726 - -INPUT: - Accumulated length: 40250 chars - New Fragment length: 2471 chars - Accumulated: 373 lines (showing first 5 and last 5) - { - "elements": [ - { - "type": "table", - "content": { - ... (363 lines omitted) ... - ["04.12.25", "05.12.25", "TICKETCORNER CH, RUEMLANG", "CH", "285.70", "", "**** **** **** 1234"], - ["05.12.25", "05.12.25", "NOII.CH DATING, WINTERTHUR", "CH", "74.10", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "TEMU.COM, BASEL", "CH", "72.50", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "326.85", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Decathlon, Hin - New Fragment: 33 lines (showing first 5 and last 5) - ```json - ["06.12.25", "08.12.25", "Decathlon, Hinwil", "CH", "150.00", "", "**** **** **** 1234"], - ["07.12.25", "09.12.25", "Migros-Genossenschafts-Bund, Zürich", "CH", "200.00", "", "**** **** **** 1234"], - ["08.12.25", "10.12.25", "Zürich HB, Zürich", "CH", "45.00", "", "**** **** **** 1234"], - ["09.12.25", "11.12.25", "Amazon Marketplace, amazon.de", "DE", "120.00", "", "**** **** **** 1234"], - ... (23 lines omitted) ... - } - } - ] - } - ``` - - - Normalized Accumulated (40250 chars) - (showing first 5 and last 5 of 373 lines) - { - "elements": [ - { - "type": "table", - "content": { - ... (363 lines omitted) ... - ["04.12.25", "05.12.25", "TICKETCORNER CH, RUEMLANG", "CH", "285.70", "", "**** **** **** 1234"], - ["05.12.25", "05.12.25", "NOII.CH DATING, WINTERTHUR", "CH", "74.10", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "TEMU.COM, BASEL", "CH", "72.50", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "326.85", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Decathlon, Hin - - Normalized New Fragment (2459 chars) - (showing first 5 and last 5 of 31 lines) - ["06.12.25", "08.12.25", "Decathlon, Hinwil", "CH", "150.00", "", "**** **** **** 1234"], - ["07.12.25", "09.12.25", "Migros-Genossenschafts-Bund, Zürich", "CH", "200.00", "", "**** **** **** 1234"], - ["08.12.25", "10.12.25", "Zürich HB, Zürich", "CH", "45.00", "", "**** **** **** 1234"], - ["09.12.25", "11.12.25", "Amazon Marketplace, amazon.de", "DE", "120.00", "", "**** **** **** 1234"], - ["10.12.25", "12.12.25", "IKEA, Dietlikon", "CH", "350.00", "", "**** **** **** 1234"], - ... (21 lines omitted) ... - ] - } - } - ] - } -STEP: PHASE 1 - Description: Finding overlap between JSON strings - ⏳ In progress... - - Overlap Detection (string (exact)): - Overlap length: 40 - ✅ Found overlap of 40 chars - Accumulated suffix (COMPLETE, 40 chars): - ============================================================================ - ["06.12.25", "08.12.25", "Decathlon, Hin - ============================================================================ - Fragment prefix (40 chars, 1 lines) - ["06.12.25", "08.12.25", "Decathlon, Hin - - Overlap found (40 chars): - Accumulated suffix: ["06.12.25", "08.12.25", "Decathlon, Hin - Fragment prefix: ["06.12.25", "08.12.25", "Decathlon, Hin -STEP: PHASE 2 - Description: Merging strings (overlap: 40 chars) - ⏳ In progress... - - - Merged String (42669 chars) - (showing first 5 and last 5 of 403 lines) - { - "elements": [ - { - "type": "table", - "content": { - ... (393 lines omitted) ... - ] - } - } - ] - } -STEP: PHASE 3 - Description: Returning merged string (may be unclosed) - ⏳ In progress... - - - Returning merged string (preserving incomplete element at end for next iteration) - -================================================================================ -MERGE RESULT: ✅ SUCCESS -================================================================================ -Final result length: 42669 chars -Final result (COMPLETE): -================================================================================ -{ - "elements": [ - { - "type": "table", - "content": { - "headers": [ - "Date", - "Valuta", - "Details", - "Currency", - "Amount", - "Amount in CHF", - "Maskierte Kreditkarte" - ], - "rows": [ - ["12.09.25", "15.09.25", "Coop-1911 Ruti, Ruti ZH", "CH", "102.05", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "food & drive GmbH, Durnten", "CH", "26.20", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "food & drive GmbH, Durnten", "CH", "4.50", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "Gartencenter Meier, Durnten", "CH", "88.40", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "THE ARTISAN ZUERICH, ZUERICH", "CH", "15.00", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "THE ARTISAN ZUERICH, ZUERICH", "CH", "15.00", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "THE ARTISAN ZUERICH, ZUERICH", "CH", "18.00", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "KONDITOREI VOLAND WALD, WALD ZH", "CH", "16.50", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "GITHUB, INC., GITHUB.COM", "US", "USD 0.02", "0.00", "**** **** **** 1234"], - ["15.09.25", "16.09.25", "Coop-1252 Wald, Wald ZH", "CH", "50.80", "", "**** **** **** 1234"], - ["15.09.25", "16.09.25", "CLAUDE.AI SUBSCRIPTION, ANTHROPIC.COM", "US", "USD 108.10", "88.60", "**** **** **** 1234"], - ["16.09.25", "17.09.25", "Shell Waldhof, Wald ZH", "CH", "113.35", "", "**** **** **** 1234"], - ["16.09.25", "17.09.25", "Shell Waldhof, Wald ZH", "CH", "3.60", "", "**** **** **** 1234"], - ["18.09.25", "19.09.25", "Coop-4991 Fallanden, Fallanden", "CH", "116.00", "", "**** **** **** 1234"], - ["18.09.25", "19.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "5.95", "", "**** **** **** 1234"], - ["18.09.25", "19.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "7.00", "", "**** **** **** 1234"], - ["19.09.25", "22.09.25", "Migros M Dubendorf Stettb, Dubendorf", "CH", "32.10", "", "**** **** **** 1234"], - ["19.09.25", "22.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "14.80", "", "**** **** **** 1234"], - ["19.09.25", "22.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "370.65", "", "**** **** **** 1234"], - ["19.09.25", "22.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "11.50", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["21.09.25", "22.09.25", "Kreuzwirt, Weissensee", "AT", "EUR 278.00", "266.50", "**** **** **** 1234"], - ["23.09.25", "24.09.25", "FILIALE, WALD ZH", "CH", "EUR 500.00", "492.15", "**** **** **** 1234"], - ["24.09.25", "25.09.25", "P2 Parkhaus Ein- & Ausfah, Zurich", "CH", "5.00", "", "**** **** **** 1234"], - ["24.09.25", "25.09.25", "A.I.R. Bakery, Zurich", "CH", "18.60", "", "**** **** **** 1234"], - ["24.09.25", "25.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "23.35", "", "**** **** **** 1234"], - ["25.09.25", "26.09.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "203.20", "", "**** **** **** 1234"], - ["25.09.25", "26.09.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "44.10", "", "**** **** **** 1234"], - ["26.09.25", "29.09.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "95.25", "", "**** **** **** 1234"], - ["26.09.25", "29.09.25", "Puls Apotheke & Drogerie, Hinwil", "CH", "140.60", "", "**** **** **** 1234"], - ["26.09.25", "29.09.25", "FILIALE, WALD ZH", "CH", "CHF 280.00", "287.00", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "NYX*LullySA, Lully", "CH", "1.00", "", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "Kisoque de Lully, Lully", "CH", "5.70", "", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "TOTAL MKT FR, NANTERRE", "FR", "EUR 79.95", "76.90", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "AREA NFC 4261525, 69BRON CEDEX", "FR", "EUR 33.50", "32.20", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "HOLIDAY APARTMENTS, PORT SAPLAYA", "ES", "EUR 1'118.15", "1'075.45", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "LE BISTROT DEL M, MEZE", "FR", "EUR 210.20", "202.15", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "SELVA HOSTELERIA, MACANET DE LA", "ES", "EUR 2.40", "2.30", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "E.S. LA SELVA, COMAJULIANA", "ES", "EUR 90.09", "86.65", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "E.S. LA SELVA, COMAJULIANA", "ES", "EUR 4.70", "4.50", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "E.S. LA SELVA, COMAJULIANA", "ES", "EUR 8.40", "8.10", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "AUTOROUTES ASF, VEDENE CEDEX", "FR", "EUR 15.60", "15.00", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "AUTOROUTES ASF, VEDENE CEDEX", "FR", "EUR 24.40", "23.45", "**** **** **** 1234"], - ["29.09.25", "30.09.25", "OROMARKET SUPERMERCADOS, OROPESA", "ES", "EUR 17.32", "16.65", "**** **** **** 1234"], - ["29.09.25", "30.09.25", "QUESADA CENTER SUPERMERCA, OROPESA", "ES", "EUR 40.40", "38.85", "**** **** **** 1234"], - ["29.09.25", "30.09.25", "QUESADA CENTER SUPERMERCA, OROPESA", "ES", "EUR 22.55", "21.70", "**** **** **** 1234"], - ["29.09.25", "30.09.25", "ALDI OROPESA, OROPESA", "ES", "EUR 129.39", "124.40", "**** **** **** 1234"], - ["30.09.25", "01.10.25", "QUESADA CENTER, OROPESA DEL M", "ES", "EUR 84.05", "80.95", "**** **** **** 1234"], - ["30.09.25", "01.10.25", "PASSION CREPES, OROPESA", "ES", "EUR 10.30", "9.90", "**** **** **** 1234"], - ["30.09.25", "01.10.25", "MERCADONA MARINA DOR, ORPESA DEL MA", "ES", "EUR 17.53", "16.90", "**** **** **** 1234"], - ["30.09.25", "01.10.25", "Restaurante DRAGON, OROPESA", "ES", "EUR 75.00", "72.25", "**** **** **** 1234"], - ["30.09.25", "01.10.25", "OPENAI *CHATGPT SUBSCR, OPENAI.COM", "US", "USD 216.20", "177.55", "**** **** **** 1234"], - ["01.10.25", "02.10.25", "GOOGLE *ADS5192965135, cc§google.com", "IE", "29.60", "", "**** **** **** 1234"], - ["01.10.25", "02.10.25", "RTE PUERTA DEL SOL, OROPESA DEL M", "ES", "EUR 169.20", "163.10", "**** **** **** 1234"], - ["01.10.25", "02.10.25", "TREN TURISTICO OROPESA, OROPESA DEL M", "ES", "EUR 15.00", "14.45", "**** **** **** 1234"], - ["01.10.25", "02.10.25", "LANGDOCK GMBH, BERLIN", "DE", "EUR 25.00", "24.10", "**** **** **** 1234"], - ["01.10.25", "02.10.25", "WWW.PERPLEXITY.AI, WWW.PERPLEXIT", "US", "USD 10.81", "8.90", "**** **** **** 1234"], - ["02.10.25", "06.10.25", "GOOGLE *YouTubePremium, g.co/helppay#", "GB", "33.90", "", "**** **** **** 1234"], - ["02.10.25", "06.10.25", "WILLY LA CONCHA, OROPESA DEL M", "ES", "EUR 98.93", "95.40", "**** **** **** 1234"], - ["03.10.25", "06.10.25", "Netflix.com, Los Gatos", "NL", "20.90", "", "**** **** **** 1234"], - ["03.10.25", "06.10.25", "COALIMENT LA CONCHA, OROPESA DEL M", "ES", "EUR 11.74", "11.30", "**** **** **** 1234"], - ["03.10.25", "06.10.25", "DONA RESU, OROPESA", "ES", "EUR 7.30", "7.05", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "MERCADONA MARINA DOR, ORPESA DEL MA", "ES", "EUR 89.50", "86.30", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "QUESADA CENTER SUPERMERCA, OROPESA", "ES", "EUR 8.45", "8.15", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "HELADERIA LAS DELICIAS, OROPESA DEL M", "ES", "EUR 10.80", "10.40", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "REST. BISTROT, OROPESA DEL M", "ES", "EUR 117.90", "113.70", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["04.10.25", "06.10.25", "Google Duolingo Langu, 650-2530000", "US", "10.00", "", "**** **** **** 1234"], - ["05.10.25", "06.10.25", "HABANA, OROPESA", "ES", "EUR 3.00", "2.90", "**** **** **** 1234"], - ["05.10.25", "06.10.25", "HABANA, OROPESA", "ES", "EUR 9.00", "8.70", "**** **** **** 1234"], - ["05.10.25", "06.10.25", "RESTAURANTE, ORPESA", "ES", "EUR 87.75", "84.60", "**** **** **** 1234"], - ["05.10.25", "06.10.25", "HABANA, OROPESA", "ES", "EUR 15.50", "14.95", "**** **** **** 1234"], - ["06.10.25", "07.10.25", "HABANA, OROPESA", "ES", "EUR 25.00", "24.05", "**** **** **** 1234"], - ["06.10.25", "07.10.25", "QUESADA CENTER SUPERMERCA, OROPESA", "ES", "EUR 3.95", "3.80", "**** **** **** 1234"], - ["06.10.25", "07.10.25", "QUESADA CENTER SUPERMERCA, OROPESA", "ES", "EUR 47.75", "45.95", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "MAGIC SPORT HALL OLYMPICS, OROPESA DEL M", "ES", "EUR 183.75", "176.70", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "BKG*HOTEL AT BOOKING.C, (888)850-3958", "NL", "EUR 172.55", "165.90", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "Wondershare, Hong Kong", "HK", "25.95", "", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "MERCADONA MARINA DOR, ORPESA DEL MA", "ES", "EUR 99.13", "95.30", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "RECEP HOTEL MAGIC SPORTS, OROPESA DEL M", "ES", "EUR 10.00", "9.60", "**** **** **** 1234"], - ["07.10.25", "08.10.25", "Google One, 650-2530000", "US", "10.00", "", "**** **** **** 1234"], - ["08.10.25", "09.10.25", "E.S.LA CLARIANA, MADRID", "ES", "EUR 98.07", "94.00", "**** **** **** 1234"], - ["08.10.25", "09.10.25", "AUTOROUTES ASF, VEDENE CEDEX", "FR", "EUR 44.20", "42.35", "**** **** **** 1234"], - ["08.10.25", "09.10.25", "A.R.E.A., 69671", "FR", "EUR 11.20", "10.75", "**** **** **** 1234"], - ["09.10.25", "10.10.25", "SOCAR station-service, Bursins", "CH", "113.10", "", "**** **** **** 1234"], - ["09.10.25", "10.10.25", "SOCAR station-service, Bursins", "CH", "6.80", "", "**** **** **** 1234"], - ["09.10.25", "10.10.25", "A.R.E.A., 69671", "FR", "EUR 15.00", "14.40", "**** **** **** 1234"], - ["08.10.25", "10.10.25", "DOMAINE DE ROZAN, LA TRONCHE", "FR", "EUR 110.00", "105.45", "**** **** **** 1234"], - ["09.10.25", "10.10.25", "DOMAINE DE ROZAN, LA TRONCHE", "FR", "EUR 40.00", "38.35", "**** **** **** 1234"], - ["10.10.25", "13.10.25", "Coop-1252 Wald, Wald ZH", "CH", "164.85", "", "**** **** **** 1234"], - ["10.10.25", "13.10.25", "CURSOR, AI POWERED IDE, CURSOR.COM", "US", "USD 20.00", "16.60", "**** **** **** 1234"], - ["11.10.25", "13.10.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "Cafe Konditorei Voland, Laupen ZH", "CH", "37.70", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "KONDITOREI VOLAND LAUP, LAUPEN ZH", "CH", "17.35", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "KONDITOREI VOLAND LAUP, LAUPEN ZH", "CH", "5.40", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "SBB Bahnhof Ruti ZH, Ruti ZH", "CH", "54.00", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "Rest Volkshaus, Zurich", "CH", "18.00", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "Sora Sushi - HB Zurich, Zurich", "CH", "74.00", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "176.32 avec, Ruti ZH", "CH", "2.45", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "Baradox AG, Zurich", "CH", "15.00", "", "**** **** **** 1234"], - ["12.09.25", "15.09.25", "Volkshausstiftung Zurich, Zurich", "CH", "3.00", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "SBB Bahnhof Ruti ZH, Ruti ZH", "CH", "9.20", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "SBB Bahnhof Wald, Wald ZH", "CH", "27.00", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "EXIL GMBH, ZUERICH", "CH", "14.00", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "EXIL GMBH, ZUERICH", "CH", "14.00", "", "**** **** **** 1234"], - ["13.09.25", "15.09.25", "URBAN FOOD CLUTURE GMB, ZURICH", "CH", "135.00", "", "**** **** **** 1234"], - ["14.09.25", "15.09.25", "Google One, 650-2530000", "US", "100.00", "", "**** **** **** 1234"], - ["15.09.25", "16.09.25", "Ex Libris AG, Dietikon", "CH", "13.00", "", "**** **** **** 1234"], - ["15.09.25", "16.09.25", "Coop-1252 Wald, Wald ZH", "CH", "51.45", "", "**** **** **** 1234"], - ["16.09.25", "17.09.25", "Shell Waldhof, Wald ZH", "CH", "5.80", "", "**** **** **** 1234"], - ["19.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "16.05", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "14.60", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "13.55", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "13.90", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "Coop-1252 Wald, Wald ZH", "CH", "60.75", "", "**** **** **** 1234"], - ["20.09.25", "22.09.25", "MORE BAR GMBH, BUBIKON", "CH", "70.00", "", "**** **** **** 1234"], - ["21.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "6.40", "", "**** **** **** 1234"], - ["21.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "4.20", "", "**** **** **** 1234"], - ["21.09.25", "22.09.25", "LS Pirates AG, Hinwil", "CH", "13.45", "", "**** **** **** 1234"], - ["22.09.25", "23.09.25", "Migros M Wald, Wald ZH", "CH", "16.80", "", "**** **** **** 1234"], - ["22.09.25", "23.09.25", "BLEICHI + HOTEL, WALD", "CH", "43.00", "", "**** **** **** 1234"], - ["23.09.25", "24.09.25", "Coop-1252 Wald, Wald ZH", "CH", "155.75", "", "**** **** **** 1234"], - ["24.09.25", "25.09.25", "BKG*HOTEL AT BOOKING.C, (888)850-3958", "NL", "EUR 177.35", "170.35", "**** **** **** 1234"], - ["27.09.25", "29.09.25", "APODRO APOTHEKE WALD, WALD ZH", "CH", "21.50", "", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "SELVA HOSTELERIA, MACANET DE LA", "ES", "15.75", "", "**** **** **** 1234"], - ["28.09.25", "29.09.25", "AREAS LA SELVA, BARCELONA", "ES", "EUR 19.11", "18.40", "**** **** **** 1234"], - ["02.10.25", "06.10.25", "GOOGLE *YouTube Member, g.co/helppay#", "GB", "15.00", "", "**** **** **** 1234"], - ["01.10.25", "06.10.25", "Eventfrog.c 737909203525, Olten", "CH", "114.95", "", "**** **** **** 1234"], - ["06.10.25", "07.10.25", "digitec Galaxus (Online), Zurich", "CH", "23.80", "", "**** **** **** 1234"], - ["08.10.25", "09.10.25", "E.S.LA CLARIANA, MADRID", "ES", "EUR 29.58", "28.35", "**** **** **** 1234"], - ["10.10.25", "13.10.25", "B2BAND.COM, BRATISLAVA", "SK", "72.45", "", "**** **** **** 1234"], - ["10.10.25", "13.10.25", "Ticketcorner*89987227, 410900800800", "CH", "199.80", "", "**** **** **** 1234"], - ["10.10.25", "13.10.25", "SP NORAYA, RUMISBERG", "CH", "79.90", "", "**** **** **** 1234"], - ["11.10.25", "13.10.25", "B2BAND.COM, BRATISLAVA", "SK", "139.95", "", "**** **** **** 1234"], - ["11.10.25", "13.10.25", "TEMU.COM, BASEL", "CH", "81.20", "", "**** **** **** 1234"], - ["11.10.25", "13.10.25", "NOII.CH DATING, WINTERTHUR", "CH", "74.10", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "Rest Volkshaus, Zurich", "CH", "9.00", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "Shell Heuberg, Forch", "CH", "100.10", "", "**** **** **** 1234"], - ["12.10.25", "13.10.25", "Parkhaus Helvetiaplatz, Zurich", "CH", "8.00", "", "**** **** **** 1234"], - ["14.10.25", "15.10.25", "P2 Parkhaus Ein- & Ausfah, Zurich CH", "CHF", "5.00", "", "**** **** **** 1234"], - ["14.10.25", "15.10.25", "Migros Zurich Airport, Zurich CH", "CHF", "16.35", "", "**** **** **** 1234"], - ["14.10.25", "15.10.25", "GITHUB, INC., GITHUB.COM US", "USD", "0.30", "0.25", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "Dosenbach Schuhe & Sport, Hinwil CH", "CHF", "50.00", "", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "Coop-4253 Hinwil Wasseri, Hinwil CH", "CHF", "257.20", "", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "Landi, Wald CH", "CHF", "67.85", "", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "Puls Apotheke & Drogerie, Hinwil CH", "CHF", "9.20", "", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "CLAUDE.AI SUBSCRIPTION, ANTHROPIC.COM US", "USD", "108.10", "89.50", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "SV (Schweiz) AG, 27960, Zurich ETH-Ze CH", "CHF", "7.80", "", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "SV (Schweiz) AG, 27960, Zurich ETH-Ze CH", "CHF", "14.50", "", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "SV (Schweiz) AG, 27960, Zurich ETH-Ze CH", "CHF", "4.20", "", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "Universitatsspital Zurich, Zurich CH", "CHF", "30.00", "", "**** **** **** 1234"], - ["18.10.25", "20.10.25", "HubSpot Germany GmbH, Berlin DE", "EUR", "267.55", "256.05", "**** **** **** 1234"], - ["18.10.25", "20.10.25", "Google PixVerse AI Vi, 650-2530000 US", "USD", "5.20", "", "**** **** **** 1234"], - ["19.10.25", "20.10.25", "Shell Waldhof, Wald ZH CH", "CHF", "7.20", "", "**** **** **** 1234"], - ["19.10.25", "20.10.25", "KONDITOREI VOLAND WALD, WALD ZH CH", "CHF", "20.30", "", "**** **** **** 1234"], - ["19.10.25", "20.10.25", "KONDITOREI VOLAND WALD, WALD ZH CH", "CHF", "11.10", "", "**** **** **** 1234"], - ["18.10.25", "20.10.25", "ANTHROPIC, ANTHROPIC.COM US", "USD", "108.10", "88.75", "**** **** **** 1234"], - ["20.10.25", "21.10.25", "APCOA, Dubendorf CH", "CHF", "20.00", "", "**** **** **** 1234"], - ["20.10.25", "21.10.25", "STWEG Ambassador House, Glattbrugg CH", "CHF", "5.00", "", "**** **** **** 1234"], - ["23.10.25", "24.10.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "199.85", "", "**** **** **** 1234"], - ["24.10.25", "24.10.25", "Ticketcorner*90004263, 410900800800 CH", "CHF", "159.75", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Google Duolingo Langu, 650-2530000 US", "USD", "10.00", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Hornbach Baumarkt Galgene, Galgenen CH", "CHF", "1.50", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Hornbach Baumarkt Galgene, Galgenen CH", "CHF", "814.10", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "REMO WUEST BACK. KOND., GALGENEN CH", "CHF", "20.00", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Google PixVerse AI Vi, 650-2530000 US", "USD", "5.20", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "12.90", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "WAL*WESTHIVE, DUEBENDORF CH", "CHF", "15.30", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "WAL*WESTHIVE, DUEBENDORF CH", "CHF", "6.50", "", "**** **** **** 1234"], - ["30.10.25", "31.10.25", "Coop-4253 Hinwil Wasseri, Hinwil CH", "CHF", "139.85", "", "**** **** **** 1234"], - ["30.10.25", "31.10.25", "Coop-4054 Hinwil Restaura, Hinwil CH", "CHF", "34.95", "", "**** **** **** 1234"], - ["31.10.25", "03.11.25", "Coop-1911 Ruti, Ruti ZH CH", "CHF", "66.50", "", "**** **** **** 1234"], - ["31.10.25", "03.11.25", "OPENAI *CHATGPT SUBSCR, OPENAI.COM US", "USD", "216.20", "178.70", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "GOOGLE *ADS5192965135, cc§google.com IE", "", "79.15", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "KONDITOREI VOLAND WALD, WALD ZH CH", "CHF", "99.60", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "LANGDOCK GMBH, BERLIN DE", "EUR", "25.00", "23.90", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "Google PixVerse AI Vi, 650-2530000 US", "USD", "5.20", "", "**** **** **** 1234"], - ["02.11.25", "03.11.25", "GOOGLE *YouTubePremium, g.co/helppay# GB", "", "33.90", "", "**** **** **** 1234"], - ["02.11.25", "03.11.25", "Agrola TopShop Wald, Wald ZH CH", "CHF", "119.45", "", "**** **** **** 1234"], - ["03.11.25", "03.11.25", "Netflix.com, Los Gatos NL", "", "20.90", "", "**** **** **** 1234"], - ["03.11.25", "04.11.25", "www.fust.ch, Oberburen CH", "CHF", "1'560.90", "", "**** **** **** 1234"], - ["06.11.25", "07.11.25", "Grand Casino Luzern AG, Luzern CH", "CHF", "100.00", "108.00", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "Migros M Mutschellen, Berikon CH", "CHF", "0.40", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "Migros M Mutschellen, Berikon CH", "CHF", "15.90", "", "**** **** **** 1234"], - ["08.11.25", "10.11.25", "wondershare.com, Hong Kong HK", "", "25.95", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "WAL*WESTHIVE, DUEBENDORF CH", "CHF", "9.85", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "Google One, 650-2530000 US", "USD", "10.00", "", "**** **** **** 1234"], - ["08.11.25", "10.11.25", "Steiner-Beck AG, Wald ZH CH", "CHF", "32.20", "", "**** **** **** 1234"], - ["08.11.25", "10.11.25", "Google PixVerse AI Vi, 650-2530000 US", "USD", "5.20", "", "**** **** **** 1234"], - ["09.11.25", "10.11.25", "KONDITOREI VOLAND WALD, WALD ZH CH", "CHF", "25.80", "", "**** **** **** 1234"], - ["09.11.25", "11.11.25", "Starfood AG (Schweiz), Rothenburg CH", "CHF", "16.00", "", "**** **** **** 1234"], - ["09.11.25", "11.11.25", "Starfood AG (Schweiz), Rothenburg CH", "CHF", "16.00", "", "**** **** **** 1234"], - ["10.11.25", "11.11.25", "Coop-2253 Jona Eisenhof, Jona CH", "CHF", "161.25", "", "**** **** **** 1234"], - ["12.11.25", "13.11.25", "Hess AG Erdbau + Recy, Laupen ZH CH", "CHF", "39.20", "", "**** **** **** 1234"], - ["12.11.25", "13.11.25", "Jumbo-6017 Hinwil, Hinwil CH", "CHF", "173.70", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "Migros MM Ruti, Ruti ZH CH", "CHF", "57.90", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "Migros MM Ruti, Ruti ZH CH", "CHF", "140.10", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "WAL*WESTHIVE, DUEBENDORF CH", "CHF", "22.30", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "UDIO.COM, UDIO.COM US", "EUR", "36.00", "34.35", "**** **** **** 1234"], - ["15.10.25", "16.10.25", "Coop-4958 Rapperswil, Rapperswil SG CH", "CHF", "4.95", "", "**** **** **** 1234"], - ["16.10.25", "17.10.25", "TEMU.COM, BASEL CH", "CHF", "61.50", "", "**** **** **** 1234"], - ["16.10.25", "17.10.25", "TEMU.COM, BASEL CH", "CHF", "12.95", "", "**** **** **** 1234"], - ["16.10.25", "17.10.25", "TEMU.COM, BASEL CH", "CHF", "32.30", "", "**** **** **** 1234"], - ["16.10.25", "17.10.25", "TEMU.COM, BASEL CH", "CHF", "17.95", "", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "SBB Bahnhof Ruti ZH, Ruti ZH CH", "CHF", "54.00", "", "**** **** **** 1234"], - ["17.10.25", "20.10.25", "Candrian Catering AG 2, Zurich CH", "CHF", "15.50", "", "**** **** **** 1234"], - ["20.10.25", "21.10.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "178.95", "", "**** **** **** 1234"], - ["21.10.25", "22.10.25", "Denner Ruti ZH, Ruti ZH CH", "CHF", "50.15", "", "**** **** **** 1234"], - ["24.10.25", "27.10.25", "TEMU.COM, BASEL CH", "CHF", "100.65", "", "**** **** **** 1234"], - ["24.10.25", "27.10.25", "WAL*HAAR SHOP CH AG, UETENDORF CH", "CHF", "70.35", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "47.00", "", "**** **** **** 1234"], - ["25.10.25", "27.10.25", "Shell Waldhof, Wald ZH CH", "CHF", "3.20", "", "**** **** **** 1234"], - ["26.10.25", "27.10.25", "TEMU.COM, BASEL CH", "CHF", "63.10", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "ONLY, Hinwil CH", "CHF", "222.60", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Coop-4253 Hinwil Wasseri, Hinwil CH", "CHF", "104.10", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Coop-4253 Hinwil Wasseri, Hinwil CH", "CHF", "24.95", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Manor AG, Hinwil CH", "CHF", "177.25", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "H & M, Hinwil CH", "CHF", "43.85", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Coop-4253 Hinwil Wasseri, Hinwil CH", "CHF", "52.30", "", "**** **** **** 1234"], - ["27.10.25", "28.10.25", "Manor AG, Hinwil CH", "CHF", "59.05", "", "**** **** **** 1234"], - ["28.10.25", "29.10.25", "Migros MM Rapperswil, Rapperswil SG CH", "CHF", "23.35", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "ROSSMANN Schweiz AG, Wallisellen CH", "CHF", "13.95", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "Migros MR Glattzentrum, Glattzentrum CH", "CHF", "42.20", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "Calzedonia, Wallisellen CH", "CHF", "178.25", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "Intimissimi, Wallisellen CH", "CHF", "90.20", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "New Yorker (Schweiz) GmbH, Wetzikon ZH CH", "CHF", "76.80", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "New Yorker (Schweiz) GmbH, Wetzikon ZH CH", "CHF", "7.95", "", "**** **** **** 1234"], - ["29.10.25", "30.10.25", "Golden Bar GmbH, Wald ZH CH", "CHF", "40.00", "", "**** **** **** 1234"], - ["30.10.25", "31.10.25", "Frischer Max, Zurich CH", "CHF", "12.60", "", "**** **** **** 1234"], - ["30.10.25", "31.10.25", "Frischer Max, Zurich CH", "CHF", "4.20", "", "**** **** **** 1234"], - ["30.10.25", "31.10.25", "Halle 622, Zurich CH", "CHF", "15.75", "", "**** **** **** 1234"], - ["31.10.25", "03.11.25", "SBB Bahnhof Ruti ZH, Ruti ZH CH", "CHF", "27.00", "", "**** **** **** 1234"], - ["31.10.25", "03.11.25", "Eventfrog.c 739003945141, Olten CH", "CHF", "67.85", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "SBB Bahnhof Ruti ZH, Ruti ZH CH", "CHF", "27.00", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "AMERON ZUERICH, ZUERICH CH", "CHF", "30.00", "", "**** **** **** 1234"], - ["31.10.25", "03.11.25", "SKYLINE EVENTS, ZUERICH CH", "CHF", "13.50", "", "**** **** **** 1234"], - ["02.11.25", "03.11.25", "AURA Event Saal, Zuerich CH", "CHF", "15.75", "", "**** **** **** 1234"], - ["02.11.25", "03.11.25", "GOOGLE *YouTube Member, g.co/helppay# GB", "", "15.00", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "VBZ Bellevue, Zurich CH", "CHF", "2.80", "", "**** **** **** 1234"], - ["01.11.25", "03.11.25", "WAL*CLUB BELLEVUE, HOERI CH", "CHF", "16.50", "", "**** **** **** 1234"], - ["02.11.25", "03.11.25", "MCDONALDS ZUERICH 2016, ZUERICH CH", "CHF", "10.50", "", "**** **** **** 1234"], - ["03.11.25", "04.11.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "191.15", "", "**** **** **** 1234"], - ["05.11.25", "06.11.25", "Coop-1252 Wald, Wald ZH CH", "CHF", "51.35", "", "**** **** **** 1234"], - ["06.11.25", "07.11.25", "Ticketcorner*90024523, 410900800800 CH", "CHF", "158.75", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "SUMUP *JW BROW&LASH, LACHEN CH", "CHF", "290.00", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "Agrola TopShop Wald, Wald ZH CH", "CHF", "104.50", "", "**** **** **** 1234"], - ["07.11.25", "10.11.25", "Agrola TopShop Wald, Wald ZH CH", "CHF", "10.30", "", "**** **** **** 1234"], - ["08.11.25", "10.11.25", "Pizza Thal GmbH, Murgenthal CH", "CHF", "19.50", "", "**** **** **** 1234"], - ["09.11.25", "10.11.25", "TEMU.COM, BASEL CH", "CHF", "190.85", "", "**** **** **** 1234"], - ["10.11.25", "11.11.25", "Sinora GmbH, Bonstetten CH", "CHF", "115.20", "", "**** **** **** 1234"], - ["10.11.25", "11.11.25", "WAL*HAAR SHOP CH AG, UETENDORF CH", "CHF", "33.85", "", "**** **** **** 1234"], - ["11.11.25", "12.11.25", "Bleiche Fitness, Wald ZH CH", "CHF", "90.00", "", "**** **** **** 1234"], - ["11.11.25", "12.11.25", "Parkhaus Urania, Zurich CH", "CHF", "14.00", "", "**** **** **** 1234"], - ["12.11.25", "13.11.25", "Coop-4958 Rapperswil, Rapperswil SG CH", "CHF", "24.80", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "TEMU.COM, BASEL CH", "CHF", "56.00", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "TEMU.COM, BASEL CH", "CHF", "5.95", "", "**** **** **** 1234"], - ["13.11.25", "14.11.25", "TEMU.COM, BASEL CH", "CHF", "15.25", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "Santa Lucia Altstetten, Zurich", "CH", "38.00", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "Agrola TopShop Wald, Wald ZH", "CH", "126.80", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "GITHUB, INC., GITHUB.COM", "US", "USD 0.70", "0.60", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Jumbo-6017 Hinwil, Hinwil", "CH", "53.85", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "57.00", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "13.95", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "NEGISHI ALTSTETTEN BAH, ZUERICH", "CH", "31.90", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Google PixVerse AI Vi, 650-2530000", "US", "5.20", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "CANVA* I04701-26464248, CANVA.COM", "US", "12.00", "", "**** **** **** 1234"], - ["17.11.25", "18.11.25", "ANTHROPIC, ANTHROPIC.COM", "US", "USD 270.25", "220.65", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "Coop-1252 Wald, Wald ZH", "CH", "7.80", "", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "19.30", "", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "TIERARZTPRAXIS BACHTEL, WALD ZH", "CH", "343.30", "", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "ANTHROPIC, ANTHROPIC.COM", "US", "USD 5.41", "4.45", "**** **** **** 1234"], - ["18.11.25", "20.11.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "19.35", "", "**** **** **** 1234"], - ["19.11.25", "20.11.25", "Wuest Partner, Zurich", "CH", "324.30", "", "**** **** **** 1234"], - ["19.11.25", "21.11.25", "SHOP.ASFINAG.AT, WIEN", "AT", "EUR 12.40", "11.80", "**** **** **** 1234"], - ["20.11.25", "21.11.25", "Coop-1252 Wald, Wald ZH", "CH", "85.35", "", "**** **** **** 1234"], - ["20.11.25", "21.11.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "17.95", "", "**** **** **** 1234"], - ["20.11.25", "21.11.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "6.30", "", "**** **** **** 1234"], - ["21.11.25", "24.11.25", "STWEG Ambassador House, Glattbrugg", "CH", "7.50", "", "**** **** **** 1234"], - ["21.11.25", "24.11.25", "Migros M Dubendorf Stettb, Dubendorf", "CH", "16.95", "", "**** **** **** 1234"], - ["21.11.25", "24.11.25", "MCDONALDS RESTAURANT G, WALLISELLEN", "CH", "13.00", "", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "Ski- und Snowboard-Center, Neuhaus SG", "CH", "128.00", "", "**** **** **** 1234"], - ["21.11.25", "24.11.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "408.25", "", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "GOOGLE *Duolingo Langu, g.co/HelpPay#", "US", "9.20", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "Migros Santispark Bad, Abtwil SG", "CH", "48.60", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "Migros Santispark Bad, Abtwil SG", "CH", "8.50", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "Migros ELS Santispark PH, Abtwil SG", "CH", "3.00", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "AVIA Service Autogrill, St. Margrethe", "CH", "121.80", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "AVIA Service Autogrill, St. Margrethe", "CH", "10.50", "", "**** **** **** 1234"], - ["23.11.25", "24.11.25", "KONDITOREI VOLAND LAUP, LAUPEN ZH", "CH", "62.80", "", "**** **** **** 1234"], - ["23.11.25", "25.11.25", "SHOP.ASFINAG.AT, WIEN", "AT", "EUR 9.30", "8.90", "**** **** **** 1234"], - ["24.11.25", "25.11.25", "Landi, Wald", "CH", "27.15", "", "**** **** **** 1234"], - ["24.11.25", "26.11.25", "SHOP.ASFINAG.AT, WIEN", "AT", "EUR 12.50", "11.95", "**** **** **** 1234"], - ["26.11.25", "27.11.25", "MyPlace, Affoltern am", "CH", "10.30", "", "**** **** **** 1234"], - ["27.11.25", "28.11.25", "Coop-1911 Ruti, Ruti ZH", "CH", "57.20", "", "**** **** **** 1234"], - ["28.11.25", "01.12.25", "Manor AG, Hinwil", "CH", "10.10", "", "**** **** **** 1234"], - ["28.11.25", "01.12.25", "Manor AG, Hinwil", "CH", "136.25", "", "**** **** **** 1234"], - ["28.11.25", "01.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "205.35", "", "**** **** **** 1234"], - ["01.12.25", "02.12.25", "GOOGLE *ADS5192965135, cc§google.com", "IE", "59.00", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "112.50", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "Coop-1252 Wald, Wald ZH", "CH", "117.70", "", "**** **** **** 1234"], - ["03.12.25", "03.12.25", "Autodesk ADY, Dublin 2", "IE", "1'989.05", "", "**** **** **** 1234"], - ["03.12.25", "03.12.25", "NETFLIX.COM, Amsterdam", "NL", "22.90", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "TAVILY AI, WWW.TAVILY.CO", "US", "USD 17.48", "14.50", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "GOOGLE *YouTubePremium, g.co/HelpPay#", "US", "33.90", "", "**** **** **** 1234"], - ["04.12.25", "05.12.25", "Migros M Dubendorf Stettb, Dubendorf", "CH", "103.20", "", "**** **** **** 1234"], - ["04.12.25", "05.12.25", "WAL*WESTHIVE, DUEBENDORF", "CH", "19.80", "", "**** **** **** 1234"], - ["05.12.25", "08.12.25", "MICROSOFT#G127221615, MSBILL.INFO", "CH", "55.20", "", "**** **** **** 1234"], - ["04.12.25", "08.12.25", "Ristorante Amalfi AG, Zurich", "CH", "67.00", "", "**** **** **** 1234"], - ["05.12.25", "08.12.25", "Landi, Wald", "CH", "11.90", "", "**** **** **** 1234"], - ["05.12.25", "08.12.25", "Notariat Wald, Wald ZH", "CH", "40.00", "", "**** **** **** 1234"], - ["05.12.25", "08.12.25", "Coop-1252 Wald, Wald ZH", "CH", "149.75", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "TIERARZTPRAXIS BACHTEL, WALD ZH", "CH", "80.30", "", "**** **** **** 1234"], - ["07.12.25", "08.12.25", "HERAHELP.COM, 0044330027088", "CY", "EUR 19.95", "19.25", "**** **** **** 1234"], - ["07.12.25", "08.12.25", "Google One, 650-2530000", "US", "10.00", "", "**** **** **** 1234"], - ["10.12.25", "11.12.25", "TAVILY AI, WWW.TAVILY.CO", "US", "USD 43.26", "35.95", "**** **** **** 1234"], - ["11.12.25", "12.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "247.40", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "ONLY, Zurich", "CH", "101.75", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "SUMUP *MARYS COSMETICS, USTER", "CH", "419.00", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "S2P*Calzedonia, 0447554090", "IT", "86.75", "", "**** **** **** 1234"], - ["14.11.25", "17.11.25", "Parkhaus Urania, Zurich", "CH", "12.00", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "JustEat, Zurich", "CH", "193.70", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "ONLY, Hinwil", "CH", "126.10", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "242.70", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Manor AG, Hinwil", "CH", "35.35", "", "**** **** **** 1234"], - ["15.11.25", "17.11.25", "Valentyna Nails, R?ti", "CH", "160.00", "", "**** **** **** 1234"], - ["13.11.25", "17.11.25", "redcare-apotheke, Sevenum", "NL", "79.90", "", "**** **** **** 1234"], - ["16.11.25", "17.11.25", "NORDSTERN, Basel", "CH", "64.20", "", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "La Makeup Sp. z. o.o., Warsaw", "PL", "104.85", "", "**** **** **** 1234"], - ["18.11.25", "19.11.25", "Bleiche Fitness, Wald ZH", "CH", "90.00", "", "**** **** **** 1234"], - ["20.11.25", "21.11.25", "Bleiche Fitness, Wald ZH", "CH", "90.00", "", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "SHELL PETTNAU 5538, PETTNAU", "AT", "94.60", "", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "SHELL PETTNAU 5538, PETTNAU", "AT", "EUR 7.39", "7.05", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "SHELL PETTNAU 5538, PETTNAU", "AT", "EUR 4.39", "4.20", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "Coop-1252 Wald, Wald ZH", "CH", "57.85", "", "**** **** **** 1234"], - ["22.11.25", "24.11.25", "ASFINAG S16 HMS ST JAKOB, ST.ANTON/ARLB", "AT", "EUR 12.50", "11.95", "**** **** **** 1234"], - ["24.11.25", "25.11.25", "Posthotel Achenkirc, Achenkirch", "AT", "EUR 1'211.80", "1'160.25", "**** **** **** 1234"], - ["24.11.25", "25.11.25", "MS* CITYPOP2NIGHTBASE, ZURICH", "CH", "15.00", "", "**** **** **** 1234"], - ["24.11.25", "25.11.25", "MS* CITYPOP2NIGHTBASE, ZURICH", "CH", "8.40", "", "**** **** **** 1234"], - ["24.11.25", "25.11.25", "BKG*BOOKING.COM HOTEL, (888)850-3958", "NL", "187.95", "", "**** **** **** 1234"], - ["25.11.25", "26.11.25", "Coop-1252 Wald, Wald ZH", "CH", "63.00", "", "**** **** **** 1234"], - ["25.11.25", "26.11.25", "Bleiche Fitness, Wald ZH", "CH", "90.00", "", "**** **** **** 1234"], - ["26.11.25", "27.11.25", "Hallenbad Wald, Wald ZH", "CH", "54.00", "", "**** **** **** 1234"], - ["27.11.25", "28.11.25", "Bestseller AS, Amsterdam", "NL", "35.90", "", "**** **** **** 1234"], - ["29.11.25", "01.12.25", "CLUB NORDSTERN, BASEL", "CH", "6.00", "", "**** **** **** 1234"], - ["29.11.25", "01.12.25", "CLUB NORDSTERN, BASEL", "CH", "6.00", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "TICKETCORNER CH, RUEMLANG", "CH", "84.90", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "Shell Waldhof, Wald ZH", "CH", "126.15", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "Shell Waldhof, Wald ZH", "CH", "3.70", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "Bleiche Fitness, Wald ZH", "CH", "90.00", "", "**** **** **** 1234"], - ["02.12.25", "03.12.25", "GOOGLE *YouTube Member, g.co/HelpPay#", "US", "15.00", "", "**** **** **** 1234"], - ["03.12.25", "04.12.25", "APODRO APOTHEKE WALD, WALD ZH", "CH", "54.90", "", "**** **** **** 1234"], - ["04.12.25", "05.12.25", "TICKETCORNER CH, RUEMLANG", "CH", "285.70", "", "**** **** **** 1234"], - ["05.12.25", "05.12.25", "NOII.CH DATING, WINTERTHUR", "CH", "74.10", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "TEMU.COM, BASEL", "CH", "72.50", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "326.85", "", "**** **** **** 1234"], - ["06.12.25", "08.12.25", "Decathlon, Hinwil", "CH", "150.00", "", "**** **** **** 1234"], -["07.12.25", "09.12.25", "Migros-Genossenschafts-Bund, Zürich", "CH", "200.00", "", "**** **** **** 1234"], -["08.12.25", "10.12.25", "Zürich HB, Zürich", "CH", "45.00", "", "**** **** **** 1234"], -["09.12.25", "11.12.25", "Amazon Marketplace, amazon.de", "DE", "120.00", "", "**** **** **** 1234"], -["10.12.25", "12.12.25", "IKEA, Dietlikon", "CH", "350.00", "", "**** **** **** 1234"], -["11.12.25", "13.12.25", "Manor, Zürich", "CH", "75.00", "", "**** **** **** 1234"], -["12.12.25", "14.12.25", "Zalando, zalando.ch", "CH", "90.00", "", "**** **** **** 1234"], -["13.12.25", "15.12.25", "SBB CFF FFS, Bern", "CH", "60.00", "", "**** **** **** 1234"], -["14.12.25", "16.12.25", "Apple Store, Zürich", "CH", "999.00", "", "**** **** **** 1234"], -["15.12.25", "17.12.25", "Migros-Genossenschafts-Bund, Zürich", "CH", "150.00", "", "**** **** **** 1234"], -["16.12.25", "18.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "250.00", "", "**** **** **** 1234"], -["17.12.25", "19.12.25", "Shell Waldhof, Wald ZH", "CH", "60.00", "", "**** **** **** 1234"], -["18.12.25", "20.12.25", "Zürich HB, Zürich", "CH", "30.00", "", "**** **** **** 1234"], -["19.12.25", "21.12.25", "Amazon Marketplace, amazon.de", "DE", "80.00", "", "**** **** **** 1234"], -["20.12.25", "22.12.25", "IKEA, Dietlikon", "CH", "400.00", "", "**** **** **** 1234"], -["21.12.25", "23.12.25", "Manor, Zürich", "CH", "100.00", "", "**** **** **** 1234"], -["22.12.25", "24.12.25", "Zalando, zalando.ch", "CH", "110.00", "", "**** **** **** 1234"], -["23.12.25", "25.12.25", "SBB CFF FFS, Bern", "CH", "70.00", "", "**** **** **** 1234"], -["24.12.25", "26.12.25", "Apple Store, Zürich", "CH", "1200.00", "", "**** **** **** 1234"], -["25.12.25", "27.12.25", "Migros-Genossenschafts-Bund, Zürich", "CH", "180.00", "", "**** **** **** 1234"], -["26.12.25", "28.12.25", "Coop-4253 Hinwil Wasseri, Hinwil", "CH", "300.00", "", "**** **** **** 1234"], -["27.12.25", "29.12.25", "Shell Waldhof, Wald ZH", "CH", "70.00", "", "**** **** **** 1234"], -["28.12.25", "30.12.25", "Zürich HB, Zürich", "CH", "40.00", "", "**** **** **** 1234"], -["29.12.25", "31.12.25", "Amazon Marketplace, amazon.de", "DE", "100.00", "", "**** **** **** 1234"], -["30.12.25", "01.01.26", "IKEA, Dietlikon", "CH", "450.00", "", "**** **** **** 1234"], -["31.12.25", "02.01.26", "Manor, Zürich", "CH", "125.00", "", "**** **** **** 1234"] -] -} -} -] -} -================================================================================ diff --git a/modules/services/serviceAi/subAiCallLooping-flow.md b/modules/services/serviceAi/subAiCallLooping-flow.md deleted file mode 100644 index 0a7ac854..00000000 --- a/modules/services/serviceAi/subAiCallLooping-flow.md +++ /dev/null @@ -1,239 +0,0 @@ -# AI Call Iteration Flow - JSON Merging System - -This document describes the iteration flow for handling large JSON responses from AI that may be truncated and need to be merged across multiple iterations. - -## Overview - -When an AI response is too large, it may be truncated (cut) at an arbitrary point. The iteration system: -1. Detects incomplete JSON -2. Requests continuation from the AI -3. Merges the continuation with the existing JSON -4. Repeats until complete or max failures reached - ---- - -## Key Variables - -| Variable | Type | Purpose | -|----------|------|---------| -| `jsonBase` | `str \| None` | The merged JSON string (CUT version for overlap matching) | -| `candidateJson` | `str` | Temporary holder for merged result until validated | -| `lastValidCompletePart` | `str \| None` | Fallback - last successfully parsed CLOSED JSON | -| `lastOverlapContext` | `str` | Context for retry/continuation prompts | -| `lastHierarchyContextForPrompt` | `str` | Context for retry/continuation prompts | -| `mergeFailCount` | `int` | Global counter (max 3 failures) | - ---- - -## Key Distinction: hierarchyContext vs completePart - -| Field | Description | Use Case | -|-------|-------------|----------| -| `hierarchyContext` | **CUT JSON** - truncated at cut point | Used as `jsonBase` for merging with next AI fragment | -| `completePart` | **CLOSED JSON** - all structures properly closed | Used for validation, parsing, and fallback | - -**Why this matters:** -- The next AI fragment starts with an **overlap** that matches the CUT point -- If we used `completePart` (closed), the overlap detection would FAIL -- We must use `hierarchyContext` (cut) so overlap matching works correctly - ---- - -## Flow Steps - -### Step 1: BUILD PROMPT - -**Location:** `subAiCallLooping.py` lines 163-212 -**Function:** `buildContinuationContext()` from `modules/shared/jsonUtils.py` - -- **First iteration:** Use original prompt -- **Continuation:** `buildContinuationContext(allSections, lastRawResponse, ...)` - - Internally calls `getContexts(lastRawResponse)` to get overlap/hierarchy - - Builds continuation prompt with `overlapContext` + `hierarchyContextForPrompt` - -### Step 2: CALL AI - -**Location:** `subAiCallLooping.py` lines 214-299 -**Function:** `self.aiService.callAi(request)` - -- Returns `response.content` as `result` -- NOTE: Do NOT update `lastRawResponse` yet! (only after successful merge) - -### Step 4: MERGE - -**Location:** `subAiCallLooping.py` lines 338-396 -**Function:** `JsonResponseHandler.mergeJsonStringsWithOverlap()` from `modules/services/serviceAi/subJsonResponseHandling.py` - -``` -IF first iteration (jsonBase is None): - → candidateJson = result -ELSE: - → mergedJsonString, hasOverlap = mergeJsonStringsWithOverlap(jsonBase, result) - - IF hasOverlap = False (MERGE FAILED): - → mergeFailCount++ - → If mergeFailCount >= 3: return lastValidCompletePart (fallback) - → Else: continue (retry with unchanged jsonBase AND lastRawResponse!) - ELSE: - → candidateJson = mergedJsonString (don't update jsonBase yet!) - -→ lastRawResponse = candidateJson (ONLY after first iteration or successful merge!) - -TRY DIRECT PARSE of candidateJson: - IF parse succeeds: - → jsonBase = candidateJson (commit) - → FINISHED! Return normalized result - ELSE: - → Proceed to Step 5 -``` - -### Step 5: GET CONTEXTS - -**Location:** `subAiCallLooping.py` lines 420-427 -**Function:** `getContexts()` from `modules/shared/jsonContinuation.py` - -```python -contexts = getContexts(candidateJson) -``` - -Returns `JsonContinuationContexts`: -- `overlapContext`: `""` if JSON is complete (no cut point) -- `hierarchyContext`: CUT JSON (for merging with next fragment) -- `hierarchyContextForPrompt`: CUT JSON with budget limits (for prompts) -- `completePart`: CLOSED JSON (repaired if needed) -- `jsonParsingSuccess`: `True` if completePart is valid JSON - -**Enhancement:** If original JSON is already complete → `overlapContext = ""` -This signals "JSON is complete, no more continuation needed" - -### Step 6: DECIDE - -**Location:** `subAiCallLooping.py` lines 429-528 - -#### Case A: `jsonParsingSuccess=true` AND `overlapContext=""` -**→ FINISHED** -- JSON is complete (no cut point) -- `jsonBase = contexts.completePart` (use CLOSED version for final result) -- Return `completePart` as result - -#### Case B: `jsonParsingSuccess=true` AND `overlapContext!=""` -**→ CONTINUE to next iteration** -- JSON parseable but has cut point -- `jsonBase = contexts.hierarchyContext` ← **CUT version for next merge!** -- `lastValidCompletePart = contexts.completePart` ← **CLOSED version for fallback** -- Store contexts for next prompt -- `mergeFailCount = 0` (reset on success) -- `lastRawResponse = jsonBase` -- Continue to next iteration - -#### Case C: `jsonParsingSuccess=false` -**→ RETRY with same prompt** -- Do NOT update `jsonBase` (keep previous valid state) -- `mergeFailCount++` -- If `mergeFailCount >= 3`: return `lastValidCompletePart` (fallback) -- Else: continue (retry with unchanged jsonBase/lastRawResponse) - ---- - -## Flow Diagram - -``` - ┌───────────────────────────────────────────────────────────────┐ - │ ITERATION START │ - └───────────────────────────┬───────────────────────────────────┘ - │ - ┌───────────────────────────▼───────────────────────────────────┐ - │ STEP 1: BUILD PROMPT │ - │ - First: original prompt │ - │ - Next: buildContinuationContext(lastRawResponse) │ - └───────────────────────────┬───────────────────────────────────┘ - │ - ┌───────────────────────────▼───────────────────────────────────┐ - │ STEP 2: CALL AI → result │ - └───────────────────────────┬───────────────────────────────────┘ - │ - ┌───────────────────────────▼───────────────────────────────────┐ - │ STEP 4: MERGE jsonBase + result → candidateJson │ - └───────────────────────────┬───────────────────────────────────┘ - │ - ┌────────────▼────────────┐ - │ Merge OK? │ - └────────────┬────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - │ NO │ YES │ - ▼ ▼ │ - ┌──────────────┐ ┌──────────────────┐ │ - │ fails++ │ │ TRY DIRECT PARSE │ │ - │ if >=3: │ │ of candidateJson │ │ - │ RETURN │ └────────┬─────────┘ │ - │ fallback │ │ │ - │ else: RETRY │ ┌────────▼─────────┐ │ - │ (continue) │ │ Parse OK? │ │ - └──────────────┘ └────────┬─────────┘ │ - │ │ - ┌─────────────────────┼─────────────────────┐ - │ YES │ NO │ - ▼ ▼ │ - ┌──────────────┐ ┌──────────────────────────────┐ - │ FINISHED ✓ │ │ STEP 5: getContexts() │ - │ Return │ │ → jsonParsingSuccess │ - │ normalized │ │ → overlapContext │ - │ result │ └────────────┬─────────────────┘ - └──────────────┘ │ - ┌────────────▼────────────────────┐ - │ STEP 6: DECIDE │ - └────────────┬────────────────────┘ - │ - ┌────────────────────────────┼────────────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌───────────────────┐ ┌───────────────────────┐ ┌───────────────────┐ -│ success=true │ │ success=true │ │ success=false │ -│ overlap="" │ │ overlap!="" │ │ │ -│ ───────────── │ │ ───────────────── │ │ ───────────── │ -│ FINISHED ✓ │ │ CONTINUE │ │ RETRY │ -│ │ │ │ │ │ -│ jsonBase = │ │ jsonBase = │ │ jsonBase unchanged│ -│ completePart │ │ hierarchyContext │ │ fails++ │ -│ (CLOSED) │ │ (CUT for merge!) │ │ │ -│ │ │ │ │ if >=3: fallback │ -│ Return result │ │ fallback = │ │ else: retry │ -│ │ │ completePart │ │ │ -│ │ │ (CLOSED) │ │ │ -│ │ │ │ │ │ -│ │ │ Next iteration → │ │ │ -└───────────────────┘ └───────────────────────┘ └───────────────────┘ -``` - ---- - -## Files Involved - -| File | Purpose | -|------|---------| -| `modules/services/serviceAi/subAiCallLooping.py` | Main iteration loop | -| `modules/shared/jsonContinuation.py` | `getContexts()` - context extraction & repair | -| `modules/shared/jsonUtils.py` | `buildContinuationContext()` - prompt building | -| `modules/services/serviceAi/subJsonResponseHandling.py` | `mergeJsonStringsWithOverlap()` | -| `modules/services/serviceAi/subJsonMerger.py` | `ModularJsonMerger` - actual merge logic | -| `modules/datamodels/datamodelAi.py` | `JsonContinuationContexts` model | - ---- - -## Error Handling - -### Merge Failures -- Max 3 consecutive failures allowed -- On failure: retry with unchanged `jsonBase` (previous valid state) -- After 3 failures: return `lastValidCompletePart` as fallback - -### Parse Failures -- If `getContexts()` cannot produce valid JSON: increment fail counter -- Retry with same prompt (don't update jsonBase) -- After 3 failures: return `lastValidCompletePart` as fallback - -### Fallback Strategy -- `lastValidCompletePart` stores the last successfully parsed CLOSED JSON -- Always available as fallback when things go wrong -- Ensures we return valid JSON even after multiple failures diff --git a/modules/services/serviceAi/subAiCallLooping.py b/modules/services/serviceAi/subAiCallLooping.py deleted file mode 100644 index 2e4edc3e..00000000 --- a/modules/services/serviceAi/subAiCallLooping.py +++ /dev/null @@ -1,665 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -AI Call Looping Module - -Handles AI calls with looping and repair logic, including: -- Looping with JSON repair and continuation -- KPI definition and tracking -- Progress tracking and iteration management - -FLOW LOGIC - -VARIABLES: -- jsonBase: str (merged JSON so far, starts empty) -- lastValidCompletePart: str (fallback for failures) -- mergeFailCount: int = 0 (max 3) - -FLOW: -┌─────────────────────────────────────────────────────────────────┐ -│ 1. BUILD PROMPT │ -│ - First: original prompt │ -│ - Next: buildContinuationContext(lastRawResponse) │ -├─────────────────────────────────────────────────────────────────┤ -│ 2. CALL AI → response fragment │ -├─────────────────────────────────────────────────────────────────┤ -│ 4. MERGE jsonBase + response │ -│ ├─ FAILS: repeat prompt, fails++ (if >=3 return fallback) │ -│ └─ SUCCEEDS: try parse │ -│ ├─ SUCCEEDS: FINISHED │ -│ └─ FAILS: → step 5 │ -├─────────────────────────────────────────────────────────────────┤ -│ 5. GET CONTEXTS (merge OK, parse failed) │ -│ getContexts(mergedJson) → │ -│ - If no cut point: overlapContext = "" │ -│ - Store contexts for next iteration │ -├─────────────────────────────────────────────────────────────────┤ -│ 6. DECIDE │ -│ ├─ jsonParsingSuccess=true AND overlapContext="": │ -│ │ FINISHED. return completePart │ -│ ├─ jsonParsingSuccess=true AND overlapContext!="": │ -│ │ CONTINUE, fails=0 │ -│ └─ ELSE: repeat prompt, fails++ │ -└─────────────────────────────────────────────────────────────────┘ - - -""" - -import json -import logging -from typing import Dict, Any, List, Optional, Callable - -from modules.datamodels.datamodelAi import ( - AiCallRequest, AiCallOptions -) -from modules.datamodels.datamodelExtraction import ContentPart -from .subJsonResponseHandling import JsonResponseHandler -from .subLoopingUseCases import LoopingUseCaseRegistry -from modules.workflows.processing.shared.stateTools import checkWorkflowStopped -from modules.shared.jsonContinuation import getContexts -from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson -from modules.shared.jsonUtils import tryParseJson -from modules.shared.jsonUtils import closeJsonStructures -from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText - -logger = logging.getLogger(__name__) - - -class AiCallLooper: - """Handles AI calls with looping and repair logic.""" - - def __init__(self, services, aiService, responseParser): - """Initialize AiCallLooper with service center, AI service, and response parser access.""" - self.services = services - self.aiService = aiService - self.responseParser = responseParser - self.useCaseRegistry = LoopingUseCaseRegistry() # Initialize use case registry - - async def callAiWithLooping( - self, - prompt: str, - options: AiCallOptions, - debugPrefix: str = "ai_call", - promptBuilder: Optional[Callable] = None, - promptArgs: Optional[Dict[str, Any]] = None, - operationId: Optional[str] = None, - userPrompt: Optional[str] = None, - contentParts: Optional[List[ContentPart]] = None, # ARCHITECTURE: Support ContentParts for large content - useCaseId: str = None # REQUIRED: Explicit use case ID - no auto-detection, no fallback - ) -> str: - """ - Shared core function for AI calls with repair-based looping system. - Automatically repairs broken JSON and continues generation seamlessly. - - Args: - prompt: The prompt to send to AI - options: AI call configuration options - debugPrefix: Prefix for debug file names - promptBuilder: Optional function to rebuild prompts for continuation - promptArgs: Optional arguments for prompt builder - operationId: Optional operation ID for progress tracking - userPrompt: Optional user prompt for KPI definition - contentParts: Optional content parts for first iteration - useCaseId: REQUIRED: Explicit use case ID - no auto-detection, no fallback - - Returns: - Complete AI response after all iterations - """ - # REQUIRED: useCaseId must be provided - no auto-detection, no fallback - if not useCaseId: - errorMsg = ( - "useCaseId is REQUIRED for callAiWithLooping. " - "No auto-detection - must explicitly specify use case ID. " - f"Available use cases: {list(self.useCaseRegistry.useCases.keys())}" - ) - logger.error(errorMsg) - raise ValueError(errorMsg) - - # Validate use case exists - useCase = self.useCaseRegistry.get(useCaseId) - if not useCase: - errorMsg = ( - f"Use case '{useCaseId}' not found in registry. " - f"Available use cases: {list(self.useCaseRegistry.useCases.keys())}" - ) - logger.error(errorMsg) - raise ValueError(errorMsg) - - maxIterations = 50 # Prevent infinite loops - iteration = 0 - allSections = [] # Accumulate all sections across iterations - lastRawResponse = None # Store last raw JSON response for continuation - - # JSON Base Iteration System: - # - jsonBase: the merged JSON string (replaces accumulatedDirectJson array) - # - After each iteration, new response is merged with jsonBase - # - On merge success: check if complete, store contexts for next iteration - # - On merge fail: retry with same prompt, increment fails - jsonBase = None # Merged JSON string (starts None, set on first response) - - # Merge fail tracking - stop after 3 consecutive merge failures - MAX_MERGE_FAILS = 3 - mergeFailCount = 0 # Global counter for merge failures across entire loop - lastValidCompletePart = None # Store last successfully parsed completePart for fallback - - # Get parent operation ID for iteration operations (parentId should be operationId, not log entry ID) - parentOperationId = operationId # Use the parent's operationId directly - - while iteration < maxIterations: - iteration += 1 - - # Create separate operation for each iteration with parent reference - iterationOperationId = None - if operationId: - iterationOperationId = f"{operationId}_iter_{iteration}" - self.services.chat.progressLogStart( - iterationOperationId, - "AI Call", - f"Iteration {iteration}", - "", - parentOperationId=parentOperationId - ) - - # Build iteration prompt - # CRITICAL: Build continuation prompt if we have sections OR if we have a previous response (even if broken) - # This ensures continuation prompts are built even when JSON is so broken that no sections can be extracted - if (len(allSections) > 0 or lastRawResponse) and promptBuilder and promptArgs: - # Extract templateStructure and basePrompt from promptArgs (REQUIRED) - templateStructure = promptArgs.get("templateStructure") - if not templateStructure: - raise ValueError( - f"templateStructure is REQUIRED in promptArgs for use case '{useCaseId}'. " - "Prompt creation functions must return (prompt, templateStructure) tuple." - ) - - basePrompt = promptArgs.get("basePrompt") - if not basePrompt: - # Fallback: use prompt parameter (should be the same) - basePrompt = prompt - logger.warning( - f"basePrompt not found in promptArgs for use case '{useCaseId}', " - "using prompt parameter instead. This may indicate a bug." - ) - - # This is a continuation - build continuation context with raw JSON and rebuild prompt - continuationContext = buildContinuationContext( - allSections, lastRawResponse, useCaseId, templateStructure - ) - if not lastRawResponse: - logger.warning(f"Iteration {iteration}: No previous response available for continuation!") - - # Store valid completePart from continuation context for fallback on merge failures - # Use getContexts to check if completePart is parseable and store it - if lastRawResponse and not lastValidCompletePart: - try: - contexts = getContexts(lastRawResponse) - if contexts.jsonParsingSuccess and contexts.completePart: - lastValidCompletePart = contexts.completePart - logger.debug(f"Iteration {iteration}: Stored initial valid completePart ({len(lastValidCompletePart)} chars)") - except Exception as e: - logger.debug(f"Iteration {iteration}: Failed to extract completePart: {e}") - - # Unified prompt builder call: Continuation builders only need continuationContext, templateStructure, and basePrompt - # All initial context (section, userPrompt, etc.) is already in basePrompt, so promptArgs is not needed - # Extract templateStructure and basePrompt from promptArgs (they're explicit parameters) - iterationPrompt = await promptBuilder( - continuationContext=continuationContext, - templateStructure=templateStructure, - basePrompt=basePrompt - ) - else: - # First iteration - use original prompt - iterationPrompt = prompt - - # Make AI call - try: - checkWorkflowStopped(self.services) - if iterationOperationId: - self.services.chat.progressLogUpdate(iterationOperationId, 0.3, "Calling AI model") - # ARCHITECTURE: Pass ContentParts directly to AiCallRequest - # This allows model-aware chunking to handle large content properly - # ContentParts are only passed in first iteration (continuations don't need them) - request = AiCallRequest( - prompt=iterationPrompt, - context="", - options=options, - contentParts=contentParts if iteration == 1 else None # Only pass ContentParts in first iteration - ) - - # Write the ACTUAL prompt sent to AI - # For section content generation: write prompt for first iteration and continuation iterations - # For document generation: write prompt for each iteration - isSectionContent = "_section_" in debugPrefix - if iteration == 1: - self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt") - elif isSectionContent: - # Save continuation prompts for section_content debugging - self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") - else: - # Document generation - save all iteration prompts - self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") - - response = await self.aiService.callAi(request) - result = response.content - - # Track bytes for progress reporting - bytesReceived = len(result.encode('utf-8')) if result else 0 - totalBytesSoFar = sum(len(section.get('content', '').encode('utf-8')) if isinstance(section.get('content'), str) else 0 for section in allSections) + bytesReceived - - # Update progress after AI call with byte information - if iterationOperationId: - # Format bytes for display (kB or MB) - if totalBytesSoFar < 1024: - bytesDisplay = f"{totalBytesSoFar}B" - elif totalBytesSoFar < 1024 * 1024: - bytesDisplay = f"{totalBytesSoFar / 1024:.1f}kB" - else: - bytesDisplay = f"{totalBytesSoFar / (1024 * 1024):.1f}MB" - self.services.chat.progressLogUpdate(iterationOperationId, 0.6, f"AI response received ({bytesDisplay})") - - # Write raw AI response to debug file - # For section content generation: write response for first iteration and continuation iterations - # For document generation: write response for each iteration - if iteration == 1: - self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") - elif isSectionContent: - # Save continuation responses for section_content debugging - self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}") - else: - # Document generation - save all iteration responses - self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}") - - # Note: Stats are now stored centrally in callAi() - no need to duplicate here - - # Check for error response using generic error detection (errorCount > 0 or modelName == "error") - if hasattr(response, 'errorCount') and response.errorCount > 0: - errorMsg = f"Iteration {iteration}: Error response detected (errorCount={response.errorCount}), stopping loop: {result[:200] if result else 'empty'}" - logger.error(errorMsg) - break - - if hasattr(response, 'modelName') and response.modelName == "error": - errorMsg = f"Iteration {iteration}: Error response detected (modelName=error), stopping loop: {result[:200] if result else 'empty'}" - logger.error(errorMsg) - break - - if not result or not result.strip(): - logger.warning(f"Iteration {iteration}: Empty response, stopping") - break - - # Check if this is a text response (not document generation) - # Text responses don't need JSON parsing - return immediately after first successful response - isTextResponse = (promptBuilder is None and promptArgs is None) or debugPrefix == "text" - - if isTextResponse: - # For text responses, return the text immediately - no JSON parsing needed - logger.info(f"Iteration {iteration}: Text response received, returning immediately") - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, True) - return result - - # NOTE: Do NOT update lastRawResponse here! - # lastRawResponse should only be updated after successful merge - # This ensures retry iterations use the correct base context - - # Handle use cases that return JSON directly (no section extraction needed) - # Check if use case supports direct return (all registered use cases do) - if useCase and not useCase.requiresExtraction: - # ===================================================================== - # ITERATION FLOW (Simplified) - # ===================================================================== - # Step 4: MERGE jsonBase + new response - # - FAILS: repeat prompt, increment fails cont (if >=3 return fallback) - # - SUCCEEDS: try parse - # - SUCCEEDS: FINISHED - # - FAILS: proceed to Step 5 - # Step 5: GET CONTEXTS (merge OK, parse failed) - # - getContexts() with repair - # - If no cut point: overlapContext = "" - # Step 6: DECIDE - # - jsonParsingSuccess=true AND overlapContext="": FINISHED - # - jsonParsingSuccess=true AND overlapContext!="": continue, fails=0 - # - ELSE: repeat prompt, increment fails count - # ===================================================================== - - # STEP 4: MERGE jsonBase + new response - # Use candidateJson to hold merged result until we confirm it's valid - candidateJson = None - - if jsonBase is None: - # First iteration - candidate is the current result - candidateJson = result - logger.debug(f"Iteration {iteration}: First response, candidateJson ({len(candidateJson)} chars)") - else: - # Merge jsonBase with new response - logger.info(f"Iteration {iteration}: Merging jsonBase ({len(jsonBase)} chars) with new response ({len(result)} chars)") - mergedJsonString, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap(jsonBase, result) - - if not hasOverlap: - # MERGE FAILED - repeat prompt with unchanged jsonBase - mergeFailCount += 1 - logger.warning( - f"Iteration {iteration}: Merge failed, no overlap found " - f"(fail {mergeFailCount}/{MAX_MERGE_FAILS})" - ) - - if mergeFailCount >= MAX_MERGE_FAILS: - # Max failures reached - return last valid completePart - logger.error( - f"Iteration {iteration}: Max merge failures ({MAX_MERGE_FAILS}) reached, " - "returning last valid completePart" - ) - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, False) - - if lastValidCompletePart: - try: - extracted = extractJsonString(lastValidCompletePart) - parsed, parseErr, _ = tryParseJson(extracted) - if parseErr is None and parsed: - normalized = self._normalizeJsonStructure(parsed, useCase) - return json.dumps(normalized, indent=2, ensure_ascii=False) - except Exception: - pass - return lastValidCompletePart - else: - # No valid fallback - return whatever we have - return jsonBase if jsonBase else "" - - # Not at max failures - retry with same prompt (jsonBase unchanged) - if iterationOperationId: - self.services.chat.progressLogUpdate( - iterationOperationId, 0.7, - f"Merge failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying" - ) - self.services.chat.progressLogFinish(iterationOperationId, True) - continue - - # MERGE SUCCEEDED - set candidate (don't update jsonBase yet!) - candidateJson = mergedJsonString - logger.debug(f"Iteration {iteration}: Merge succeeded, candidateJson ({len(candidateJson)} chars)") - - # Update lastRawResponse ONLY after we have a valid candidateJson - # (first iteration or successful merge - NOT on merge failure!) - # This ensures retry iterations use the correct base context - lastRawResponse = candidateJson - - # Try direct parse of candidate - try: - extracted = extractJsonString(candidateJson) - parsed, parseErr, _ = tryParseJson(extracted) - if parseErr is None and parsed: - # Direct parse succeeded - FINISHED - # Commit candidate to jsonBase - jsonBase = candidateJson - logger.info(f"Iteration {iteration}: Direct parse succeeded, JSON is complete") - normalized = self._normalizeJsonStructure(parsed, useCase) - result = json.dumps(normalized, indent=2, ensure_ascii=False) - - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, True) - - if not useCase.finalResultHandler: - raise ValueError( - f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback." - ) - return useCase.finalResultHandler( - result, normalized, extracted, debugPrefix, self.services - ) - except Exception as e: - logger.debug(f"Iteration {iteration}: Direct parse failed: {e}") - - # STEP 5: GET CONTEXTS (merge OK, parse failed = cut JSON) - # Use candidateJson for context extraction - contexts = getContexts(candidateJson) - overlapInfo = "(empty=complete)" if contexts.overlapContext == "" else f"({len(contexts.overlapContext)} chars)" - logger.debug( - f"Iteration {iteration}: getContexts() -> " - f"jsonParsingSuccess={contexts.jsonParsingSuccess}, " - f"overlapContext={overlapInfo}" - ) - - # STEP 6: DECIDE based on jsonParsingSuccess and overlapContext - if contexts.jsonParsingSuccess and contexts.overlapContext == "": - # JSON is complete (no cut point) - FINISHED - # Use completePart for final result (closed, repaired JSON) - # No more merging needed, so we don't need the cut version - jsonBase = contexts.completePart - logger.info(f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext='', JSON complete") - - # Store and parse completePart - lastValidCompletePart = contexts.completePart - - try: - extracted = extractJsonString(contexts.completePart) - parsed, parseErr, _ = tryParseJson(extracted) - if parseErr is None and parsed: - normalized = self._normalizeJsonStructure(parsed, useCase) - result = json.dumps(normalized, indent=2, ensure_ascii=False) - - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, True) - - if not useCase.finalResultHandler: - raise ValueError( - f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback." - ) - return useCase.finalResultHandler( - result, normalized, extracted, debugPrefix, self.services - ) - except Exception as e: - logger.warning(f"Iteration {iteration}: Failed to parse completePart: {e}") - - # Fallback: return completePart as-is - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, True) - return contexts.completePart - - elif contexts.jsonParsingSuccess and contexts.overlapContext != "": - # JSON parseable but has cut point - CONTINUE to next iteration - # CRITICAL: Use hierarchyContext (CUT json) as jsonBase for next merge! - # - hierarchyContext = the truncated JSON at cut point (needed for overlap matching) - # - completePart = closed JSON (for validation/fallback only) - # The next AI fragment's overlap must match the CUT point, not closed structures - jsonBase = contexts.hierarchyContext - logger.info( - f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext not empty, " - f"continuing iteration (jsonBase updated to hierarchyContext: {len(jsonBase)} chars)" - ) - - # Store valid completePart as fallback (different from jsonBase!) - lastValidCompletePart = contexts.completePart - - # Reset fail counter on successful progress - mergeFailCount = 0 - - # Update lastRawResponse for continuation prompt building - # Use the CUT version for prompt context as well - lastRawResponse = jsonBase - - if iterationOperationId: - self.services.chat.progressLogUpdate(iterationOperationId, 0.7, "JSON incomplete, requesting continuation") - self.services.chat.progressLogFinish(iterationOperationId, True) - continue - - else: - # JSON not parseable after repair - repeat prompt, increment fails - # Do NOT update jsonBase - keep previous valid state - mergeFailCount += 1 - logger.warning( - f"Iteration {iteration}: jsonParsingSuccess=false, " - f"repeat prompt (fail {mergeFailCount}/{MAX_MERGE_FAILS})" - ) - - if mergeFailCount >= MAX_MERGE_FAILS: - # Max failures reached - return last valid completePart - logger.error( - f"Iteration {iteration}: Max failures ({MAX_MERGE_FAILS}) reached, " - "returning last valid completePart" - ) - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, False) - - if lastValidCompletePart: - try: - extracted = extractJsonString(lastValidCompletePart) - parsed, parseErr, _ = tryParseJson(extracted) - if parseErr is None and parsed: - normalized = self._normalizeJsonStructure(parsed, useCase) - return json.dumps(normalized, indent=2, ensure_ascii=False) - except Exception: - pass - return lastValidCompletePart - else: - return jsonBase if jsonBase else "" - - # Not at max - retry with same prompt - # Do NOT update jsonBase or lastRawResponse - keep previous for retry - if iterationOperationId: - self.services.chat.progressLogUpdate( - iterationOperationId, 0.7, - f"Parse failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying" - ) - self.services.chat.progressLogFinish(iterationOperationId, True) - continue - - except Exception as e: - logger.error(f"Error in AI call iteration {iteration}: {str(e)}") - if iterationOperationId: - self.services.chat.progressLogFinish(iterationOperationId, False) - break - - if iteration >= maxIterations: - logger.warning(f"AI call stopped after maximum iterations ({maxIterations})") - - # This code path should never be reached because all registered use cases - # return early when JSON is complete. This would only execute for use cases that - # require section extraction, but no such use cases are currently registered. - logger.error(f"Unexpected code path: reached end of loop without return for use case '{useCaseId}'") - return result if result else "" - - def _isJsonStringIncomplete(self, jsonString: str) -> bool: - """ - Check if JSON string is incomplete (truncated) BEFORE closing/parsing. - - This is critical because if JSON is truncated, closing it makes it appear complete, - but we need to detect the truncation to continue iteration. - - Args: - jsonString: JSON string to check - - Returns: - True if JSON string appears incomplete/truncated, False otherwise - """ - if not jsonString or not jsonString.strip(): - return False - - # Normalize JSON string - normalized = stripCodeFences(normalizeJsonText(jsonString)).strip() - if not normalized: - return False - - # Find first '{' or '[' to start - startIdx = -1 - for i, char in enumerate(normalized): - if char in '{[': - startIdx = i - break - - if startIdx == -1: - return False - - jsonContent = normalized[startIdx:] - - # Check if structures are balanced (all opened structures are closed) - braceCount = 0 - bracketCount = 0 - inString = False - escapeNext = False - - for char in jsonContent: - if escapeNext: - escapeNext = False - continue - - if char == '\\': - escapeNext = True - continue - - if char == '"': - inString = not inString - continue - - if not inString: - if char == '{': - braceCount += 1 - elif char == '}': - braceCount -= 1 - elif char == '[': - bracketCount += 1 - elif char == ']': - bracketCount -= 1 - - # If structures are unbalanced, JSON is incomplete - if braceCount > 0 or bracketCount > 0: - return True - - # Check if JSON ends with incomplete value (e.g., unclosed string, incomplete number, trailing comma) - trimmed = jsonContent.rstrip() - if not trimmed: - return False - - # Check for trailing comma (might indicate incomplete) - if trimmed.endswith(','): - # Trailing comma might indicate incomplete, but could also be valid - # Check if there's a closing bracket/brace after the comma - return False # Trailing comma alone doesn't mean incomplete - - # Check if ends with incomplete string (odd number of quotes) - quoteCount = jsonContent.count('"') - if quoteCount % 2 == 1: - # Odd number of quotes - string is not closed - return True - - # Check if ends mid-value (e.g., ends with "417 instead of "4170. 41719"]) - # Look for patterns that suggest truncation: - # - Ends with incomplete number (e.g., "417) - # - Ends with incomplete array element (e.g., ["417) - # - Ends with incomplete object property (e.g., {"key": "val) - - # If JSON parses successfully without closing, it's complete - parsed, parseErr, _ = tryParseJson(jsonContent) - if parseErr is None: - # Parses successfully - it's complete - return False - - # If it doesn't parse, try closing it and see if that helps - closed = closeJsonStructures(jsonContent) - parsedClosed, parseErrClosed, _ = tryParseJson(closed) - - if parseErrClosed is None: - # Only parses after closing - it was incomplete - return True - - # Doesn't parse even after closing - might be malformed, but assume incomplete to be safe - return True - - def _normalizeJsonStructure(self, parsed: Any, useCase) -> Any: - """ - Normalize JSON structure to ensure consistent format before merging. - Handles different response formats and converts them to expected structure. - - Args: - parsed: Parsed JSON object (can be dict, list, or primitive) - useCase: LoopingUseCase instance with jsonNormalizer callback - - Returns: - Normalized JSON structure - """ - # Use callback to normalize JSON structure (REQUIRED - no fallback) - if not useCase or not useCase.jsonNormalizer: - raise ValueError( - f"Use case '{useCase.useCaseId if useCase else 'unknown'}' is missing required 'jsonNormalizer' callback. " - "All use cases must provide a jsonNormalizer function." - ) - return useCase.jsonNormalizer(parsed, useCase.useCaseId) - diff --git a/modules/services/serviceAi/subContentExtraction.py b/modules/services/serviceAi/subContentExtraction.py deleted file mode 100644 index a7250a3a..00000000 --- a/modules/services/serviceAi/subContentExtraction.py +++ /dev/null @@ -1,721 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Content Extraction Module - -Handles content extraction and preparation, including: -- Extracting content from documents based on intents -- Processing pre-extracted documents -- Vision AI for image text extraction -- AI processing of text content -""" -import json -import logging -import base64 -from typing import Dict, Any, List, Optional - -from modules.datamodels.datamodelChat import ChatDocument -from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent, ExtractionOptions, MergeStrategy -from modules.workflows.processing.shared.stateTools import checkWorkflowStopped - -logger = logging.getLogger(__name__) - - -class ContentExtractor: - """Handles content extraction and preparation.""" - - def __init__(self, services, aiService, intentAnalyzer): - """Initialize ContentExtractor with service center, AI service, and intent analyzer access.""" - self.services = services - self.aiService = aiService - self.intentAnalyzer = intentAnalyzer - - async def extractAndPrepareContent( - self, - documents: List[ChatDocument], - documentIntents: List[DocumentIntent], - parentOperationId: str, - getIntentForDocument: callable - ) -> List[ContentPart]: - """ - Phase 5B: Extrahiert Content basierend auf Intents und bereitet ContentParts mit Metadaten vor. - Gibt Liste von ContentParts im passenden Format zurück. - - WICHTIG: Ein Dokument kann mehrere ContentParts erzeugen, wenn mehrere Intents vorhanden sind. - Beispiel: Bild mit intents=["extract", "render"] erzeugt: - - ContentPart(contentFormat="object", ...) für Rendering - - ContentPart(contentFormat="extracted", ...) für Text-Analyse - - Args: - documents: Liste der zu verarbeitenden Dokumente - documentIntents: Liste von DocumentIntent-Objekten - parentOperationId: Parent Operation-ID für ChatLog-Hierarchie - getIntentForDocument: Callable to get intent for document ID - - Returns: - Liste von ContentParts mit vollständigen Metadaten - """ - # Erstelle Operation-ID für Extraktion - extractionOperationId = f"{parentOperationId}_content_extraction" - - # Starte ChatLog mit Parent-Referenz - self.services.chat.progressLogStart( - extractionOperationId, - "Content Extraction", - "Extraction", - f"Extracting from {len(documents)} documents", - parentOperationId=parentOperationId - ) - - try: - allContentParts = [] - - for document in documents: - checkWorkflowStopped(self.services) - # Check if document is already a ContentExtracted document (pre-extracted JSON) - logger.debug(f"Checking document {document.id} ({document.fileName}, mimeType={document.mimeType}) for pre-extracted content") - preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(document) - - if preExtracted: - logger.info(f"✅ Found pre-extracted document: {document.fileName} -> Original: {preExtracted['originalDocument']['fileName']}") - logger.info(f" Pre-extracted document ID: {document.id}, Original document ID: {preExtracted['originalDocument']['id']}") - logger.info(f" ContentParts count: {len(preExtracted['contentExtracted'].parts) if preExtracted['contentExtracted'].parts else 0}") - - # Verwende bereits extrahierte ContentParts direkt - contentExtracted = preExtracted["contentExtracted"] - - # WICHTIG: Intent muss für das JSON-Dokument gefunden werden, nicht für das Original - # (Intent-Analyse mappt bereits zurück zu JSON-Dokument-ID) - intent = getIntentForDocument(document.id, documentIntents) - logger.info(f" Intent lookup for document {document.id}: found={intent is not None}") - if intent: - logger.info(f" Intent: {intent.intents}, extractionPrompt: {intent.extractionPrompt[:100] if intent.extractionPrompt else None}...") - else: - logger.warning(f" ⚠️ No intent found for pre-extracted document {document.id}! Available intent documentIds: {[i.documentId for i in documentIntents]}") - - if contentExtracted.parts: - # CRITICAL: Process pre-extracted parts - analyze structure parts for nested content - processedParts = [] - for part in contentExtracted.parts: - # Überspringe leere Parts (Container ohne Daten) - if not part.data or (isinstance(part.data, str) and len(part.data.strip()) == 0): - if part.typeGroup == "container": - continue # Überspringe leere Container - - # CRITICAL: Check if structure part contains nested parts (e.g., JSON with documentData.parts) - if part.typeGroup == "structure" and part.mimeType == "application/json" and part.data: - nestedParts = self._extractNestedPartsFromStructure(part, document, preExtracted, intent) - if nestedParts: - # Replace structure part with extracted nested parts - processedParts.extend(nestedParts) - logger.info(f"✅ Extracted {len(nestedParts)} nested parts from structure part {part.id}") - continue # Skip original structure part - - # Keep original part if no nested parts found - processedParts.append(part) - - # Use processed parts (with nested parts extracted) - for part in processedParts: - if not part.metadata: - part.metadata = {} - - # Ensure metadata is complete - if "documentId" not in part.metadata: - part.metadata["documentId"] = document.id - - # WICHTIG: Prüfe Intent für dieses Part - partIntent = intent.intents if intent else ["extract"] - - # Debug-Logging für Intent-Verarbeitung - logger.debug(f"Processing part {part.id}: typeGroup={part.typeGroup}, intents={partIntent}, hasData={bool(part.data)}, dataLength={len(str(part.data)) if part.data else 0}") - - # WICHTIG: Ein Part kann mehrere Intents haben - erstelle für jeden Intent einen ContentPart - # Generische Intent-Verarbeitung für ALLE Content-Typen - hasReferenceIntent = "reference" in partIntent - hasRenderIntent = "render" in partIntent - hasExtractIntent = "extract" in partIntent - hasPartData = bool(part.data) and (not isinstance(part.data, str) or len(part.data.strip()) > 0) - - logger.debug(f"Part {part.id}: reference={hasReferenceIntent}, render={hasRenderIntent}, extract={hasExtractIntent}, hasData={hasPartData}") - - # SAFETY: For images with any intent, always ensure render is included - # This ensures the image object part is always available for later rendering - isImage = part.typeGroup == "image" or (part.mimeType and part.mimeType.startswith("image/")) - if isImage and hasPartData and not hasRenderIntent: - logger.info(f"🖼️ Auto-adding render intent for image {part.id} (original intents: {partIntent})") - hasRenderIntent = True - - # Track ob der originale Part bereits hinzugefügt wurde - originalPartAdded = False - - # 1. Reference Intent: Erstelle Reference ContentPart - if hasReferenceIntent: - referencePart = ContentPart( - id=f"ref_{document.id}_{part.id}", - label=f"Reference: {part.label or 'Content'}", - typeGroup="reference", - mimeType=part.mimeType or "application/octet-stream", - data="", # Leer - nur Referenz - metadata={ - "contentFormat": "reference", - "documentId": document.id, - "documentReference": f"docItem:{document.id}:{preExtracted['originalDocument']['fileName']}", - "intent": "reference", - "usageHint": f"Reference: {preExtracted['originalDocument']['fileName']}", - "originalFileName": preExtracted["originalDocument"]["fileName"] - } - ) - allContentParts.append(referencePart) - logger.debug(f"✅ Created reference ContentPart for {part.id}") - - # 2. Render Intent: Erstelle Object ContentPart (für Binary/Image Rendering) - if hasRenderIntent and hasPartData: - # Prüfe ob es ein Binary/Image ist (kann gerendert werden) - isRenderable = ( - part.typeGroup == "image" or - part.typeGroup == "binary" or - (part.mimeType and ( - part.mimeType.startswith("image/") or - part.mimeType.startswith("video/") or - part.mimeType.startswith("audio/") or - self._isBinary(part.mimeType) - )) - ) - - if isRenderable: - objectPart = ContentPart( - id=f"obj_{document.id}_{part.id}", - label=f"Object: {part.label or 'Content'}", - typeGroup=part.typeGroup, - mimeType=part.mimeType or "application/octet-stream", - data=part.data, # Base64/Binary data ist bereits vorhanden - metadata={ - "contentFormat": "object", - "documentId": document.id, - "intent": "render", - "usageHint": f"Render as visual element: {preExtracted['originalDocument']['fileName']}", - "originalFileName": preExtracted["originalDocument"]["fileName"], - "relatedExtractedPartId": f"extracted_{document.id}_{part.id}" if hasExtractIntent else None - } - ) - allContentParts.append(objectPart) - logger.debug(f"✅ Created object ContentPart for {part.id} (render intent)") - else: - logger.warning(f"⚠️ Part {part.id} has render intent but is not renderable (typeGroup={part.typeGroup}, mimeType={part.mimeType})") - elif hasRenderIntent and not hasPartData: - logger.warning(f"⚠️ Part {part.id} has render intent but no data, skipping render part") - - # 3. Extract Intent: Erstelle Extracted ContentPart (NO AI processing here - happens during section generation) - if hasExtractIntent: - # For images: Keep as image part with extract intent - Vision AI extraction happens during section generation - if part.typeGroup == "image" and hasPartData: - logger.info(f"📷 Image {part.id} with extract intent - will be processed with Vision AI during section generation") - # Keep image part as-is, mark with extract intent - part.metadata.update({ - "contentFormat": "extracted", # Marked for extraction, but not yet extracted - "intent": "extract", - "originalFileName": preExtracted["originalDocument"]["fileName"], - "relatedObjectPartId": f"obj_{document.id}_{part.id}" if hasRenderIntent else None, - "extractionPrompt": intent.extractionPrompt if intent and intent.extractionPrompt else "Extract all text content from this image.", - "needsVisionExtraction": True # Flag to indicate Vision AI extraction needed - }) - allContentParts.append(part) - originalPartAdded = True - else: - # For text/table content: Use directly as extracted (no AI processing here) - # AI processing with extractionPrompt happens during section generation - if not originalPartAdded: - part.metadata.update({ - "contentFormat": "extracted", - "intent": "extract", - "fromExtractContent": True, - "skipExtraction": True, # Already extracted (raw extraction) - "originalFileName": preExtracted["originalDocument"]["fileName"], - "relatedObjectPartId": f"obj_{document.id}_{part.id}" if hasRenderIntent else None, - "extractionPrompt": intent.extractionPrompt if intent and intent.extractionPrompt else None - }) - # Stelle sicher dass contentFormat gesetzt ist - if "contentFormat" not in part.metadata: - part.metadata["contentFormat"] = "extracted" - allContentParts.append(part) - originalPartAdded = True - logger.debug(f"✅ Using pre-extracted ContentPart {part.id} as extracted (no AI processing needed)") - - # 4. Fallback: Wenn kein Intent vorhanden oder Part wurde noch nicht hinzugefügt - # (sollte normalerweise nicht vorkommen, da default "extract" ist) - if not hasReferenceIntent and not hasRenderIntent and not hasExtractIntent and not originalPartAdded: - logger.warning(f"⚠️ Part {part.id} has no recognized intents, adding as extracted by default") - part.metadata.update({ - "contentFormat": "extracted", - "intent": "extract", - "fromExtractContent": True, - "skipExtraction": True, - "originalFileName": preExtracted["originalDocument"]["fileName"] - }) - allContentParts.append(part) - originalPartAdded = True - - logger.info(f"✅ Using {len([p for p in contentExtracted.parts if p.data and len(str(p.data)) > 0])} pre-extracted ContentParts from ContentExtracted document {document.fileName}") - logger.info(f" Original document: {preExtracted['originalDocument']['fileName']}") - continue # Skip normal extraction for this document - - # Check if it's standardized JSON format (has "documents" or "sections") - if document.mimeType == "application/json": - try: - docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) - if docBytes: - docData = docBytes.decode('utf-8') - jsonData = json.loads(docData) - - if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): - logger.info(f"Document is already in standardized JSON format, using as reference") - # Create reference ContentPart for structured JSON - contentPart = ContentPart( - id=f"ref_{document.id}", - label=f"Reference: {document.fileName}", - typeGroup="structure", - mimeType="application/json", - data=docData, - metadata={ - "contentFormat": "reference", - "documentId": document.id, - "documentReference": f"docItem:{document.id}:{document.fileName}", - "skipExtraction": True, - "intent": "reference" - } - ) - allContentParts.append(contentPart) - logger.info(f"✅ Using JSON document directly without extraction") - continue # Skip normal extraction for this document - except Exception as e: - logger.warning(f"Could not parse JSON document {document.fileName}, will extract normally: {str(e)}") - # Continue with normal extraction - - # Normal extraction path - intent = getIntentForDocument(document.id, documentIntents) - - if not intent: - # Try to find intent by similar UUID (fix for AI UUID hallucination) - correctedIntent = self._findIntentBySimilarId(document.id, documentIntents) - if correctedIntent: - logger.warning(f"Found intent for document {document.id} using UUID correction (original: {correctedIntent.documentId})") - # Create new intent with correct document ID - intent = DocumentIntent( - documentId=document.id, - intents=correctedIntent.intents, - extractionPrompt=correctedIntent.extractionPrompt, - reasoning=f"Intent matched by UUID similarity (original: {correctedIntent.documentId})" - ) - else: - # Default: extract für alle Dokumente ohne Intent - logger.warning(f"No intent found for document {document.id}, using default 'extract'") - intent = DocumentIntent( - documentId=document.id, - intents=["extract"], - extractionPrompt="Extract all content from the document", - reasoning="Default intent: no specific intent found" - ) - - # WICHTIG: Prüfe alle Intents - ein Dokument kann mehrere ContentParts erzeugen - - if "reference" in intent.intents: - # Erstelle Reference ContentPart - contentPart = ContentPart( - id=f"ref_{document.id}", - label=f"Reference: {document.fileName}", - typeGroup="reference", - mimeType=document.mimeType, - data="", - metadata={ - "contentFormat": "reference", - "documentId": document.id, - "documentReference": f"docItem:{document.id}:{document.fileName}", - "intent": "reference", - "usageHint": f"Reference document: {document.fileName}" - } - ) - allContentParts.append(contentPart) - - # WICHTIG: "render" und "extract" können beide vorhanden sein! - # In diesem Fall erzeugen wir BEIDE ContentParts - - # SAFETY: For images with any intent, always create object part for later rendering - isImageDocument = document.mimeType and document.mimeType.startswith("image/") - shouldAutoRender = isImageDocument and "render" not in intent.intents and ("extract" in intent.intents or "reference" in intent.intents) - if shouldAutoRender: - logger.info(f"🖼️ Auto-adding render for image document {document.id} (original intents: {intent.intents})") - - if "render" in intent.intents or shouldAutoRender: - # Für Images/Binary: extrahiere als Object - if document.mimeType.startswith("image/") or self._isBinary(document.mimeType): - try: - # Lade Binary-Daten (getFileData ist nicht async - keine await nötig) - binaryData = self.services.interfaceDbComponent.getFileData(document.fileId) - if not binaryData: - logger.warning(f"No binary data found for document {document.id}") - continue - base64Data = base64.b64encode(binaryData).decode('utf-8') - - contentPart = ContentPart( - id=f"obj_{document.id}", - label=f"Object: {document.fileName}", - typeGroup="image" if document.mimeType.startswith("image/") else "binary", - mimeType=document.mimeType, - data=base64Data, - metadata={ - "contentFormat": "object", - "documentId": document.id, - "intent": "render", - "usageHint": f"Render as visual element: {document.fileName}", - "originalFileName": document.fileName, - # Verknüpfung zu extracted Part (falls vorhanden) - "relatedExtractedPartId": f"ext_{document.id}" if "extract" in intent.intents else None - } - ) - allContentParts.append(contentPart) - except Exception as e: - logger.error(f"Failed to load binary data for document {document.id}: {str(e)}") - - if "extract" in intent.intents: - # Extrahiere Content mit Extraction Service - extractionPrompt = intent.extractionPrompt or "Extract all content from the document" - - # Debug-Log (harmonisiert) - self.services.utils.writeDebugFile( - extractionPrompt, - f"content_extraction_prompt_{document.id}" - ) - - # Führe Extraktion aus - - extractionOptions = ExtractionOptions( - prompt=extractionPrompt, - mergeStrategy=MergeStrategy() - ) - - # extractContent ist nicht async - keine await nötig - checkWorkflowStopped(self.services) - extractedResults = self.services.extraction.extractContent( - [document], - extractionOptions, - operationId=extractionOperationId, - parentOperationId=extractionOperationId - ) - - # Konvertiere extrahierte Ergebnisse zu ContentParts mit Metadaten - # Check if object part exists (either explicit render or auto-render for images) - hasObjectPart = "render" in intent.intents or shouldAutoRender - - for extracted in extractedResults: - for part in extracted.parts: - # Markiere als extracted Format - part.metadata.update({ - "contentFormat": "extracted", - "documentId": document.id, - "extractionPrompt": extractionPrompt, - "intent": "extract", - "usageHint": f"Use extracted content from {document.fileName}", - # Verknüpfung zu object Part (falls vorhanden - including auto-render for images) - "relatedObjectPartId": f"obj_{document.id}" if hasObjectPart else None - }) - - # For images: Mark that Vision AI extraction is needed during section generation - if part.typeGroup == "image": - part.metadata["needsVisionExtraction"] = True - logger.info(f"📷 Image part {part.id} marked for Vision AI extraction during section generation") - - # Stelle sicher, dass ID eindeutig ist (falls object Part existiert) - if hasObjectPart: - part.id = f"ext_{document.id}_{part.id}" - allContentParts.append(part) - - # Debug-Log (harmonisiert) - self.services.utils.writeDebugFile( - json.dumps([part.dict() for part in allContentParts], indent=2, default=str), - "content_extraction_result" - ) - - # State 2 Validation: Validate and auto-fix ContentParts - validatedParts = [] - for part in allContentParts: - # Validation 2.1: Skip ContentParts without documentId - if not part.metadata.get("documentId"): - logger.warning(f"Skipping ContentPart {part.id} - missing documentId in metadata") - continue - - # Validation 2.2: Skip ContentParts with invalid contentFormat - contentFormat = part.metadata.get("contentFormat") - if contentFormat not in ["extracted", "object", "reference"]: - logger.warning( - f"Skipping ContentPart {part.id} - invalid contentFormat: {contentFormat}" - ) - continue - - validatedParts.append(part) - - # ChatLog abschließen - self.services.chat.progressLogFinish(extractionOperationId, True) - - return validatedParts - - except Exception as e: - self.services.chat.progressLogFinish(extractionOperationId, False) - logger.error(f"Error in extractAndPrepareContent: {str(e)}") - raise - - async def extractTextFromImage(self, imagePart: ContentPart, extractionPrompt: str) -> Optional[str]: - """ - Extrahiere Text aus einem Image-Part mit Vision AI. - - Args: - imagePart: ContentPart mit typeGroup="image" - extractionPrompt: Prompt für die Text-Extraktion - - Returns: - Extrahierter Text oder None bei Fehler - """ - try: - from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum - - # Final extraction prompt - finalPrompt = extractionPrompt or "Extract all text content from this image. Return only the extracted text, no additional formatting." - - # Debug-Log (harmonisiert) - self.services.utils.writeDebugFile( - finalPrompt, - f"content_extraction_prompt_image_{imagePart.id}" - ) - - # Erstelle AI-Call-Request mit Image-Part - request = AiCallRequest( - prompt=finalPrompt, - context="", - options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), - contentParts=[imagePart] - ) - - # Verwende AI-Service für Vision AI-Verarbeitung - checkWorkflowStopped(self.services) - response = await self.aiService.callAi(request) - - # Debug-Log für Response (harmonisiert) - if response and response.content: - self.services.utils.writeDebugFile( - response.content, - f"content_extraction_response_image_{imagePart.id}" - ) - - if response and response.content: - return response.content.strip() - - # Kein Content zurückgegeben - return error message für Debugging - errorMsg = f"Vision AI extraction failed: No content returned for image {imagePart.id}" - logger.warning(errorMsg) - return f"[ERROR: {errorMsg}]" - except Exception as e: - errorMsg = f"Vision AI extraction failed for image {imagePart.id}: {str(e)}" - logger.error(errorMsg) - import traceback - logger.debug(f"Traceback: {traceback.format_exc()}") - # Return error message statt None für Debugging - return f"[ERROR: {errorMsg}]" - - async def processTextContentWithAi(self, textPart: ContentPart, extractionPrompt: str) -> Optional[str]: - """ - Verarbeite Text-Content mit AI basierend auf extractionPrompt. - - WICHTIG: Pre-extracted ContentParts von context.extractContent enthalten RAW extrahierten Text - (z.B. aus PDF-Text-Layer). Wenn "extract" Intent vorhanden ist, muss dieser Text mit AI - verarbeitet werden (Transformation, Strukturierung, etc.) basierend auf extractionPrompt. - - Args: - textPart: ContentPart mit typeGroup="text" (oder anderer Text-basierter Typ) - extractionPrompt: Prompt für die AI-Verarbeitung des Textes - - Returns: - AI-verarbeiteter Text oder None bei Fehler - """ - try: - from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum - - # Final extraction prompt - finalPrompt = extractionPrompt or "Process and extract the key information from the following text content." - - # Debug-Log (harmonisiert) - log prompt with text preview - textPreview = textPart.data[:500] + "..." if textPart.data and len(textPart.data) > 500 else (textPart.data or "") - promptWithContext = f"{finalPrompt}\n\n--- Text Content (preview) ---\n{textPreview}" - self.services.utils.writeDebugFile( - promptWithContext, - f"content_extraction_prompt_text_{textPart.id}" - ) - - # Erstelle Text-ContentPart für AI-Verarbeitung - # Verwende den vorhandenen Text als Input - textContentPart = ContentPart( - id=textPart.id, - label=textPart.label, - typeGroup="text", - mimeType="text/plain", - data=textPart.data if textPart.data else "", - metadata=textPart.metadata.copy() if textPart.metadata else {} - ) - - # Erstelle AI-Call-Request mit Text-Part - request = AiCallRequest( - prompt=finalPrompt, - context="", - options=AiCallOptions(operationType=OperationTypeEnum.DATA_EXTRACT), - contentParts=[textContentPart] - ) - - # Verwende AI-Service für Text-Verarbeitung - checkWorkflowStopped(self.services) - response = await self.aiService.callAi(request) - - # Debug-Log für Response (harmonisiert) - if response and response.content: - self.services.utils.writeDebugFile( - response.content, - f"content_extraction_response_text_{textPart.id}" - ) - - if response and response.content: - return response.content.strip() - - # Kein Content zurückgegeben - return error message für Debugging - errorMsg = f"AI text processing failed: No content returned for text part {textPart.id}" - logger.warning(errorMsg) - return f"[ERROR: {errorMsg}]" - except Exception as e: - errorMsg = f"AI text processing failed for text part {textPart.id}: {str(e)}" - logger.error(errorMsg) - import traceback - logger.debug(f"Traceback: {traceback.format_exc()}") - # Return error message statt None für Debugging - return f"[ERROR: {errorMsg}]" - - def _isBinary(self, mimeType: str) -> bool: - """Prüfe ob MIME-Type binary ist.""" - binaryTypes = [ - "application/octet-stream", - "application/pdf", - "application/zip", - "application/x-zip-compressed" - ] - return mimeType in binaryTypes or mimeType.startswith("image/") or mimeType.startswith("video/") or mimeType.startswith("audio/") - - def _extractNestedPartsFromStructure( - self, - structurePart: ContentPart, - document: ChatDocument, - preExtracted: Dict[str, Any], - intent: Optional[Any] - ) -> List[ContentPart]: - """ - Extract nested parts from a structure ContentPart (e.g., JSON with documentData.parts). - - This is a generic function that analyzes pre-processed ContentParts and extracts - any nested parts that are embedded in structure data (typically JSON). - - Works with standard ContentExtracted format: documentData.parts array. - Each nested part is extracted as a separate ContentPart with proper metadata. - - Args: - structurePart: ContentPart with typeGroup="structure" containing nested parts - document: The document this part belongs to - preExtracted: Pre-extracted document metadata - intent: Document intent for nested parts - - Returns: - List of extracted ContentParts, empty if no nested parts found - """ - nestedParts = [] - - try: - # Parse JSON structure - jsonData = json.loads(structurePart.data) - - # Check for standard ContentExtracted format: documentData.parts - if isinstance(jsonData, dict): - documentData = jsonData.get("documentData") - if isinstance(documentData, dict): - parts = documentData.get("parts", []) - if isinstance(parts, list) and len(parts) > 0: - # Extract each nested part - for nestedPartData in parts: - if not isinstance(nestedPartData, dict): - continue - - nestedPartId = nestedPartData.get("id") or f"nested_{len(nestedParts)}" - nestedTypeGroup = nestedPartData.get("typeGroup", "text") - nestedMimeType = nestedPartData.get("mimeType", "text/plain") - nestedLabel = nestedPartData.get("label", structurePart.label) - nestedData = nestedPartData.get("data", "") - nestedMetadata = nestedPartData.get("metadata", {}) - - # Create ContentPart for nested part - nestedPart = ContentPart( - id=f"{structurePart.id}_{nestedPartId}", - parentId=structurePart.id, - label=nestedLabel, - typeGroup=nestedTypeGroup, - mimeType=nestedMimeType, - data=nestedData, - metadata={ - **nestedMetadata, - "documentId": document.id, - "fromNestedStructure": True, - "parentStructurePartId": structurePart.id, - "originalFileName": preExtracted["originalDocument"]["fileName"] - } - ) - - nestedParts.append(nestedPart) - logger.debug(f"✅ Extracted nested part: {nestedPart.id} (typeGroup={nestedTypeGroup}, mimeType={nestedMimeType})") - - # If no nested parts found, return empty list (original part will be kept) - if not nestedParts: - logger.debug(f"No nested parts found in structure part {structurePart.id}") - - except json.JSONDecodeError as e: - logger.warning(f"Could not parse structure part {structurePart.id} as JSON: {str(e)}") - except Exception as e: - logger.error(f"Error extracting nested parts from structure part {structurePart.id}: {str(e)}") - - return nestedParts - - def _findIntentBySimilarId(self, documentId: str, documentIntents: List[DocumentIntent]) -> Optional[DocumentIntent]: - """ - Versucht ein Intent zu finden, dessen UUID ähnlich zur angegebenen Dokument-ID ist. - Dies hilft bei AI UUID-Halluzinationen (z.B. 4451 -> 4551). - - Args: - documentId: Die Dokument-ID für die ein Intent gesucht wird - documentIntents: Liste aller verfügbaren DocumentIntents - - Returns: - DocumentIntent mit ähnlicher UUID falls gefunden, sonst None - """ - if not documentId or len(documentId) != 36: # UUID Format: 8-4-4-4-12 - return None - - # Prüfe ob es eine UUID ist (Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) - if documentId.count('-') != 4: - return None - - for intent in documentIntents: - intentId = intent.documentId - if len(intentId) != 36: - continue - - # Zähle unterschiedliche Zeichen - differences = sum(c1 != c2 for c1, c2 in zip(documentId, intentId)) - - # Wenn nur 1-2 Zeichen unterschiedlich sind, ist es wahrscheinlich ein Typo - if differences <= 2: - # Prüfe ob die Struktur ähnlich ist (gleiche Positionen der Bindestriche) - if documentId.count('-') == intentId.count('-'): - return intent - - return None - diff --git a/modules/services/serviceAi/subDocumentIntents.py b/modules/services/serviceAi/subDocumentIntents.py deleted file mode 100644 index 274a8a5a..00000000 --- a/modules/services/serviceAi/subDocumentIntents.py +++ /dev/null @@ -1,369 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Document Intent Analysis Module - -Handles analysis of document intents, including: -- Clarifying which documents need extraction vs reference -- Resolving pre-extracted documents -- Building intent analysis prompts -""" -import json -import logging -from typing import Dict, Any, List, Optional - -from modules.datamodels.datamodelChat import ChatDocument -from modules.datamodels.datamodelExtraction import DocumentIntent -from modules.workflows.processing.shared.stateTools import checkWorkflowStopped - -logger = logging.getLogger(__name__) - - -class DocumentIntentAnalyzer: - """Handles document intent analysis and resolution.""" - - def __init__(self, services, aiService): - """Initialize DocumentIntentAnalyzer with service center and AI service access.""" - self.services = services - self.aiService = aiService - - async def clarifyDocumentIntents( - self, - documents: List[ChatDocument], - userPrompt: str, - actionParameters: Dict[str, Any], - parentOperationId: str - ) -> List[DocumentIntent]: - """ - Phase 5A: Analysiert, welche Dokumente Extraktion vs Referenz benötigen. - Gibt DocumentIntent für jedes Dokument zurück. - - Args: - documents: Liste der zu verarbeitenden Dokumente - userPrompt: User-Anfrage - actionParameters: Action-spezifische Parameter (z.B. resultType, outputFormat) - parentOperationId: Parent Operation-ID für ChatLog-Hierarchie - - Returns: - Liste von DocumentIntent-Objekten - """ - # Erstelle Operation-ID für Intent-Analyse - intentOperationId = f"{parentOperationId}_intent_analysis" - - # Starte ChatLog mit Parent-Referenz - self.services.chat.progressLogStart( - intentOperationId, - "Document Intent Analysis", - "Intent Analysis", - f"Analyzing {len(documents)} documents", - parentOperationId=parentOperationId - ) - - try: - # Mappe pre-extracted JSONs zu ursprünglichen Dokument-IDs für Intent-Analyse - documentMapping = {} # Maps original doc ID -> JSON doc ID - resolvedDocuments = [] - - for doc in documents: - preExtracted = self.resolvePreExtractedDocument(doc) - if preExtracted: - originalDocId = preExtracted["originalDocument"]["id"] - documentMapping[originalDocId] = doc.id - # Erstelle temporäres ChatDocument für ursprüngliches Dokument - originalDoc = ChatDocument( - id=originalDocId, - fileName=preExtracted["originalDocument"]["fileName"], - mimeType=preExtracted["originalDocument"]["mimeType"], - fileSize=preExtracted["originalDocument"].get("fileSize", doc.fileSize), - fileId=doc.fileId, # Behalte fileId vom JSON - messageId=doc.messageId if hasattr(doc, 'messageId') else None # Behalte messageId falls vorhanden - ) - resolvedDocuments.append(originalDoc) - else: - resolvedDocuments.append(doc) - - # Baue Intent-Analyse-Prompt mit ursprünglichen Dokumenten - intentPrompt = self._buildIntentAnalysisPrompt(userPrompt, resolvedDocuments, actionParameters) - - # AI-Call (verwende callAiPlanning für einfache JSON-Responses) - # Debug-Logs werden bereits von callAiPlanning geschrieben - checkWorkflowStopped(self.services) - aiResponse = await self.aiService.callAiPlanning( - prompt=intentPrompt, - debugType="document_intent_analysis" - ) - - # Parse Result und mappe zurück zu JSON-Dokument-IDs falls nötig - intentsData = json.loads(self.services.utils.jsonExtractString(aiResponse)) - documentIntents = [] - for intent in intentsData.get("intents", []): - docId = intent.get("documentId") - # Wenn Intent für ursprüngliches Dokument, mappe zurück zu JSON-Dokument-ID - if docId in documentMapping: - intent["documentId"] = documentMapping[docId] - documentIntents.append(DocumentIntent(**intent)) - - # Debug-Log (harmonisiert) - self.services.utils.writeDebugFile( - json.dumps([intent.dict() for intent in documentIntents], indent=2), - "document_intent_analysis_result" - ) - - # State 1 Validation: Validate and auto-fix document intents - documentIds = {d.id for d in documents} - validatedIntents = [] - - for intent in documentIntents: - # Validation 1.2: Skip intents for unknown documents - if intent.documentId not in documentIds: - # Try to find similar UUID (fix AI hallucination/typo) - correctedDocId = self._findSimilarDocumentId(intent.documentId, documentIds) - if correctedDocId: - logger.warning(f"Corrected UUID typo in AI response: {intent.documentId} -> {correctedDocId}") - intent.documentId = correctedDocId - else: - logger.warning(f"Skipping intent for unknown document: {intent.documentId}") - continue - validatedIntents.append(intent) - - # Validation 1.1: Documents without intents are OK (not needed) - # Intents for non-existing documents are already filtered above - documentIntents = validatedIntents - - # ChatLog abschließen - self.services.chat.progressLogFinish(intentOperationId, True) - - return documentIntents - - except Exception as e: - self.services.chat.progressLogFinish(intentOperationId, False) - logger.error(f"Error in clarifyDocumentIntents: {str(e)}") - raise - - def resolvePreExtractedDocument(self, document: ChatDocument) -> Optional[Dict[str, Any]]: - """ - Prüft ob ein JSON-Dokument bereits extrahierte ContentParts enthält. - Gibt Dict zurück mit: - - originalDocument: ChatDocument-Info des ursprünglichen Dokuments - - contentExtracted: ContentExtracted-Objekt mit Parts - - parts: Liste der ContentParts - - Returns None wenn kein pre-extracted Format erkannt wird. - """ - if document.mimeType != "application/json": - logger.debug(f"Document {document.id} is not JSON (mimeType={document.mimeType}), skipping pre-extracted check") - return None - - try: - docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) - if not docBytes: - return None - - docData = docBytes.decode('utf-8') - jsonData = json.loads(docData) - - if not isinstance(jsonData, dict): - return None - - # Check for ContentExtracted format - # Nur Format 1 (ActionDocument-Format mit validationMetadata) wird unterstützt - documentData = None - - validationMetadata = jsonData.get("validationMetadata", {}) - actionType = validationMetadata.get("actionType") - logger.debug(f"JSON document {document.id}: validationMetadata.actionType={actionType}, keys={list(jsonData.keys())}") - - if actionType == "context.extractContent": - # Format: {"validationMetadata": {"actionType": "context.extractContent"}, "documentData": {...}} - documentData = jsonData.get("documentData") - logger.debug(f"Found ContentExtracted via validationMetadata for {document.fileName}, documentData keys: {list(documentData.keys()) if documentData else None}") - else: - logger.debug(f"JSON document {document.id} does not have actionType='context.extractContent' (got: {actionType})") - - if documentData: - - try: - # Stelle sicher, dass "id" vorhanden ist - if "id" not in documentData: - documentData["id"] = document.id - - contentExtracted = ContentExtracted(**documentData) - - if contentExtracted.parts: - # Extrahiere ursprüngliche Dokument-Info aus den Parts - originalDocId = None - originalFileName = None - originalMimeType = None - - for part in contentExtracted.parts: - if part.metadata: - # Versuche ursprüngliche Dokument-Info zu finden - if not originalDocId and part.metadata.get("documentId"): - originalDocId = part.metadata.get("documentId") - if not originalFileName and part.metadata.get("originalFileName"): - originalFileName = part.metadata.get("originalFileName") - if not originalMimeType and part.metadata.get("documentMimeType"): - originalMimeType = part.metadata.get("documentMimeType") - - # Falls nicht gefunden, versuche aus documentName zu extrahieren - if not originalFileName: - # Versuche aus documentName zu extrahieren (z.B. "B2025-02c_28_extracted_...json" -> "B2025-02c_28.pdf") - if document.fileName and "_extracted_" in document.fileName: - originalFileName = document.fileName.split("_extracted_")[0] + ".pdf" - - return { - "originalDocument": { - "id": originalDocId or document.id, - "fileName": originalFileName or document.fileName, - "mimeType": originalMimeType or "application/pdf", - "fileSize": document.fileSize - }, - "contentExtracted": contentExtracted, - "parts": contentExtracted.parts - } - except Exception as parseError: - logger.warning(f"Could not parse ContentExtracted format from {document.fileName}: {str(parseError)}") - logger.debug(f"JSON keys: {list(jsonData.keys())}, has parts: {'parts' in jsonData}") - import traceback - logger.debug(f"Parse error traceback: {traceback.format_exc()}") - return None - else: - logger.debug(f"JSON document {document.id} has no documentData (actionType={actionType})") - - return None - except Exception as e: - logger.debug(f"Error resolving pre-extracted document {document.fileName}: {str(e)}") - return None - - def _buildIntentAnalysisPrompt( - self, - userPrompt: str, - documents: List[ChatDocument], - actionParameters: Dict[str, Any] - ) -> str: - """Baue Prompt für Intent-Analyse.""" - # Baue Dokument-Liste - zeige ursprüngliche Dokumente für pre-extracted JSONs - docListText = "" - for i, doc in enumerate(documents, 1): - # Prüfe ob es ein pre-extracted JSON ist - preExtracted = self.resolvePreExtractedDocument(doc) - - if preExtracted: - # Zeige ursprüngliches Dokument statt JSON - originalDoc = preExtracted["originalDocument"] - partsInfo = f" (contains {len(preExtracted['parts'])} pre-extracted parts: {', '.join([p.typeGroup for p in preExtracted['parts'] if p.data and len(str(p.data)) > 0])})" - docListText += f"\n{i}. Document ID: {originalDoc['id']}\n" - docListText += f" File Name: {originalDoc['fileName']}{partsInfo}\n" - docListText += f" MIME Type: {originalDoc['mimeType']}\n" - docListText += f" File Size: {originalDoc.get('fileSize', doc.fileSize)} bytes\n" - else: - # Normales Dokument - docListText += f"\n{i}. Document ID: {doc.id}\n" - docListText += f" File Name: {doc.fileName}\n" - docListText += f" MIME Type: {doc.mimeType}\n" - docListText += f" File Size: {doc.fileSize} bytes\n" - - outputFormat = actionParameters.get("outputFormat", "txt") - - # FENCE user input to prevent prompt injection - fencedUserPrompt = f"""```user_request -{userPrompt} -```""" - - prompt = f"""USER REQUEST: -{fencedUserPrompt} - -DOCUMENTS TO ANALYZE: -{docListText} - -TASK: For each document, determine its intents (can be multiple): -- "extract": Content extraction needed (text, structure, OCR, etc.) -- "render": Image/binary should be rendered as-is (visual element) -- "reference": Document reference/attachment (no extraction, just reference) - -TASK: For each document, determine: -1. Intents (can be multiple): "extract", "render", "reference" -Note: Output format and language are NOT determined here - they will be - determined during structure generation (Phase 3) in the chapter structure JSON - -OUTPUT FORMAT: {outputFormat} (global fallback - for reference only) - -RETURN JSON: -{{ - "intents": [ - {{ - "documentId": "doc_1", - "intents": ["extract"], - "extractionPrompt": "Extract all text content, preserving structure", - "reasoning": "User needs text content for document generation" - }}, - {{ - "documentId": "doc_2", - "intents": ["extract", "render"], - "extractionPrompt": "Extract text content from image using vision AI", - "reasoning": "Image contains text that needs extraction, but also should be rendered visually" - }}, - {{ - "documentId": "doc_3", - "intents": ["reference"], - "extractionPrompt": null, - "reasoning": "Document is only used as reference, no extraction needed" - }} - ] -}} - -CRITICAL RULES: -1. For images (mimeType starts with "image/"): - - If user wants to "include" or "show" images → add "render" - - If user wants to "analyze", "read text", or "extract text" from images → add "extract" - - Can have BOTH "extract" and "render" if image needs both text extraction and visual rendering - -2. For text documents: - - If user mentions "template" or "structure" → "reference" or "extract" based on context - - If user mentions "reference" or "context" → "reference" - - Default → "extract" - -3. Consider output format: - - For formats like PDF, DOCX, PPTX: images usually need "render" - - For formats like CSV, JSON: usually "extract" only - - For HTML: can have both "extract" and "render" - -Return ONLY valid JSON following the structure above. -""" - return prompt - - def _findSimilarDocumentId(self, incorrectId: str, validIds: set) -> Optional[str]: - """ - Versucht eine ähnliche Dokument-ID zu finden, falls die AI die UUID geändert hat. - Prüft auf UUID-Typo (z.B. 4451 -> 4551). - - Args: - incorrectId: Die falsche UUID aus der AI-Response - validIds: Set von gültigen Dokument-IDs - - Returns: - Korrigierte UUID falls gefunden, sonst None - """ - if not incorrectId or len(incorrectId) != 36: # UUID Format: 8-4-4-4-12 - return None - - # Prüfe ob es eine UUID ist (Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) - if incorrectId.count('-') != 4: - return None - - # Versuche Levenshtein-ähnliche Suche: Prüfe ob nur 1-2 Zeichen unterschiedlich sind - for validId in validIds: - if len(validId) != 36: - continue - - # Zähle unterschiedliche Zeichen - differences = sum(c1 != c2 for c1, c2 in zip(incorrectId, validId)) - - # Wenn nur 1-2 Zeichen unterschiedlich sind, ist es wahrscheinlich ein Typo - if differences <= 2: - # Prüfe ob die Struktur ähnlich ist (gleiche Positionen der Bindestriche) - if incorrectId.count('-') == validId.count('-'): - return validId - - return None - diff --git a/modules/services/serviceAi/subJsonMerger.py b/modules/services/serviceAi/subJsonMerger.py deleted file mode 100644 index c5a7b058..00000000 --- a/modules/services/serviceAi/subJsonMerger.py +++ /dev/null @@ -1,2081 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Modular JSON Merger - Intelligent JSON Fragment Merging - -A clean, modular approach to merging JSON fragments that may be cut randomly. -Designed to be simple, robust, and always return valid data. - -Architecture: -1. Data Extractor: Extracts all possible data from fragments (even incomplete) -2. Structure Detector: Detects JSON structure type (elements, documents, files, etc.) -3. Data Merger: Intelligently merges data with overlap detection -4. Result Builder: Always returns valid JSON structure -""" - -import json -import re -import logging -import os -from datetime import datetime -from typing import Dict, Any, List, Optional, Tuple, Union - -from modules.shared.jsonUtils import ( - normalizeJsonText, stripCodeFences, closeJsonStructures, tryParseJson -) - -logger = logging.getLogger(__name__) - - -class JsonMergeLogger: - """Consolidated logger for JSON merging process.""" - - _logBuffer: List[str] = [] - _mergeId: int = 0 - _currentLogFile: Optional[str] = None - _appendMode: bool = False - - @staticmethod - def initializeLogFile(logFileName: Optional[str] = None): - """Initialize a new log file for a test run.""" - JsonMergeLogger._logBuffer = [] - JsonMergeLogger._mergeId = 0 - - if logFileName: - JsonMergeLogger._currentLogFile = logFileName - JsonMergeLogger._appendMode = False - # Clear existing file - try: - currentFileDir = os.path.dirname(os.path.abspath(__file__)) - logFilePath = os.path.join(currentFileDir, logFileName) - with open(logFilePath, 'w', encoding='utf-8') as f: - f.write("") # Clear file - except Exception: - pass - else: - JsonMergeLogger._currentLogFile = None - JsonMergeLogger._appendMode = False - - @staticmethod - def startMerge(accumulated: str, newFragment: str) -> str: - """Start a new merge operation and return merge ID.""" - JsonMergeLogger._mergeId += 1 - mergeId = f"merge_{JsonMergeLogger._mergeId}" - - JsonMergeLogger._log(f"{'='*80}") - JsonMergeLogger._log(f"JSON MERGE OPERATION #{JsonMergeLogger._mergeId}") - JsonMergeLogger._log(f"{'='*80}") - JsonMergeLogger._log(f"Timestamp: {datetime.now().isoformat()}") - JsonMergeLogger._log("") - - JsonMergeLogger._log("INPUT:") - JsonMergeLogger._log(f" Accumulated length: {len(accumulated)} chars") - JsonMergeLogger._log(f" New Fragment length: {len(newFragment)} chars") - # Log only summary (first 5 and last 5 lines) to avoid log spam - accLines = accumulated.split('\n') - fragLines = newFragment.split('\n') - JsonMergeLogger._log(f" Accumulated: {len(accLines)} lines (showing first 5 and last 5)") - if len(accLines) > 10: - for line in accLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(accLines) - 10} lines omitted) ...") - for line in accLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in accLines: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" New Fragment: {len(fragLines)} lines (showing first 5 and last 5)") - if len(fragLines) > 10: - for line in fragLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(fragLines) - 10} lines omitted) ...") - for line in fragLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in fragLines: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log("") - - return mergeId - - @staticmethod - def logStep(stepName: str, description: str, result: Any = None, error: Optional[str] = None): - """Log a step with its result.""" - JsonMergeLogger._log(f"STEP: {stepName}") - JsonMergeLogger._log(f" Description: {description}") - - if error: - JsonMergeLogger._log(f" ❌ ERROR: {error}") - elif result is not None: - if isinstance(result, str): - resultLines = result.split('\n') - JsonMergeLogger._log(f" ✅ Result (string, {len(result)} chars, {len(resultLines)} lines)") - if len(resultLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in resultLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(resultLines) - 10} lines omitted) ...") - for line in resultLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in resultLines: - JsonMergeLogger._log(f" {line}") - elif isinstance(result, dict): - keys = list(result.keys()) - JsonMergeLogger._log(f" ✅ Result (dict): keys={keys}, size={len(str(result))} chars") - # Log full structure with JSON formatting - NO TRUNCATION - try: - jsonStr = json.dumps(result, indent=2, ensure_ascii=False) - JsonMergeLogger._log(f" Full data (COMPLETE, {len(jsonStr)} chars):") - JsonMergeLogger._log(" " + "="*76) - for line in jsonStr.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*76) - except Exception as e: - JsonMergeLogger._log(f" Could not serialize: {e}") - strRepr = str(result) - strLines = strRepr.split('\n') - JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") - if len(strLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in strLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") - for line in strLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in strLines: - JsonMergeLogger._log(f" {line}") - # Log structure details - if "elements" in result: - elemCount = len(result["elements"]) if isinstance(result["elements"], list) else 0 - JsonMergeLogger._log(f" - elements: {elemCount} items") - if isinstance(result["elements"], list) and elemCount > 0: - JsonMergeLogger._log(f" First element type: {result['elements'][0].get('type', 'unknown') if isinstance(result['elements'][0], dict) else 'not a dict'}") - if "documents" in result: - docCount = len(result["documents"]) if isinstance(result["documents"], list) else 0 - JsonMergeLogger._log(f" - documents: {docCount} items") - elif isinstance(result, list): - JsonMergeLogger._log(f" ✅ Result (list): {len(result)} items (COMPLETE)") - if len(result) > 0: - JsonMergeLogger._log(f" First item type: {type(result[0]).__name__}") - try: - jsonStr = json.dumps(result, indent=2, ensure_ascii=False) # ALL items - JsonMergeLogger._log(f" All items (COMPLETE, {len(jsonStr)} chars):") - JsonMergeLogger._log(" " + "="*76) - for line in jsonStr.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*76) - except Exception: - strRepr = str(result) - strLines = strRepr.split('\n') - JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") - if len(strLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in strLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") - for line in strLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in strLines: - JsonMergeLogger._log(f" {line}") - else: - JsonMergeLogger._log(f" ✅ Result: {type(result).__name__} = {str(result)[:200]}") - else: - JsonMergeLogger._log(f" ⏳ In progress...") - - JsonMergeLogger._log("") - - @staticmethod - def logExtraction(strategy: str, success: bool, data: Any = None, error: Optional[str] = None): - """Log extraction strategy result.""" - status = "✅ SUCCESS" if success else "❌ FAILED" - JsonMergeLogger._log(f" Extraction Strategy: {strategy} - {status}") - if error: - JsonMergeLogger._log(f" Error: {error}") - elif data is not None: - if isinstance(data, dict): - keys = list(data.keys()) - JsonMergeLogger._log(f" Extracted keys: {keys}") - # Log full extracted data - NO TRUNCATION - try: - jsonStr = json.dumps(data, indent=2, ensure_ascii=False) - JsonMergeLogger._log(f" Extracted data (COMPLETE, {len(jsonStr)} chars):") - JsonMergeLogger._log(" " + "="*76) - for line in jsonStr.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*76) - except Exception as e: - JsonMergeLogger._log(f" Could not serialize extracted data: {e}") - strRepr = str(data) - strLines = strRepr.split('\n') - JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") - if len(strLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in strLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") - for line in strLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in strLines: - JsonMergeLogger._log(f" {line}") - elif isinstance(data, list): - JsonMergeLogger._log(f" Extracted {len(data)} items (COMPLETE)") - if len(data) > 0: - try: - jsonStr = json.dumps(data, indent=2, ensure_ascii=False) # ALL items - JsonMergeLogger._log(f" All items (COMPLETE, {len(jsonStr)} chars):") - JsonMergeLogger._log(" " + "="*76) - for line in jsonStr.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*76) - except Exception as e: - JsonMergeLogger._log(f" Could not serialize list: {e}") - strRepr = str(data) - strLines = strRepr.split('\n') - JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") - if len(strLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in strLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") - for line in strLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in strLines: - JsonMergeLogger._log(f" {line}") - - @staticmethod - def logOverlap(overlapType: str, overlapLen: int, accSuffix: Any = None, fragPrefix: Any = None): - """Log overlap detection result.""" - JsonMergeLogger._log(f" Overlap Detection ({overlapType}):") - JsonMergeLogger._log(f" Overlap length: {overlapLen}") - if overlapLen > 0: - JsonMergeLogger._log(f" ✅ Found overlap of {overlapLen} chars") - if accSuffix is not None: - if isinstance(accSuffix, str): - JsonMergeLogger._log(f" Accumulated suffix (COMPLETE, {len(accSuffix)} chars):") - JsonMergeLogger._log(" " + "="*76) - for line in accSuffix.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*76) - else: - # For lists/arrays, only log summary to avoid log flooding - if isinstance(accSuffix, list): - JsonMergeLogger._log(f" Accumulated suffix: list with {len(accSuffix)} items") - else: - JsonMergeLogger._log(f" Accumulated suffix: {type(accSuffix).__name__}") - if fragPrefix is not None: - if isinstance(fragPrefix, str): - prefixLines = fragPrefix.split('\n') - JsonMergeLogger._log(f" Fragment prefix ({len(fragPrefix)} chars, {len(prefixLines)} lines)") - if len(prefixLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") - for line in prefixLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(prefixLines) - 10} lines omitted) ...") - for line in prefixLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in prefixLines: - JsonMergeLogger._log(f" {line}") - else: - # For lists/arrays, only log summary to avoid log flooding - if isinstance(fragPrefix, list): - JsonMergeLogger._log(f" Fragment prefix: list with {len(fragPrefix)} items") - else: - JsonMergeLogger._log(f" Fragment prefix: {type(fragPrefix).__name__}") - else: - JsonMergeLogger._log(f" ⚠️ No overlap detected - appending all") - - @staticmethod - def logValidation(validationType: str, success: bool, error: Optional[str] = None): - """Log validation result.""" - status = "✅ VALID" if success else "❌ INVALID" - JsonMergeLogger._log(f" Validation ({validationType}): {status}") - if error: - JsonMergeLogger._log(f" Error: {error}") - - @staticmethod - def finishMerge(mergeId: str, finalResult: str, success: bool): - """Finish merge operation and write log file.""" - JsonMergeLogger._log("") - JsonMergeLogger._log(f"{'='*80}") - JsonMergeLogger._log(f"MERGE RESULT: {'✅ SUCCESS' if success else '❌ FAILED'}") - JsonMergeLogger._log(f"{'='*80}") - JsonMergeLogger._log(f"Final result length: {len(finalResult)} chars") - JsonMergeLogger._log("Final result (COMPLETE):") - JsonMergeLogger._log("="*80) - for line in finalResult.split('\n'): - JsonMergeLogger._log(line) - JsonMergeLogger._log("="*80) - JsonMergeLogger._log("") - - # Write log content to buffer (will be written at end of test run) - logContent = "\n".join(JsonMergeLogger._logBuffer) - - # If we have a current log file, append to it - if JsonMergeLogger._currentLogFile: - try: - currentFileDir = os.path.dirname(os.path.abspath(__file__)) - logFilePath = os.path.join(currentFileDir, JsonMergeLogger._currentLogFile) - mode = 'a' if JsonMergeLogger._appendMode else 'w' - with open(logFilePath, mode, encoding='utf-8') as f: - f.write(logContent) - f.write("\n\n") # Add separator between merges - JsonMergeLogger._appendMode = True # Next writes will append - logger.debug(f"JSON merge log appended to: {logFilePath}") - except Exception as e: - logger.error(f"Failed to write merge log file: {e}") - else: - # No log file set - write individual file (fallback) - currentFileDir = os.path.dirname(os.path.abspath(__file__)) - logDir = currentFileDir - os.makedirs(logDir, exist_ok=True) - logFilePath = os.path.join(logDir, f"{mergeId}.txt") - try: - with open(logFilePath, 'w', encoding='utf-8') as f: - f.write(logContent) - logger.info(f"JSON merge log written to: {logFilePath}") - except Exception as e: - logger.error(f"Failed to write merge log file: {e}") - - # Clear buffer for next merge - JsonMergeLogger._logBuffer = [] - - @staticmethod - def _log(message: str): - """Internal log method.""" - JsonMergeLogger._logBuffer.append(message) - # Debug logging disabled to avoid log spam with large JSON data - # logger.debug(message) - - -class JsonDataExtractor: - """Extracts data from JSON fragments, even if incomplete.""" - - @staticmethod - def extract(jsonString: str, mergeId: Optional[str] = None, removeFromEnd: bool = True) -> Dict[str, Any]: - """ - Extract complete data from JSON fragment. - - For merging: We know exactly where to clean: - - accumulated: remove incomplete parts at the END - - newFragment: remove incomplete parts at the BEGINNING - - Simple approach: Remove incomplete parts at specified position, then parse. - """ - if mergeId: - position = "END" if removeFromEnd else "BEGINNING" - JsonMergeLogger.logStep("EXTRACTION", f"Extracting data from JSON fragment ({len(jsonString)} chars) - cleaning from {position}") - - if not jsonString or not jsonString.strip(): - if mergeId: - JsonMergeLogger.logExtraction("Empty input", False, error="Input is empty") - return {} - - normalized = stripCodeFences(normalizeJsonText(jsonString)).strip() - if not normalized: - if mergeId: - JsonMergeLogger.logExtraction("Normalization", False, error="Normalized string is empty") - return {} - - # Try to parse as complete JSON first - parsed, parseErr, _ = tryParseJson(normalized) - if parseErr is None and parsed is not None: - if isinstance(parsed, dict): - finalResult = parsed - elif isinstance(parsed, list): - finalResult = {"elements": parsed} - else: - finalResult = {"elements": [parsed]} if parsed else {} - - if mergeId: - JsonMergeLogger.logExtraction("Direct parsing", True, finalResult) - JsonMergeLogger.logStep("EXTRACTION", "Direct parsing successful", finalResult) - - return finalResult if finalResult else {} - - # Remove incomplete parts from specified position - if removeFromEnd: - cleaned = JsonDataExtractor._removeIncompleteFromEnd(normalized) - else: - cleaned = JsonDataExtractor._removeIncompleteFromBeginning(normalized) - - if cleaned: - # Close structures and try to parse - closed = closeJsonStructures(cleaned) - parsed, parseErr2, _ = tryParseJson(closed) - if parseErr2 is None and parsed is not None: - if isinstance(parsed, dict): - finalResult = parsed - elif isinstance(parsed, list): - finalResult = {"elements": parsed} - else: - finalResult = {"elements": [parsed]} if parsed else {} - - if mergeId: - JsonMergeLogger.logExtraction("Remove incomplete + close", True, finalResult) - JsonMergeLogger.logStep("EXTRACTION", "Remove incomplete + close successful", finalResult) - - return finalResult if finalResult else {} - - # Return empty dict if nothing worked - if mergeId: - JsonMergeLogger.logStep("EXTRACTION", "No data extracted", {}, error="All strategies failed") - return {} - - @staticmethod - def _removeIncompleteFromEnd(jsonString: str) -> str: - """ - Remove incomplete parts from the END of JSON string. - Goes through structure level by level, keeps complete elements, removes incomplete ones at the end. - """ - # Find first '{' or '[' to start - startIdx = -1 - for i, char in enumerate(jsonString): - if char in '{[': - startIdx = i - break - - if startIdx == -1: - return "" - - # Remove incomplete parts from end recursively - cleaned = JsonDataExtractor._cleanJsonFromEnd(jsonString[startIdx:]) - return cleaned - - @staticmethod - def _removeIncompleteFromBeginning(jsonString: str) -> str: - """ - Remove incomplete parts from the BEGINNING of JSON string. - Finds where valid JSON starts and removes everything before it. - """ - # Find first '{' or '[' to start - startIdx = -1 - for i, char in enumerate(jsonString): - if char in '{[': - startIdx = i - break - - if startIdx == -1: - return "" - - # Return from start position - beginning cleanup is just finding the start - return jsonString[startIdx:] - - @staticmethod - def _cleanJsonFromEnd(jsonStr: str) -> str: - """ - Recursively clean JSON from the END: keep complete elements, remove incomplete ones at the end. - Goes through structure level by level. - """ - # Try to parse as-is first - try: - parsed = json.loads(jsonStr) - return jsonStr - except Exception: - pass - - # If dict: go through each key-value pair, remove incomplete ones at the end - if jsonStr.strip().startswith('{'): - return JsonDataExtractor._cleanDictFromEnd(jsonStr) - - # If array: go through each element, remove incomplete ones at the end - if jsonStr.strip().startswith('['): - return JsonDataExtractor._cleanArrayFromEnd(jsonStr) - - return "" - - @staticmethod - def _cleanDictFromEnd(jsonStr: str) -> str: - """Clean dict from END: keep complete key-value pairs, remove incomplete ones at the end.""" - if not jsonStr.strip().startswith('{'): - return "" - - result = ['{'] - i = 1 # Skip opening '{' - first = True - - while i < len(jsonStr): - # Skip whitespace - while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': - i += 1 - - if i >= len(jsonStr): - break - - # Check if we hit closing brace - if jsonStr[i] == '}': - break - - # Skip comma - if jsonStr[i] == ',': - i += 1 - continue - - # Try to extract key-value pair - keyStart = i - # Find key (string) - if jsonStr[i] == '"': - i += 1 - while i < len(jsonStr) and jsonStr[i] != '"': - if jsonStr[i] == '\\': - i += 2 - else: - i += 1 - if i < len(jsonStr): - i += 1 # Skip closing quote - else: - # Invalid key - stop here (incomplete at end) - break - - # Skip whitespace and colon - while i < len(jsonStr) and jsonStr[i] in ' \n\r\t:': - i += 1 - - if i >= len(jsonStr): - break - - # Try to extract value - valueStart = i - valueEnd = JsonDataExtractor._findCompleteValue(jsonStr, i) - - if valueEnd > valueStart: - # Try to parse this key-value pair - pairStr = jsonStr[keyStart:valueEnd] - try: - # Test if it's valid JSON - testStr = '{' + pairStr + '}' - json.loads(testStr) - # Valid pair - add it - if not first: - result.append(',') - result.append(pairStr) - first = False - i = valueEnd - except Exception: - # Invalid pair - stop here (incomplete at end) - break - else: - # Incomplete value - stop here (incomplete at end) - break - - result.append('}') - return ''.join(result) - - @staticmethod - def _cleanArrayFromEnd(jsonStr: str) -> str: - """Clean array from END: keep complete elements, remove incomplete ones at the end.""" - if not jsonStr.strip().startswith('['): - return "" - - result = ['['] - i = 1 # Skip opening '[' - first = True - - while i < len(jsonStr): - # Skip whitespace - while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': - i += 1 - - if i >= len(jsonStr): - break - - # Check if we hit closing bracket - if jsonStr[i] == ']': - break - - # Skip comma - if jsonStr[i] == ',': - i += 1 - continue - - # Try to extract element - elemStart = i - elemEnd = JsonDataExtractor._findCompleteValue(jsonStr, i) - - if elemEnd > elemStart: - # Try to parse this element - elemStr = jsonStr[elemStart:elemEnd] - try: - # Test if it's valid JSON - json.loads(elemStr) - # Valid element - add it - if not first: - result.append(',') - result.append(elemStr) - first = False - i = elemEnd - except Exception: - # Invalid element - stop here (incomplete at end) - break - else: - # Incomplete element - stop here (incomplete at end) - break - - result.append(']') - return ''.join(result) - - @staticmethod - def _findCompleteValue(jsonStr: str, start: int) -> int: - """Find the end of a complete JSON value starting at start position.""" - if start >= len(jsonStr): - return start - - i = start - - # Skip whitespace - while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': - i += 1 - - if i >= len(jsonStr): - return start - - char = jsonStr[i] - - # String - if char == '"': - i += 1 - while i < len(jsonStr): - if jsonStr[i] == '\\': - i += 2 - elif jsonStr[i] == '"': - return i + 1 - else: - i += 1 - return start # Incomplete string - - # Number, boolean, null - if char in '-0123456789tfn': - while i < len(jsonStr) and jsonStr[i] not in ',}]': - i += 1 - return i - - # Object - if char == '{': - braceCount = 1 - i += 1 - while i < len(jsonStr) and braceCount > 0: - if jsonStr[i] == '\\': - i += 2 - elif jsonStr[i] == '"': - # Skip string - i += 1 - while i < len(jsonStr): - if jsonStr[i] == '\\': - i += 2 - elif jsonStr[i] == '"': - i += 1 - break - else: - i += 1 - elif jsonStr[i] == '{': - braceCount += 1 - i += 1 - elif jsonStr[i] == '}': - braceCount -= 1 - i += 1 - else: - i += 1 - if braceCount == 0: - return i - return start # Incomplete object - - # Array - if char == '[': - bracketCount = 1 - i += 1 - while i < len(jsonStr) and bracketCount > 0: - if jsonStr[i] == '\\': - i += 2 - elif jsonStr[i] == '"': - # Skip string - i += 1 - while i < len(jsonStr): - if jsonStr[i] == '\\': - i += 2 - elif jsonStr[i] == '"': - i += 1 - break - else: - i += 1 - elif jsonStr[i] == '[': - bracketCount += 1 - i += 1 - elif jsonStr[i] == ']': - bracketCount -= 1 - i += 1 - else: - i += 1 - if bracketCount == 0: - return i - return start # Incomplete array - - return start - - @staticmethod - def _extractAllCompleteObjects(jsonString: str) -> List[Dict[str, Any]]: - """ - Extract ALL complete objects from JSON string using balanced brace matching. - Ignores incomplete objects at the end. - - Core principle: Every fragment can be cut anywhere - extract only complete objects. - """ - foundObjs = [] - braceCount = 0 - startPos = -1 - - for i, char in enumerate(jsonString): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - # Found a complete object - objStr = jsonString[startPos:i+1] - try: - obj = json.loads(objStr) - if isinstance(obj, dict) and obj: - foundObjs.append(obj) - except Exception: - # Not valid JSON - skip it - pass - startPos = -1 - elif braceCount < 0: - # Unbalanced - reset - braceCount = 0 - startPos = -1 - - # If we end with an incomplete object (startPos >= 0 and braceCount > 0), ignore it - # It will be in the next fragment - - return foundObjs - - @staticmethod - def _extractElements(jsonString: str) -> List[Dict[str, Any]]: - """Extract elements array from JSON string - extracts ALL complete elements.""" - elements = [] - - # Pattern 1: Look for "elements": [...] (including incomplete at end) - elementsPattern = r'"elements"\s*:\s*\[(.*)' - match = re.search(elementsPattern, jsonString, re.DOTALL) - if match: - elementsContent = match.group(1) - # Extract ALL complete element objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(elementsContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - elementStr = elementsContent[startPos:i+1] - try: - element = json.loads(elementStr) - if isinstance(element, dict): - elements.append(element) - except Exception: - # Try to extract table rows from incomplete element - rows = JsonDataExtractor._extractTableRowsFromElement(elementStr) - if rows: - elements.append({ - "type": "table", - "content": { - "rows": rows - } - }) - startPos = -1 - elif braceCount < 0: - break # Unbalanced - stop - - # Pattern 2: Look for table structure directly (even if incomplete) - if not elements: - # Look for "type": "table" pattern - tablePattern = r'"type"\s*:\s*"table"[^}]*"rows"\s*:\s*\[(.*?)(?:\]|$)' - tableMatch = re.search(tablePattern, jsonString, re.DOTALL) - if tableMatch: - rowsContent = tableMatch.group(1) - rows = JsonDataExtractor._extractRowsFromContent(rowsContent) - if rows: - elements.append({ - "type": "table", - "content": { - "rows": rows - } - }) - - # Pattern 3: Look for table rows directly (without structure) - if not elements: - rows = JsonDataExtractor._extractTableRows(jsonString) - if rows: - elements.append({ - "type": "table", - "content": { - "rows": rows - } - }) - - return elements - - @staticmethod - def _extractTableRowsFromElement(elementStr: str) -> List[List[str]]: - """Extract table rows from incomplete element string.""" - # Look for rows array in element - rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' - match = re.search(rowsPattern, elementStr, re.DOTALL) - if match: - return JsonDataExtractor._extractRowsFromContent(match.group(1)) - return [] - - @staticmethod - def _extractRowsFromContent(rowsContent: str) -> List[List[str]]: - """Extract rows from rows content string.""" - rows = [] - # Extract all array patterns: ["value1", "value2"] - # Use non-greedy matching but ensure we get complete arrays - arrayPattern = r'\[(.*?)\]' - arrayMatches = re.findall(arrayPattern, rowsContent) - for arrayContent in arrayMatches: - # Extract cells - handle both quoted strings and numbers - # First try to find quoted strings - cellPattern = r'"([^"]*)"' - cells = re.findall(cellPattern, arrayContent) - # If no quoted strings, try numbers or other values - if not cells: - # Try to find any values (numbers, booleans, etc.) - valuePattern = r'(-?\d+\.?\d*|true|false|null)' - cells = re.findall(valuePattern, arrayContent) - # Only add rows with at least 1 cell (allow single-column tables) - if len(cells) >= 1: - rows.append(cells) - return rows - - @staticmethod - def _extractTableRows(jsonString: str) -> List[List[str]]: - """Extract table rows from JSON string using multiple strategies.""" - rows = [] - - # Strategy 1: Look for "rows": [[...], [...]] - rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' - match = re.search(rowsPattern, jsonString, re.DOTALL) - if match: - rowsContent = match.group(1) - rows = JsonDataExtractor._extractRowsFromContent(rowsContent) - if rows: - return rows - - # Strategy 2: Look for standalone array patterns ["value1", "value2"] - # Pattern for complete arrays with 2 columns - completeArrayPattern = r'\["([^"]*)",\s*"([^"]*)"\]' - matches = re.findall(completeArrayPattern, jsonString) - if len(matches) >= 2: # Need at least 2 rows to be confident - return [[m[0], m[1]] for m in matches] - - # Strategy 3: Extract any array patterns (more lenient) - # Find all [ ... ] patterns that contain quoted strings - allArrays = re.findall(r'\[([^\]]*)\]', jsonString) - for arrayContent in allArrays: - # Extract quoted strings - cells = re.findall(r'"([^"]*)"', arrayContent) - if len(cells) >= 2: # At least 2 columns - rows.append(cells) - - # Only return if we have multiple rows (likely a table) - if len(rows) >= 2: - return rows - - return [] - - @staticmethod - def _extractDocuments(jsonString: str) -> List[Dict[str, Any]]: - """ - Extract documents structure from JSON string - extracts ALL complete documents/chapters/sections. - Ignores incomplete ones at the end. - - Core principle: Fragment can be cut anywhere - extract only complete objects. - """ - documents = [] - - # Pattern 1: Look for "documents": [...] structure (including incomplete at end) - documentsPattern = r'"documents"\s*:\s*\[(.*)' - match = re.search(documentsPattern, jsonString, re.DOTALL) - if match: - documentsContent = match.group(1) - # Extract ALL complete document objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(documentsContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - # Found a complete document object - docStr = documentsContent[startPos:i+1] - try: - doc = json.loads(docStr) - if isinstance(doc, dict): - # Extract chapters/sections from document - chapters = JsonDataExtractor._extractChaptersFromDocument(docStr) - sections = JsonDataExtractor._extractSectionsFromDocument(docStr) - if chapters: - doc["chapters"] = chapters - if sections: - doc["sections"] = sections - if doc: - documents.append(doc) - except Exception: - # Not valid JSON - try to extract chapters/sections directly - chapters = JsonDataExtractor._extractChaptersFromDocument(docStr) - sections = JsonDataExtractor._extractSectionsFromDocument(docStr) - if chapters or sections: - doc = {} - if chapters: - doc["chapters"] = chapters - if sections: - doc["sections"] = sections - if doc: - documents.append(doc) - startPos = -1 - elif braceCount < 0: - break - - # If we end with an incomplete document (startPos >= 0 and braceCount > 0), ignore it - # It will be in the next fragment - - if documents: - return documents - - # Pattern 2: Look for "chapters": [...] pattern directly (fragment might start mid-document) - chapters = JsonDataExtractor._extractChaptersFromString(jsonString) - if chapters: - documents.append({"chapters": chapters}) - - # Pattern 3: Look for "sections": [...] pattern directly - sections = JsonDataExtractor._extractSectionsFromString(jsonString) - if sections: - documents.append({"sections": sections}) - - return documents - - @staticmethod - def _extractChaptersFromDocument(docStr: str) -> List[Dict[str, Any]]: - """Extract chapters array from document string.""" - return JsonDataExtractor._extractChaptersFromString(docStr) - - @staticmethod - def _extractChaptersFromString(jsonString: str) -> List[Dict[str, Any]]: - """ - Extract chapters array from JSON string - extracts ALL complete chapters. - Ignores incomplete chapters at the end. - - Core principle: Fragment can be cut anywhere - extract only complete objects. - """ - chapters = [] - - # Look for "chapters": [...] pattern (including incomplete at end) - chaptersPattern = r'"chapters"\s*:\s*\[(.*)' - match = re.search(chaptersPattern, jsonString, re.DOTALL) - if match: - chaptersContent = match.group(1) - # Extract ALL complete chapter objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(chaptersContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - # Found a complete chapter object - chapterStr = chaptersContent[startPos:i+1] - try: - chapter = json.loads(chapterStr) - if isinstance(chapter, dict): - chapters.append(chapter) - except Exception: - # Not valid JSON - skip it (incomplete chapter) - pass - startPos = -1 - elif braceCount < 0: - # Unbalanced - stop here - break - - # If we end with an incomplete chapter (startPos >= 0 and braceCount > 0), ignore it - # It will be in the next fragment - - # Also try to extract chapters that might be standalone (fragment starts mid-array) - # Look for complete chapter objects anywhere in the string - if not chapters: - # Try to find complete chapter objects using balanced brace matching - allObjs = JsonDataExtractor._extractAllCompleteObjects(jsonString) - # Filter for objects that look like chapters (have id and title) - for obj in allObjs: - if isinstance(obj, dict) and "id" in obj and "title" in obj: - chapters.append(obj) - - return chapters - - @staticmethod - def _extractSectionsFromDocument(docStr: str) -> List[Dict[str, Any]]: - """Extract sections array from document string.""" - return JsonDataExtractor._extractSectionsFromString(docStr) - - @staticmethod - def _extractSectionsFromString(jsonString: str) -> List[Dict[str, Any]]: - """Extract sections array from JSON string, even if incomplete.""" - sections = [] - - # Look for "sections": [...] - sectionsPattern = r'"sections"\s*:\s*\[(.*?)(?:\]|$)' - match = re.search(sectionsPattern, jsonString, re.DOTALL) - if match: - sectionsContent = match.group(1) - # Extract section objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(sectionsContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - sectionStr = sectionsContent[startPos:i+1] - try: - section = json.loads(sectionStr) - if isinstance(section, dict): - sections.append(section) - except Exception: - # Incomplete section - try to extract what we can - idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', sectionStr) - contentTypeMatch = re.search(r'"content_type"\s*:\s*"([^"]*)"', sectionStr) - if idMatch or contentTypeMatch: - section = {} - if idMatch: - section["id"] = idMatch.group(1) - if contentTypeMatch: - section["content_type"] = contentTypeMatch.group(1) - if section: - sections.append(section) - startPos = -1 - - return sections - - @staticmethod - def _extractFiles(jsonString: str) -> List[Dict[str, Any]]: - """Extract files array from JSON string, even if incomplete.""" - files = [] - - # Look for "files": [...] - filesPattern = r'"files"\s*:\s*\[(.*?)(?:\]|$)' - match = re.search(filesPattern, jsonString, re.DOTALL) - if match: - filesContent = match.group(1) - # Extract file objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(filesContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - fileStr = filesContent[startPos:i+1] - try: - fileObj = json.loads(fileStr) - if isinstance(fileObj, dict): - files.append(fileObj) - except Exception: - # Incomplete file - try to extract what we can - idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', fileStr) - filenameMatch = re.search(r'"filename"\s*:\s*"([^"]*)"', fileStr) - if idMatch or filenameMatch: - fileObj = {} - if idMatch: - fileObj["id"] = idMatch.group(1) - if filenameMatch: - fileObj["filename"] = filenameMatch.group(1) - if fileObj: - files.append(fileObj) - startPos = -1 - - return files - - @staticmethod - def _extractImages(jsonString: str) -> List[Dict[str, Any]]: - """Extract images array from JSON string, even if incomplete.""" - images = [] - - # Look for "images": [...] - imagesPattern = r'"images"\s*:\s*\[(.*?)(?:\]|$)' - match = re.search(imagesPattern, jsonString, re.DOTALL) - if match: - imagesContent = match.group(1) - # Extract image objects using balanced brace matching - braceCount = 0 - startPos = -1 - for i, char in enumerate(imagesContent): - if char == '{': - if braceCount == 0: - startPos = i - braceCount += 1 - elif char == '}': - braceCount -= 1 - if braceCount == 0 and startPos >= 0: - imageStr = imagesContent[startPos:i+1] - try: - image = json.loads(imageStr) - if isinstance(image, dict): - images.append(image) - except Exception: - # Incomplete image - try to extract what we can - idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', imageStr) - urlMatch = re.search(r'"url"\s*:\s*"([^"]*)"', imageStr) - if idMatch or urlMatch: - image = {} - if idMatch: - image["id"] = idMatch.group(1) - if urlMatch: - image["url"] = urlMatch.group(1) - if image: - images.append(image) - startPos = -1 - - return images - - -class JsonStructureDetector: - """Detects JSON structure type from extracted data.""" - - @staticmethod - def detect(data: Dict[str, Any], mergeId: Optional[str] = None) -> str: - """ - Detect structure type from data - GENERIC approach. - - Only checks for top-level keys, no content analysis. - - Returns: - Structure type: "elements", "documents", "files", "images", or "unknown" - """ - if "elements" in data: - structureType = "elements" - elif "documents" in data: - structureType = "documents" - elif "files" in data: - structureType = "files" - elif "images" in data: - structureType = "images" - else: - # Unknown structure - will be handled generically - structureType = "unknown" - - if mergeId: - JsonMergeLogger.logStep("DETECTION", f"Detected structure type: {structureType}", structureType) - - return structureType - - -class JsonDataMerger: - """Merges JSON data intelligently with overlap detection.""" - - @staticmethod - def merge( - accumulated: Dict[str, Any], - newFragment: Dict[str, Any], - structureType: str, - mergeId: Optional[str] = None - ) -> Dict[str, Any]: - """ - Merge two JSON data structures. - - Args: - accumulated: Previously accumulated data - newFragment: New fragment data - structureType: Detected structure type - mergeId: Optional merge ID for logging - - Returns: - Merged data structure - """ - if mergeId: - JsonMergeLogger.logStep("MERGING", f"Merging {structureType} structures", { - "acc_keys": list(accumulated.keys()) if accumulated else [], - "frag_keys": list(newFragment.keys()) if newFragment else [] - }) - - if not accumulated: - if mergeId: - JsonMergeLogger.logStep("MERGING", "No accumulated data, returning fragment", newFragment) - return newFragment if newFragment else {} - if not newFragment: - if mergeId: - JsonMergeLogger.logStep("MERGING", "No fragment data, returning accumulated", accumulated) - return accumulated - - # Merge based on structure type - if structureType == "elements": - result = JsonDataMerger._mergeElements(accumulated, newFragment) - elif structureType == "documents": - result = JsonDataMerger._mergeDocuments(accumulated, newFragment) - elif structureType == "files": - result = JsonDataMerger._mergeFiles(accumulated, newFragment) - elif structureType == "images": - result = JsonDataMerger._mergeImages(accumulated, newFragment) - else: - # Unknown structure - try to merge generically - result = JsonDataMerger._mergeGeneric(accumulated, newFragment) - - if mergeId: - JsonMergeLogger.logStep("MERGING", f"Merged {structureType} structures", result) - - return result - - @staticmethod - def _mergeElements(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: - """Merge elements structures.""" - accElements = accumulated.get("elements", []) - fragElements = newFragment.get("elements", []) - - if not accElements: - return {"elements": fragElements} if fragElements else accumulated - if not fragElements: - return {"elements": accElements} - - # Merge elements with overlap detection - mergedElements = JsonDataMerger._mergeElementList(accElements, fragElements) - - return {"elements": mergedElements} - - @staticmethod - def _mergeElementList(accElements: List[Dict[str, Any]], fragElements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Merge two element lists with overlap detection.""" - if not accElements: - return fragElements - if not fragElements: - return accElements - - # Special handling: if both have table elements, merge them intelligently - accTables = [e for e in accElements if isinstance(e, dict) and e.get("type") == "table"] - fragTables = [e for e in fragElements if isinstance(e, dict) and e.get("type") == "table"] - - if accTables and fragTables: - # Merge table elements - mergedTable = JsonDataMerger._mergeTableElements(accTables[0], fragTables[0]) - if mergedTable: - # Replace tables with merged table - otherAccElements = [e for e in accElements if not (isinstance(e, dict) and e.get("type") == "table")] - otherFragElements = [e for e in fragElements if not (isinstance(e, dict) and e.get("type") == "table")] - return otherAccElements + [mergedTable] + otherFragElements - - # Find overlap by comparing elements - overlapStart = JsonDataMerger._findOverlap(accElements, fragElements, None, "elements") - - if overlapStart > 0: - # Found overlap - remove overlapping elements from fragment - merged = accElements + fragElements[overlapStart:] - return merged - else: - # No overlap - append all - return accElements + fragElements - - @staticmethod - def _mergeTableElements(accTable: Dict[str, Any], fragTable: Dict[str, Any]) -> Dict[str, Any]: - """Merge two table elements by merging their rows.""" - accRows = JsonDataMerger._getTableRows(accTable) - fragRows = JsonDataMerger._getTableRows(fragTable) - - if not accRows: - return fragTable - if not fragRows: - return accTable - - # Find overlap in rows - overlapStart = JsonDataMerger._findOverlap(accRows, fragRows, None, "table_rows") - - # Merge rows - mergedRows = accRows + fragRows[overlapStart:] if overlapStart > 0 else accRows + fragRows - - # Build merged table - mergedTable = accTable.copy() - content = mergedTable.get("content", {}) - if not isinstance(content, dict): - content = {} - content["rows"] = mergedRows - - # Preserve headers - if "headers" not in content: - fragContent = fragTable.get("content", {}) - if isinstance(fragContent, dict) and "headers" in fragContent: - content["headers"] = fragContent["headers"] - - mergedTable["content"] = content - return mergedTable - - @staticmethod - def _findOverlap(accList: List[Any], fragList: List[Any], mergeId: Optional[str] = None, overlapType: str = "generic") -> int: - """Find overlap between two lists. Returns index where overlap starts in fragList.""" - if not accList or not fragList: - if mergeId: - JsonMergeLogger.logOverlap(overlapType, 0) - return 0 - - # Try to find longest common suffix/prefix - maxOverlap = min(len(accList), len(fragList)) - - for overlapLen in range(maxOverlap, 0, -1): - accSuffix = accList[-overlapLen:] - fragPrefix = fragList[:overlapLen] - - # Compare elements - if JsonDataMerger._listsEqual(accSuffix, fragPrefix): - if mergeId: - JsonMergeLogger.logOverlap(overlapType, overlapLen, accSuffix, fragPrefix) - return overlapLen - - if mergeId: - JsonMergeLogger.logOverlap(overlapType, 0) - return 0 - - @staticmethod - def _listsEqual(list1: List[Any], list2: List[Any]) -> bool: - """Check if two lists are equal (deep comparison for dicts).""" - if len(list1) != len(list2): - return False - - for i in range(len(list1)): - if isinstance(list1[i], dict) and isinstance(list2[i], dict): - # Compare dicts by comparing their content - if not JsonDataMerger._dictsEqual(list1[i], list2[i]): - return False - elif list1[i] != list2[i]: - return False - - return True - - @staticmethod - def _dictsEqual(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> bool: - """Check if two dicts are equal (comparing key content).""" - # For table elements, compare rows - if dict1.get("type") == "table" and dict2.get("type") == "table": - rows1 = JsonDataMerger._getTableRows(dict1) - rows2 = JsonDataMerger._getTableRows(dict2) - return rows1 == rows2 - - # For other elements, compare type and key content - if dict1.get("type") != dict2.get("type"): - return False - - # Compare content - content1 = dict1.get("content", {}) - content2 = dict2.get("content", {}) - - if isinstance(content1, dict) and isinstance(content2, dict): - # Compare rows for tables - if "rows" in content1 and "rows" in content2: - return content1["rows"] == content2["rows"] - # Compare items for lists - if "items" in content1 and "items" in content2: - return content1["items"] == content2["items"] - - return dict1 == dict2 - - @staticmethod - def _getTableRows(element: Dict[str, Any]) -> List[List[str]]: - """Extract table rows from element.""" - content = element.get("content", {}) - if isinstance(content, dict): - return content.get("rows", []) - return element.get("rows", []) - - @staticmethod - def _mergeDocuments(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: - """Merge documents structures.""" - accDocs = accumulated.get("documents", []) - fragDocs = newFragment.get("documents", []) - - if not accDocs: - return {"documents": fragDocs} if fragDocs else accumulated - if not fragDocs: - return {"documents": accDocs} - - # Merge documents (simplified - would need proper merging logic) - mergedDocs = accDocs + fragDocs - return {"documents": mergedDocs} - - @staticmethod - def _mergeFiles(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: - """Merge files structures.""" - accFiles = accumulated.get("files", []) - fragFiles = newFragment.get("files", []) - - if not accFiles: - return {"files": fragFiles} if fragFiles else accumulated - if not fragFiles: - return {"files": accFiles} - - mergedFiles = accFiles + fragFiles - return {"files": mergedFiles} - - @staticmethod - def _mergeImages(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: - """Merge images structures.""" - accImages = accumulated.get("images", []) - fragImages = newFragment.get("images", []) - - if not accImages: - return {"images": fragImages} if fragImages else accumulated - if not fragImages: - return {"images": accImages} - - mergedImages = accImages + fragImages - return {"images": mergedImages} - - @staticmethod - def _mergeGeneric(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: - """Generic merge for unknown structures.""" - # Try to merge by combining keys - merged = accumulated.copy() - for key, value in newFragment.items(): - if key in merged: - # Key exists - try to merge values - if isinstance(merged[key], list) and isinstance(value, list): - merged[key] = merged[key] + value - elif isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = JsonDataMerger._mergeGeneric(merged[key], value) - else: - merged[key] = value - else: - merged[key] = value - - return merged - - -class JsonResultBuilder: - """Builds final JSON result, ensuring it's always valid.""" - - @staticmethod - def build(mergedData: Dict[str, Any], structureType: str, mergeId: Optional[str] = None) -> str: - """ - Build final JSON string from merged data. - - Args: - mergedData: Merged data structure - structureType: Detected structure type - - Returns: - Valid JSON string (never empty) - """ - if not mergedData: - # Return empty structure based on type - if structureType == "elements": - return json.dumps({"elements": []}, indent=2, ensure_ascii=False) - elif structureType == "documents": - return json.dumps({"documents": [{}]}, indent=2, ensure_ascii=False) - elif structureType == "files": - return json.dumps({"files": []}, indent=2, ensure_ascii=False) - elif structureType == "images": - return json.dumps({"images": []}, indent=2, ensure_ascii=False) - else: - return json.dumps({}, indent=2, ensure_ascii=False) - - # Ensure structure is correct - GENERIC approach - if structureType == "elements" and "elements" not in mergedData: - # Try to wrap data in elements structure - if isinstance(mergedData, dict): - # Generic: If it has any data, wrap it as an element - if mergedData: - mergedData = {"elements": [mergedData]} - if mergeId: - JsonMergeLogger.logStep("BUILDING", "Wrapping single object as element (generic)", mergedData) - else: - # Empty dict - return empty elements - mergedData = {"elements": []} - - elif structureType == "documents" and "documents" not in mergedData: - # Try to wrap data in documents structure - if isinstance(mergedData, dict): - if mergedData: - # Generic: Wrap single object in documents structure - # Try to detect if it should be chapters or sections by checking accumulated data - # But for now, use generic approach: wrap in documents with a generic key - mergedData = {"documents": [mergedData]} - if mergeId: - JsonMergeLogger.logStep("BUILDING", "Wrapping single object in documents structure (generic)", mergedData) - else: - mergedData = {"documents": [{}]} - - elif structureType == "files" and "files" not in mergedData: - # Try to wrap data in files structure - if isinstance(mergedData, dict): - if mergedData: - mergedData = {"files": [mergedData]} - if mergeId: - JsonMergeLogger.logStep("BUILDING", "Wrapping single object in files structure (generic)", mergedData) - else: - mergedData = {"files": []} - - elif structureType == "images" and "images" not in mergedData: - # Try to wrap data in images structure - if isinstance(mergedData, dict): - if mergedData: - mergedData = {"images": [mergedData]} - if mergeId: - JsonMergeLogger.logStep("BUILDING", "Wrapping single object in images structure (generic)", mergedData) - else: - mergedData = {"images": []} - - elif structureType == "unknown" and isinstance(mergedData, dict) and mergedData: - # Unknown structure but has data - wrap generically as elements - mergedData = {"elements": [mergedData]} - if mergeId: - JsonMergeLogger.logStep("BUILDING", "Unknown structure, wrapping as elements (generic)", mergedData) - - # Clean data structure before serialization - cleanedData = JsonResultBuilder._cleanDataStructure(mergedData) - - # Try to serialize - try: - jsonString = json.dumps(cleanedData, indent=2, ensure_ascii=False) - - # Validate the JSON string by trying to parse it - try: - parsed, parseErr, _ = tryParseJson(jsonString) - if parseErr is None: - # Valid JSON - return it - return jsonString - else: - # Invalid JSON - try to repair - logger.warning(f"Generated JSON is invalid: {parseErr}, attempting repair") - repaired = closeJsonStructures(jsonString) - parsed2, parseErr2, _ = tryParseJson(repaired) - if parseErr2 is None: - return repaired - else: - # Repair failed - return minimal valid structure - logger.error(f"Repair failed: {parseErr2}, returning minimal structure") - return json.dumps({"elements": []}, indent=2, ensure_ascii=False) - except Exception as parseEx: - # Parse validation failed - try repair - logger.warning(f"Parse validation failed: {parseEx}, attempting repair") - try: - repaired = closeJsonStructures(jsonString) - parsed2, parseErr2, _ = tryParseJson(repaired) - if parseErr2 is None: - return repaired - except Exception: - pass - # Return minimal valid structure - return json.dumps({"elements": []}, indent=2, ensure_ascii=False) - - except (TypeError, ValueError) as e: - logger.error(f"Error serializing JSON: {e}") - # Try to clean more aggressively and retry - try: - cleanedData2 = JsonResultBuilder._cleanDataStructure(cleanedData, aggressive=True) - jsonString = json.dumps(cleanedData2, indent=2, ensure_ascii=False) - # Validate - parsed, parseErr, _ = tryParseJson(jsonString) - if parseErr is None: - return jsonString - except Exception: - pass - # Fallback to empty structure - return json.dumps({"elements": []}, indent=2, ensure_ascii=False) - except Exception as e: - logger.error(f"Unexpected error building JSON: {e}") - # Fallback to empty structure - return json.dumps({"elements": []}, indent=2, ensure_ascii=False) - - @staticmethod - def _cleanDataStructure(data: Any, aggressive: bool = False) -> Any: - """ - Clean data structure to ensure it's JSON-serializable. - - Removes None values, ensures lists contain only valid items, - and repairs incomplete structures. - """ - if data is None: - return {} if aggressive else None - - if isinstance(data, dict): - cleaned = {} - for key, value in data.items(): - if value is None and aggressive: - continue # Skip None values in aggressive mode - cleaned[key] = JsonResultBuilder._cleanDataStructure(value, aggressive) - return cleaned - - elif isinstance(data, list): - cleaned = [] - for item in data: - cleanedItem = JsonResultBuilder._cleanDataStructure(item, aggressive) - if cleanedItem is not None or not aggressive: - cleaned.append(cleanedItem) - return cleaned - - elif isinstance(data, (str, int, float, bool)): - return data - - else: - # Unknown type - try to convert to string or skip - if aggressive: - return str(data) - return data - - -class ModularJsonMerger: - """ - Modular JSON Merger - Main entry point. - - Simple pipeline: - 1. Find overlap between JSON strings - 2. Merge strings together - 3. Parse and clean the merged JSON - """ - - @staticmethod - def _findStringOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: - """ - Find overlap between two JSON strings - GENERIC solution. - - Works for any JSON structure (arrays, objects, nested, minified, formatted). - Uses multiple strategies to find overlap regardless of JSON format. - - Strategy: - 1. Exact suffix/prefix match (fastest, works for any format) - 2. Structure-aware: Find last complete JSON elements in accumulated that match start of fragment - 3. Line-based: If JSON is formatted, use line matching (for better performance) - 4. Partial match: Handle incomplete elements at cut point - - Returns the length of the overlap (number of characters). - """ - if not accStr or not fragStr: - if mergeId: - JsonMergeLogger.logOverlap("string", 0) - return 0 - - # Strategy 1: Try exact suffix/prefix match (fastest, works for any format) - maxOverlap = min(len(accStr), len(fragStr)) - - # Start from maximum possible overlap and work backwards - for overlapLen in range(maxOverlap, 0, -1): - accSuffix = accStr[-overlapLen:] - fragPrefix = fragStr[:overlapLen] - - if accSuffix == fragPrefix: - if mergeId: - JsonMergeLogger.logOverlap("string (exact)", overlapLen, accSuffix[:200], fragPrefix[:200]) - return overlapLen - - # Strategy 2: Structure-aware overlap detection (GENERIC - works for any JSON structure) - # Find last complete JSON elements in accumulated and check if they appear at start of fragment - overlapLen = ModularJsonMerger._findStructureBasedOverlap(accStr, fragStr, mergeId) - if overlapLen > 0: - return overlapLen - - # Strategy 3: Line-based overlap (works well for formatted JSON) - # Only use if JSON appears to be formatted (has newlines) - if '\n' in accStr and '\n' in fragStr: - overlapLen = ModularJsonMerger._findLineBasedOverlap(accStr, fragStr, mergeId) - if overlapLen > 0: - return overlapLen - - # Strategy 4: Partial overlap (incomplete element at cut point) - overlapLen = ModularJsonMerger._findPartialOverlap(accStr, fragStr, mergeId) - if overlapLen > 0: - return overlapLen - - if mergeId: - JsonMergeLogger.logOverlap("string", 0) - return 0 - - @staticmethod - def _findStructureBasedOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: - """ - Find overlap by detecting complete JSON elements (structure-aware, GENERIC). - - Works for ANY JSON structure: - - Arrays: Finds last complete array elements - - Objects: Finds last complete object properties - - Nested structures: Recursively finds complete elements - - Minified or formatted JSON: Structure-aware, not format-dependent - - Any use case: section_content, chapter_structure, code_structure, etc. - - Strategy: Find last complete JSON elements in accumulated that match start of fragment. - Uses balanced bracket/brace matching to identify complete elements regardless of format. - """ - accTrimmed = accStr.rstrip() - fragTrimmed = fragStr.lstrip() - - if not accTrimmed or not fragTrimmed: - return 0 - - # Find last complete elements in accumulated by parsing backwards - # Look for complete array elements or object properties - - # Strategy: Find where accumulated has complete elements at the end - # and check if fragment starts with the same elements - - # Use a sliding window approach: check different suffix lengths from accumulated - maxCheckLength = min(2000, len(accTrimmed), len(fragTrimmed)) - - # Check in reverse order (largest to smallest) to find longest overlap first - for checkLen in range(maxCheckLength, 50, -5): # Step by 5 for performance - if checkLen > len(accTrimmed) or checkLen > len(fragTrimmed): - continue - - accSuffix = accTrimmed[-checkLen:] - fragPrefix = fragTrimmed[:checkLen] - - # Check if accSuffix ends with complete JSON element(s) and fragPrefix starts with same - # A complete element ends with proper closing brackets/braces - - # Verify that accSuffix ends with complete structure - # and fragPrefix starts with the same structure - if ModularJsonMerger._isCompleteJsonElement(accSuffix) and \ - ModularJsonMerger._startsWithSameElement(accSuffix, fragPrefix): - # Found overlap! Verify it's meaningful (not just whitespace) - if len(accSuffix.strip()) > 20: - if mergeId: - JsonMergeLogger.logOverlap("string (structure-based)", checkLen, accSuffix[:200], fragPrefix[:200]) - return checkLen - - # Alternative: Try to find common substring that represents complete elements - # Look for patterns like complete array rows or object properties - # Check last 500 chars of accumulated against first 500 chars of fragment - checkWindow = min(500, len(accTrimmed), len(fragTrimmed)) - if checkWindow > 100: - accWindow = accTrimmed[-checkWindow:] - fragWindow = fragTrimmed[:checkWindow] - - # Find longest common substring that represents complete elements - # Look for boundaries like ], [ or }, { or ", " - for i in range(checkWindow - 50, 50, -5): - accSub = accWindow[-i:] - fragSub = fragWindow[:i] - - if accSub == fragSub: - # Check if it's a complete element boundary - if ModularJsonMerger._isCompleteElementBoundary(accSub): - if mergeId: - JsonMergeLogger.logOverlap("string (structure-boundary)", i, accSub[:200], fragSub[:200]) - return i - - return 0 - - @staticmethod - def _isCompleteJsonElement(jsonStr: str) -> bool: - """Check if string ends with a complete JSON element (balanced brackets/braces).""" - jsonStr = jsonStr.strip() - if not jsonStr: - return False - - # Check if it ends with complete structure markers - # Complete array element: ends with ] or ], or ], - # Complete object element: ends with } or }, or }, - if jsonStr[-1] in ']}': - # Check if brackets/braces are balanced - braceCount = jsonStr.count('{') - jsonStr.count('}') - bracketCount = jsonStr.count('[') - jsonStr.count(']') - return braceCount == 0 and bracketCount == 0 - - return False - - @staticmethod - def _startsWithSameElement(accSuffix: str, fragPrefix: str) -> bool: - """Check if fragment prefix starts with the same element as accumulated suffix.""" - # Normalize whitespace for comparison - accNorm = accSuffix.strip() - fragNorm = fragPrefix.strip() - - # Check if fragPrefix starts with accSuffix (or vice versa for partial matches) - if fragNorm.startswith(accNorm): - return True - - # Check if they have common prefix (for partial element completion) - minLen = min(len(accNorm), len(fragNorm)) - if minLen > 20: - # Check if first 80% of accSuffix matches start of fragPrefix - checkLen = int(minLen * 0.8) - return accNorm[:checkLen] == fragNorm[:checkLen] - - return False - - @staticmethod - def _isCompleteElementBoundary(jsonStr: str) -> bool: - """Check if string represents a complete element boundary (e.g., ], [ or }, {).""" - jsonStr = jsonStr.strip() - if not jsonStr: - return False - - # Check if it contains complete element boundaries - # Pattern: ends with ], or }, or ],\n or },\n - if jsonStr.rstrip().endswith(('],', '},', ']', '}')): - return True - - # Check if it's a complete array element or object property - if '],' in jsonStr or '},' in jsonStr: - return True - - return False - - @staticmethod - def _findLineBasedOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: - """ - Find overlap using line-based matching (for formatted JSON). - """ - accLines = accStr.rstrip().split('\n') - fragLines = fragStr.lstrip().split('\n') - - # Try to find matching lines from the end of accumulated at the start of fragment - maxLinesToCheck = min(10, len(accLines), len(fragLines)) - - for numLines in range(maxLinesToCheck, 0, -1): - # Get last N lines from accumulated (excluding empty lines) - accLastLines = [line.strip() for line in accLines[-numLines:] if line.strip()] - # Get first N lines from fragment (excluding empty lines) - fragFirstLines = [line.strip() for line in fragLines[:numLines] if line.strip()] - - # Check if they match - if len(accLastLines) > 0 and len(fragFirstLines) > 0: - # Try to find where accLastLines match fragFirstLines - for i in range(len(accLastLines)): - # Check if accLastLines[i:] matches fragFirstLines[:len(accLastLines)-i] - accSuffixLines = accLastLines[i:] - fragPrefixLines = fragFirstLines[:len(accSuffixLines)] - - if accSuffixLines == fragPrefixLines and len(accSuffixLines) > 0: - # Found overlap! Calculate character length - accSuffixText = '\n'.join(accLastLines[i:]) - fragPrefixText = '\n'.join(fragPrefixLines) - - # Find where this text appears in the original strings - accPos = accStr.rfind(accSuffixText) - fragPos = fragStr.find(fragPrefixText) - - if accPos >= 0 and fragPos == 0: - # Found valid overlap - overlapLen = len(accSuffixText) - if mergeId: - JsonMergeLogger.logOverlap("string (line-based)", overlapLen, accSuffixText[:200], fragPrefixText[:200]) - return overlapLen - - return 0 - - @staticmethod - def _findPartialOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: - """ - Find partial overlap (incomplete element at cut point). - """ - accLines = accStr.rstrip().split('\n') - fragLines = fragStr.lstrip().split('\n') - - if accLines and fragLines: - lastAccLine = accLines[-1].strip() - firstFragLine = fragLines[0].strip() - - # Check if lastAccLine is a prefix of firstFragLine (incomplete line completed) - if lastAccLine and firstFragLine.startswith(lastAccLine): - # Also check if there are more matching lines after - overlapLen = len(lastAccLine) - # Try to extend overlap with more lines - for i in range(1, min(len(accLines), len(fragLines))): - if accLines[-1-i].strip() == fragLines[i].strip(): - overlapLen += len('\n' + fragLines[i]) - else: - break - - if overlapLen > 20: # Only if meaningful overlap - if mergeId: - JsonMergeLogger.logOverlap("string (partial line)", overlapLen, lastAccLine[:200], firstFragLine[:200]) - return overlapLen - - return 0 - - @staticmethod - def _mergeStrings(accStr: str, fragStr: str, overlapLength: int) -> str: - """ - Merge two JSON strings together, removing the overlap. - Handles whitespace at cut points properly for seamless merging. - """ - if overlapLength > 0: - # Remove overlap from fragment and append - # CRITICAL: Handle whitespace properly - if accumulated ends with whitespace - # and fragment starts with the same content, we need to preserve whitespace structure - merged = accStr + fragStr[overlapLength:] - else: - # No overlap - just concatenate (might need comma or other separator) - # CRITICAL: Preserve whitespace structure when merging - - # Get trailing whitespace from accumulated (spaces, tabs, but not newlines) - accTrailingWs = "" - i = len(accStr) - 1 - while i >= 0 and accStr[i] in [' ', '\t']: - accTrailingWs = accStr[i] + accTrailingWs - i -= 1 - - # Get leading whitespace from fragment (spaces, tabs, but not newlines) - fragLeadingWs = "" - i = 0 - while i < len(fragStr) and fragStr[i] in [' ', '\t']: - fragLeadingWs += fragStr[i] - i += 1 - - # Trim for content detection but preserve whitespace structure - accTrimmed = accStr.rstrip().rstrip(',') - fragTrimmed = fragStr.lstrip().lstrip(',') - - # Check if we need a separator - if accTrimmed and fragTrimmed: - # If accumulated ends with } or ] and fragment starts with { or [, we might need comma - if (accTrimmed[-1] in '}]' and fragTrimmed[0] in '{['): - # Add comma with appropriate whitespace - merged = accTrimmed + ',' + fragLeadingWs + fragTrimmed - else: - # Merge with preserved whitespace structure - # Use the whitespace from fragment (it knows the proper spacing) - merged = accTrimmed + accTrailingWs + fragLeadingWs + fragTrimmed - else: - # One is empty - just concatenate with preserved whitespace - merged = accStr + fragStr - - return merged - - @staticmethod - def merge(accumulated: str, newFragment: str) -> Tuple[str, bool]: - """ - Merge two JSON fragments intelligently. - - Args: - accumulated: Previously accumulated JSON string - newFragment: New fragment JSON string - - Returns: - Tuple of (merged_json_string, has_overlap): - - merged_json_string: Merged JSON string (closed if no overlap, unclosed if overlap found) - - has_overlap: True if overlap was found (iterations should continue), False if no overlap (iterations should stop) - """ - # Start logging - mergeId = JsonMergeLogger.startMerge(accumulated, newFragment) - - if not accumulated: - result = newFragment if newFragment else "{}" - JsonMergeLogger.finishMerge(mergeId, result, True) - return (result, False) # No overlap if no accumulated data - if not newFragment: - JsonMergeLogger.finishMerge(mergeId, accumulated, True) - return (accumulated, False) # No overlap if no new fragment - - try: - # Normalize both strings - accNormalized = stripCodeFences(normalizeJsonText(accumulated)).strip() - fragNormalized = stripCodeFences(normalizeJsonText(newFragment)).strip() - - JsonMergeLogger._log(f"\n Normalized Accumulated ({len(accNormalized)} chars)") - accNormLines = accNormalized.split('\n') - if len(accNormLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(accNormLines)} lines)") - for line in accNormLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(accNormLines) - 10} lines omitted) ...") - for line in accNormLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in accNormLines: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f"\n Normalized New Fragment ({len(fragNormalized)} chars)") - fragNormLines = fragNormalized.split('\n') - if len(fragNormLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(fragNormLines)} lines)") - for line in fragNormLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(fragNormLines) - 10} lines omitted) ...") - for line in fragNormLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in fragNormLines: - JsonMergeLogger._log(f" {line}") - - # Step 1: Find overlap between JSON strings - JsonMergeLogger.logStep("PHASE 1", "Finding overlap between JSON strings", None) - overlapLength = ModularJsonMerger._findStringOverlap(accNormalized, fragNormalized, mergeId) - - if overlapLength > 0: - accSuffix = accNormalized[-overlapLength:] - fragPrefix = fragNormalized[:overlapLength] - JsonMergeLogger._log(f"\n Overlap found ({overlapLength} chars):") - JsonMergeLogger._log(f" Accumulated suffix: {accSuffix}") - JsonMergeLogger._log(f" Fragment prefix: {fragPrefix}") - else: - # CRITICAL: No overlap found - this means iterations should stop - JsonMergeLogger._log(f"\n ⚠️ NO OVERLAP FOUND - This indicates iterations should stop") - JsonMergeLogger._log(f" Closing JSON and returning final result") - - # Close the accumulated JSON (it's complete as far as we can tell) - closedJson = closeJsonStructures(accNormalized) - JsonMergeLogger._log(f"\n Closed JSON ({len(closedJson)} chars):") - JsonMergeLogger._log(" " + "="*78) - for line in closedJson.split('\n'): - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(" " + "="*78) - - JsonMergeLogger.finishMerge(mergeId, closedJson, True) - # Return closed JSON with has_overlap=False to indicate iterations should stop - return (closedJson, False) - - # Step 2: Merge strings together (only if overlap was found) - JsonMergeLogger.logStep("PHASE 2", f"Merging strings (overlap: {overlapLength} chars)", None) - mergedString = ModularJsonMerger._mergeStrings(accNormalized, fragNormalized, overlapLength) - - JsonMergeLogger._log(f"\n Merged String ({len(mergedString)} chars)") - mergedLines = mergedString.split('\n') - if len(mergedLines) > 10: - JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(mergedLines)} lines)") - for line in mergedLines[:5]: - JsonMergeLogger._log(f" {line}") - JsonMergeLogger._log(f" ... ({len(mergedLines) - 10} lines omitted) ...") - for line in mergedLines[-5:]: - JsonMergeLogger._log(f" {line}") - else: - for line in mergedLines: - JsonMergeLogger._log(f" {line}") - - # Step 3: Return merged string (with incomplete element at end for next iteration) - JsonMergeLogger.logStep("PHASE 3", "Returning merged string (may be unclosed)", None) - JsonMergeLogger._log(f"\n Returning merged string (preserving incomplete element at end for next iteration)") - - JsonMergeLogger.finishMerge(mergeId, mergedString, True) - # Return merged string with has_overlap=True to indicate iterations should continue - return (mergedString, True) - - except Exception as e: - logger.error(f"Error in modular merger: {e}") - JsonMergeLogger.logStep("ERROR", f"Exception occurred: {str(e)}", None, error=str(e)) - # Fallback: try to return accumulated if valid - try: - accParsed, accErr, _ = tryParseJson(accumulated) - if accErr is None: - JsonMergeLogger.finishMerge(mergeId, accumulated, False) - return (accumulated, False) # No overlap on error - except Exception: - pass - # Last resort: return empty valid JSON - fallback = json.dumps({"elements": []}, indent=2, ensure_ascii=False) - JsonMergeLogger.finishMerge(mergeId, fallback, False) - return (fallback, False) # No overlap on error diff --git a/modules/services/serviceAi/subJsonResponseHandling.py b/modules/services/serviceAi/subJsonResponseHandling.py deleted file mode 100644 index 3adb613c..00000000 --- a/modules/services/serviceAi/subJsonResponseHandling.py +++ /dev/null @@ -1,3121 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -JSON Response Handling Module - -Handles merging of JSON responses from multiple AI iterations, including: -- Section merging with intelligent overlap detection -- JSON fragment detection and merging -- Deep recursive structure merging -- Overlap detection for complex nested structures -- String accumulation for iterative JSON generation -""" -import json -import logging -import re -from typing import Dict, Any, List, Optional, Tuple - -from modules.shared.jsonUtils import extractJsonString, repairBrokenJson, extractSectionsFromDocument -from modules.datamodels.datamodelAi import JsonAccumulationState - -logger = logging.getLogger(__name__) - - -class JsonResponseHandler: - """Handles JSON response merging and fragment detection for iterative AI generation.""" - - @staticmethod - def mergeSectionsIntelligently( - existingSections: List[Dict[str, Any]], - newSections: List[Dict[str, Any]], - iteration: int - ) -> List[Dict[str, Any]]: - """ - Intelligently merge sections from multiple iterations. - - This is a GENERIC merging strategy that handles broken JSON iterations. - The break can occur anywhere - in any section, at any depth. - - Merging strategies (in order of priority): - 1. Same Section ID: Merge sections with identical IDs - 2. Same Content-Type + Position: If last section is incomplete and new section continues it - 3. Same Order: Merge sections with same order value - 4. Structural Analysis: Detect continuation based on content structure - - Args: - existingSections: Sections accumulated from previous iterations - newSections: Sections extracted from current iteration - iteration: Current iteration number - - Returns: - Merged list of sections - """ - if not newSections: - return existingSections - - if not existingSections: - return newSections - - mergedSections = existingSections.copy() - - for newSection in newSections: - merged = False - - # Strategy 1: Same Section ID - merge directly - newSectionId = newSection.get("id") - if newSectionId: - for i, existingSection in enumerate(mergedSections): - if existingSection.get("id") == newSectionId: - # Merge sections with same ID - mergedSections[i] = JsonResponseHandler.mergeSectionContent( - existingSection, newSection, iteration - ) - merged = True - logger.debug(f"Iteration {iteration}: Merged section by ID '{newSectionId}'") - break - - if merged: - continue - - # Strategy 2: Same Content-Type + Position (continuation detection) - # Check if last section is incomplete and new section continues it - if mergedSections: - lastSection = mergedSections[-1] - lastContentType = lastSection.get("content_type") - newContentType = newSection.get("content_type") - - if lastContentType == newContentType: - # Same content type - check if last section is incomplete - if JsonResponseHandler.isSectionIncomplete(lastSection): - # Last section is incomplete, merge with new section - mergedSections[-1] = JsonResponseHandler.mergeSectionContent( - lastSection, newSection, iteration - ) - merged = True - logger.debug(f"Iteration {iteration}: Merged section by content-type continuation ({lastContentType})") - continue - - # Strategy 3: Same Order value - newOrder = newSection.get("order") - if newOrder is not None: - for i, existingSection in enumerate(mergedSections): - existingOrder = existingSection.get("order") - if existingOrder is not None and existingOrder == newOrder: - # Merge sections with same order - mergedSections[i] = JsonResponseHandler.mergeSectionContent( - existingSection, newSection, iteration - ) - merged = True - logger.debug(f"Iteration {iteration}: Merged section by order {newOrder}") - break - - if merged: - continue - - # Strategy 4: Structural Analysis - detect continuation - # For code_block and table: if last section matches new section type, merge them - if mergedSections: - lastSection = mergedSections[-1] - lastContentType = lastSection.get("content_type") - newContentType = newSection.get("content_type") - - # Both are code blocks - merge them - if lastContentType == "code_block" and newContentType == "code_block": - mergedSections[-1] = JsonResponseHandler.mergeSectionContent( - lastSection, newSection, iteration - ) - merged = True - logger.debug(f"Iteration {iteration}: Merged code_block sections by structural analysis") - continue - - # Both are tables - merge them (common case for broken JSON iterations) - if lastContentType == "table" and newContentType == "table": - mergedSections[-1] = JsonResponseHandler.mergeSectionContent( - lastSection, newSection, iteration - ) - merged = True - logger.debug(f"Iteration {iteration}: Merged table sections by structural analysis") - continue - - # No merge strategy matched - add as new section - if not merged: - mergedSections.append(newSection) - logger.debug(f"Iteration {iteration}: Added new section '{newSection.get('id', 'no-id')}' ({newSection.get('content_type', 'unknown')})") - - return mergedSections - - @staticmethod - def isSectionIncomplete(section: Dict[str, Any]) -> bool: - """ - Check if a section is incomplete (broken at the end). - - This detects incomplete sections based on content analysis: - - Code blocks: ends mid-line, ends with comma, ends with incomplete structure - - Text sections: ends mid-sentence, ends with incomplete structure - - Other types: check for incomplete elements - """ - contentType = section.get("content_type", "") - elements = section.get("elements", []) - - if not elements: - return False - - # Handle list of elements - if isinstance(elements, list) and len(elements) > 0: - lastElement = elements[-1] - else: - lastElement = elements - - if not isinstance(lastElement, dict): - return False - - # Check code_block for incomplete code - if contentType == "code_block": - code = lastElement.get("code", "") - if code: - # Check if code ends incompletely: - # - Ends with comma (incomplete CSV line) - # - Ends with number but no newline (incomplete line) - # - Ends mid-token (e.g., "23431,23" - incomplete number) - codeStripped = code.rstrip() - if codeStripped: - # Check for incomplete patterns - if codeStripped.endswith(',') or (',' in codeStripped and not codeStripped.endswith('\n')): - # Ends with comma or has comma but no final newline - likely incomplete - return True - # Check if last line is incomplete (doesn't end with newline and has partial content) - if not code.endswith('\n') and codeStripped: - # No final newline - might be incomplete - # More sophisticated: check if last number is complete - lastLine = codeStripped.split('\n')[-1] - if lastLine and ',' in lastLine: - # Has commas but might be incomplete - parts = lastLine.split(',') - if parts and len(parts[-1]) < 5: # Last part is very short - might be incomplete - return True - - # Check table for incomplete rows - if contentType == "table": - rows = lastElement.get("rows", []) - if rows: - # Check if last row is incomplete (ends with incomplete data) - lastRow = rows[-1] if isinstance(rows, list) else [] - if isinstance(lastRow, list) and lastRow: - # CRITICAL: Check if last row doesn't have expected number of columns (if headers exist) - # This is the PRIMARY indicator of incomplete table rows - headers = lastElement.get("headers", []) - if headers and isinstance(headers, list): - expectedCols = len(headers) - if len(lastRow) < expectedCols: - logger.debug(f"Table section incomplete: last row has {len(lastRow)} columns, expected {expectedCols}") - return True - # Also check if last row ends with incomplete data (e.g., incomplete string) - lastCell = lastRow[-1] if lastRow else "" - if isinstance(lastCell, str): - # If last cell is incomplete (ends with quote or is very short), section might be incomplete - if lastCell.endswith('"') or (len(lastCell) < 3 and lastCell): - logger.debug(f"Table section incomplete: last cell appears incomplete: '{lastCell}'") - return True - # Additional check: if last row has fewer cells than previous rows, it's likely incomplete - if len(rows) > 1: - prevRow = rows[-2] if isinstance(rows, list) and len(rows) > 1 else [] - if isinstance(prevRow, list) and len(prevRow) > len(lastRow): - logger.debug(f"Table section incomplete: last row has {len(lastRow)} cells, previous row has {len(prevRow)}") - return True - - # Check paragraph/text for incomplete sentences - if contentType in ["paragraph", "heading"]: - text = lastElement.get("text", "") - if text: - # Simple heuristic: if doesn't end with sentence-ending punctuation - textStripped = text.rstrip() - if textStripped and not textStripped[-1] in '.!?': - # Might be incomplete, but this is less reliable - # Only mark as incomplete if very short (likely cut off) - if len(textStripped) < 20: - return True - - # Check lists for incomplete items - if contentType in ["bullet_list", "numbered_list"]: - items = lastElement.get("items", []) - if items and isinstance(items, list): - # Check if last item is incomplete (very short or ends with incomplete string) - lastItem = items[-1] if items else None - if isinstance(lastItem, str) and len(lastItem) < 3: - return True - - # Check image for incomplete base64 data - if contentType == "image": - imageData = lastElement.get("base64Data", "") - if imageData: - # Base64 strings should end with padding ('=' or '==') - # If it doesn't, it might be incomplete - stripped = imageData.rstrip() - if stripped and not stripped.endswith(('=', '==')): - # Check if it's a valid base64 character sequence that was cut off - if len(stripped) > 0 and stripped[-1] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=': - return True - # If length is not a multiple of 4 (base64 requirement), it might be incomplete - if len(stripped) % 4 != 0: - return True - - # GENERIC CHECK: Recursively analyze structure for incompleteness - # This works for ANY structure: arrays, objects, nested, primitives - return JsonResponseHandler._isStructureIncomplete(lastElement) - - @staticmethod - def _isStructureIncomplete(structure: Any, max_depth: int = 10) -> bool: - """ - GENERIC recursive check for incomplete structures. - - Detects incompleteness by analyzing patterns: - - Arrays: Last item shorter than previous items, incomplete patterns - - Objects: Last object has fewer keys than pattern, incomplete values - - Strings: Very short, ends abruptly, incomplete patterns - - Nested: Recursively checks nested structures - - Works for ANY JSON structure of any depth/complexity. - """ - if max_depth <= 0: - return False - - # Arrays/Lists - check for incomplete patterns - if isinstance(structure, list): - if len(structure) == 0: - return False - - # Check if last item is incomplete compared to previous items - last_item = structure[-1] - - # If we have previous items, compare structure - if len(structure) > 1: - prev_item = structure[-2] - - # If last item is a list and previous is a list, check length - if isinstance(last_item, list) and isinstance(prev_item, list): - if len(last_item) < len(prev_item): - return True # Last row/item has fewer elements - likely incomplete - - # If last item is a dict and previous is a dict, check keys - if isinstance(last_item, dict) and isinstance(prev_item, dict): - if len(last_item) < len(prev_item): - return True # Last object has fewer keys - likely incomplete - - # Recursively check last item for incompleteness - if JsonResponseHandler._isStructureIncomplete(last_item, max_depth - 1): - return True - - # Objects/Dicts - check for incomplete values - elif isinstance(structure, dict): - for key, value in structure.items(): - # Recursively check each value - if JsonResponseHandler._isStructureIncomplete(value, max_depth - 1): - return True - - # Check for incomplete strings - if isinstance(value, str): - # Very short strings might be incomplete - if len(value) > 0 and len(value) < 3: - return True - # Strings ending with incomplete patterns (comma, quote, etc.) - stripped = value.rstrip() - if stripped and stripped.endswith((',', '"', '\\')): - return True - - # Strings - check for incomplete patterns - elif isinstance(structure, str): - # Very short strings might be incomplete - if len(structure) > 0 and len(structure) < 3: - return True - # Strings ending with incomplete patterns - stripped = structure.rstrip() - if stripped and stripped.endswith((',', '"', '\\')): - return True - - return False - - @staticmethod - def mergeSectionContent( - existingSection: Dict[str, Any], - newSection: Dict[str, Any], - iteration: int - ) -> Dict[str, Any]: - """ - Merge content from two sections. - - Handles different content types: - - code_block: Append code, handle overlaps, merge incomplete lines - - paragraph/heading: Append text - - table: Merge rows - - list: Merge items - - Other: Merge elements - """ - contentType = existingSection.get("content_type", "") - existingElements = existingSection.get("elements", []) - newElements = newSection.get("elements", []) - - if not newElements: - return existingSection - - # Handle list of elements - if isinstance(existingElements, list): - existingElem = existingElements[-1] if existingElements else {} - else: - existingElem = existingElements - - if isinstance(newElements, list): - newElem = newElements[0] if newElements else {} - else: - newElem = newElements - - if not isinstance(existingElem, dict) or not isinstance(newElem, dict): - return existingSection - - # Merge based on content type - if contentType == "code_block": - existingCode = existingElem.get("code", "") - newCode = newElem.get("code", "") - - if existingCode and newCode: - mergedCode = JsonResponseHandler.mergeCodeBlocks(existingCode, newCode, iteration) - existingElem["code"] = mergedCode - # Preserve language from existing or new - if "language" not in existingElem and "language" in newElem: - existingElem["language"] = newElem["language"] - - elif contentType in ["paragraph", "heading"]: - existingText = existingElem.get("text", "") - newText = newElem.get("text", "") - - if existingText and newText: - # Append text with space if needed - if existingText.rstrip() and not existingText.rstrip()[-1] in '.!?\n': - mergedText = existingText.rstrip() + " " + newText.lstrip() - else: - mergedText = existingText.rstrip() + "\n" + newText.lstrip() - existingElem["text"] = mergedText - - elif contentType == "table": - # Merge table rows with sophisticated overlap detection - # CRITICAL: Tables can have rows in two places: - # 1. Direct: existingElem["rows"] (legacy format) - # 2. Nested: existingElem["content"]["rows"] (current format) - existingRows = None - newRows = None - - # Check nested structure first (current format) - if "content" in existingElem and isinstance(existingElem["content"], dict): - existingRows = existingElem["content"].get("rows", []) - # Fallback to direct structure (legacy format) - if not existingRows: - existingRows = existingElem.get("rows", []) - - # Check nested structure first (current format) - if "content" in newElem and isinstance(newElem["content"], dict): - newRows = newElem["content"].get("rows", []) - # Fallback to direct structure (legacy format) - if not newRows: - newRows = newElem.get("rows", []) - - if existingRows and newRows: - # Use sophisticated overlap detection that handles multiple overlapping rows - mergedRows = JsonResponseHandler.mergeRowsWithOverlap(existingRows, newRows, iteration) - # Store in nested structure (current format) - if "content" not in existingElem: - existingElem["content"] = {} - existingElem["content"]["rows"] = mergedRows - # Also set type if missing - if "type" not in existingElem: - existingElem["type"] = "table" - logger.debug(f"Iteration {iteration}: Merged table rows - existing: {len(existingRows)}, new: {len(newRows)}, total: {len(mergedRows)}") - elif newRows: - # If existing has no rows but new does, use new rows - if "content" not in existingElem: - existingElem["content"] = {} - existingElem["content"]["rows"] = newRows - if "type" not in existingElem: - existingElem["type"] = "table" - # Preserve headers from existing (or use new if existing has none) - # Headers can be in content.headers or directly in element - existingHeaders = existingElem.get("content", {}).get("headers", []) if "content" in existingElem else existingElem.get("headers", []) - newHeaders = newElem.get("content", {}).get("headers", []) if "content" in newElem else newElem.get("headers", []) - if not existingHeaders and newHeaders: - if "content" not in existingElem: - existingElem["content"] = {} - existingElem["content"]["headers"] = newHeaders - # Preserve caption from existing (or use new if existing has none) - existingCaption = existingElem.get("content", {}).get("caption") if "content" in existingElem else existingElem.get("caption") - newCaption = newElem.get("content", {}).get("caption") if "content" in newElem else newElem.get("caption") - if not existingCaption and newCaption: - if "content" not in existingElem: - existingElem["content"] = {} - existingElem["content"]["caption"] = newCaption - - elif contentType in ["bullet_list", "numbered_list"]: - # Merge list items with sophisticated overlap detection - existingItems = existingElem.get("items", []) - newItems = newElem.get("items", []) - if existingItems and newItems: - mergedItems = JsonResponseHandler.mergeItemsWithOverlap(existingItems, newItems, iteration) - existingElem["items"] = mergedItems - elif newItems: - existingElem["items"] = newItems - - elif contentType == "image": - # Images are typically complete - if new image is provided, replace existing - # But check if existing image data is incomplete (e.g., base64 string cut off) - existingImageData = existingElem.get("base64Data", "") - newImageData = newElem.get("base64Data", "") - if existingImageData and newImageData: - # If existing image data doesn't end with valid base64 padding, it might be incomplete - # Base64 padding is '=' or '==' at the end - if not existingImageData.rstrip().endswith(('=', '==')): - # Existing image might be incomplete - merge by appending new data - # This handles cases where base64 string was cut off - existingElem["base64Data"] = existingImageData + newImageData - logger.debug(f"Iteration {iteration}: Merged incomplete image base64 data") - else: - # Existing image is complete - replace with new (or keep existing if new is empty) - if newImageData: - existingElem["base64Data"] = newImageData - elif newImageData: - existingElem["base64Data"] = newImageData - # Preserve other image metadata - if not existingElem.get("altText") and newElem.get("altText"): - existingElem["altText"] = newElem["altText"] - if not existingElem.get("caption") and newElem.get("caption"): - existingElem["caption"] = newElem["caption"] - - else: - # GENERIC FALLBACK: Use deep recursive merging for complex nested structures - # This handles any content type with arbitrary depth and complexity - merged_element = JsonResponseHandler.mergeDeepStructures( - existingElem, - newElem, - iteration, - f"section.{contentType}" - ) - existingElem = merged_element - - # Update section with merged content - mergedSection = existingSection.copy() - if isinstance(existingElements, list): - # Update the last element in the list with merged content - if existingElements: - existingElements[-1] = existingElem - mergedSection["elements"] = existingElements - else: - mergedSection["elements"] = existingElem - - # Preserve metadata from new section if missing in existing - if "order" not in mergedSection and "order" in newSection: - mergedSection["order"] = newSection["order"] - - return mergedSection - - @staticmethod - def mergeCodeBlocks(existingCode: str, newCode: str, iteration: int) -> str: - """ - Merge two code blocks intelligently, handling overlaps and incomplete lines. - """ - if not existingCode: - return newCode - if not newCode: - return existingCode - - existingLines = existingCode.rstrip().split('\n') - newLines = newCode.strip().split('\n') - - if not existingLines or not newLines: - return existingCode + "\n" + newCode - - lastExistingLine = existingLines[-1].strip() - firstNewLine = newLines[0].strip() - - # Strategy 1: Exact overlap - remove duplicate line - if lastExistingLine == firstNewLine: - newLines = newLines[1:] - logger.debug(f"Iteration {iteration}: Removed exact duplicate line in code merge") - - # Strategy 2: Incomplete line merge - # If last existing line ends with comma or is incomplete, merge with first new line - elif lastExistingLine.endswith(',') or (',' in lastExistingLine and len(lastExistingLine.split(',')[-1]) < 5): - # Last line is incomplete - merge with first new line - # Remove trailing comma from existing line - mergedLine = lastExistingLine.rstrip(',') + ',' + firstNewLine.lstrip() - existingLines[-1] = mergedLine - newLines = newLines[1:] - logger.debug(f"Iteration {iteration}: Merged incomplete line with continuation") - - # Strategy 3: Partial overlap detection - # Check if first new line starts with the end of last existing line - elif ',' in lastExistingLine and ',' in firstNewLine: - lastExistingParts = lastExistingLine.split(',') - firstNewParts = firstNewLine.split(',') - - # Check for overlap: if last part of existing matches first part of new - if lastExistingParts and firstNewParts: - lastExistingPart = lastExistingParts[-1].strip() - firstNewPart = firstNewParts[0].strip() - - # If they match, there's overlap - if lastExistingPart == firstNewPart and len(lastExistingParts) > 1: - # Remove overlapping part from new line - newLines[0] = ','.join(firstNewParts[1:]) - logger.debug(f"Iteration {iteration}: Removed partial overlap in code merge") - - # Reconstruct merged code - mergedCode = '\n'.join(existingLines) - if newLines: - if mergedCode and not mergedCode.endswith('\n'): - mergedCode += '\n' - mergedCode += '\n'.join(newLines) - - return mergedCode - - @staticmethod - def detectAndParseJsonFragment( - result: str, - allSections: List[Dict[str, Any]] - ) -> Optional[Dict[str, Any]]: - """ - GENERIC fragment detection for ANY JSON structure. - - Detects if response is a JSON fragment (continuation content) rather than full document structure. - Works for ANY JSON type: arrays, objects, primitives, nested structures of any depth/complexity. - - Fragment = Any JSON that: - 1. Does NOT have "documents" or "sections" keys (not full document structure) - 2. Can be ANY structure: array, object, nested, primitive, etc. - 3. Is continuation content that needs to be merged into existing sections - - Examples (all handled generically): - - Array: [["37643", ...], ...] (table rows, list items, any array) - - Object: {"rows": [...], "headers": [...]} (partial element) - - Primitive: "continuation text" (rare but possible) - - Nested: {"data": {"items": [...]}} (any nested structure) - - Returns fragment info dict with: - - fragment_data: The parsed fragment content (ANY type) - - target_section_id: ID of last incomplete section (generic, not type-specific) - - CRITICAL: Fully generic - no specific logic for tables, paragraphs, etc. - """ - try: - extracted = extractJsonString(result) - parsed = json.loads(extracted) - - # GENERIC fragment detection: Check if it's NOT a full document structure - is_full_document = False - if isinstance(parsed, dict): - # Full document structure has "documents" or "sections" keys - if "documents" in parsed or "sections" in parsed: - is_full_document = True - - # If it's a full document structure, it's not a fragment - if is_full_document: - return None - - # Otherwise, it's a fragment (can be ANY structure: array, object, primitive, nested) - # Find target: last incomplete section (generic, regardless of content type) - target_section_id = JsonResponseHandler.findLastIncompleteSectionId(allSections) - - logger.info(f"Detected GENERIC JSON fragment (type: {type(parsed).__name__}), target: {target_section_id}") - - return { - "fragment_data": parsed, # Can be ANY JSON structure - "target_section_id": target_section_id - } - - except Exception as e: - logger.error(f"Error detecting JSON fragment: {e}") - logger.debug(f"Fragment detection failed for result: {result[:500]}...") - - return None - - @staticmethod - def findLastIncompleteSectionId( - allSections: List[Dict[str, Any]] - ) -> Optional[str]: - """ - GENERIC: Find the last incomplete section (regardless of content type). - - This is fully generic - works for ANY content type, ANY structure. - Returns the ID of the last section that is incomplete, or None if all are complete. - """ - # Find the last incomplete section (generic, not type-specific) - for section in reversed(allSections): - if JsonResponseHandler.isSectionIncomplete(section): - return section.get("id") - # If no incomplete section found, return last section as fallback - if allSections: - return allSections[-1].get("id") - return None - - @staticmethod - def mergeFragmentIntoSection( - fragment: Dict[str, Any], - allSections: List[Dict[str, Any]], - iteration: int - ) -> Optional[List[Dict[str, Any]]]: - """ - GENERIC fragment merging for ANY JSON structure. - - Merges a JSON fragment (ANY structure: array, object, nested, primitive) into the last incomplete section. - Uses ONLY deep recursive merging - no specific logic for content types. - - Handles ALL cases: - 1. Fragments with overlap (detected and merged intelligently) - 2. Fragments without overlap (continuation after cut-off, appended) - 3. Any JSON structure (arrays, objects, nested, primitives) - 4. Accumulative merging (uses merged data from past iterations) - - CRITICAL: Fully generic - works for ANY JSON structure, ANY content type. - NO FALLBACKS: Returns None if merge fails (no target section found). - """ - fragment_data = fragment.get("fragment_data") - target_section_id = fragment.get("target_section_id") - - if fragment_data is None: - logger.error(f"Iteration {iteration}: ❌ Fragment has no fragment_data - merge FAILED") - return None - - # Find the target section (last incomplete section, generic) - target_section = None - target_index = -1 - - if target_section_id: - for i, section in enumerate(allSections): - if section.get("id") == target_section_id: - target_section = section - target_index = i - break - - # NO FALLBACKS: If target not found by ID, try to find incomplete section - if not target_section: - for i, section in enumerate(reversed(allSections)): - if JsonResponseHandler.isSectionIncomplete(section): - target_section = section - target_index = len(allSections) - 1 - i - break - - # NO FALLBACKS: If no target found, merge FAILS - if not target_section: - logger.error(f"Iteration {iteration}: ❌ MERGE FAILED - No target section found for fragment!") - logger.error(f"Iteration {iteration}: Available sections: {[s.get('id') + ' (' + s.get('content_type', 'unknown') + ')' for s in allSections]}") - return None - - # Get the last element from target section (where fragment will be merged) - merged_section = target_section.copy() - elements = merged_section.get("elements", []) - - if not isinstance(elements, list): - elements = [elements] if elements else [] - - if not elements: - elements = [{}] - - last_element = elements[-1] if elements else {} - if not isinstance(last_element, dict): - last_element = {} - elements.append(last_element) - - # CRITICAL: GENERIC fragment merging for ALL structure types - # Automatically detects the structure type and merges accordingly - # Works for: tables, lists, code blocks, paragraphs, images, and any nested structures - merged_element = JsonResponseHandler._mergeFragmentIntoElement( - last_element, - fragment_data, - target_section, - iteration, - f"section.{target_section_id}.fragment" - ) - - # Update elements with merged content - elements[-1] = merged_element - merged_section["elements"] = elements - - # Update allSections (this ensures accumulative merging - merged data is used for next iteration) - merged_sections = allSections.copy() - merged_sections[target_index] = merged_section - - logger.info(f"Iteration {iteration}: ✅ Merged GENERIC fragment (type: {type(fragment_data).__name__}) into section '{target_section_id}'") - - # Log merged JSON for debugging - try: - from modules.shared.debugLogger import writeDebugFile - merged_json_str = json.dumps(merged_sections, indent=2, ensure_ascii=False) - writeDebugFile(merged_json_str, f"merged_json_iteration_{iteration}.json") - except Exception as e: - logger.debug(f"Iteration {iteration}: Failed to write merged JSON debug file: {e}") - - return merged_sections - - @staticmethod - def completeIncompleteStructures(allSections: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Complete any incomplete structures in sections by ensuring proper JSON structure. - - This ensures JSON is properly closed even if merge failed or iterations stopped early. - Works generically for ANY structure type - recursively processes all nested structures. - - Returns sections with completed structures. - """ - completed_sections = [] - for section in allSections: - completed_section = JsonResponseHandler._completeStructure(section) - completed_sections.append(completed_section) - return completed_sections - - @staticmethod - def _completeStructure(structure: Any) -> Any: - """ - Recursively complete incomplete structures by ensuring arrays/objects are properly structured. - Works generically for ANY JSON structure - no specific logic for content types. - """ - if isinstance(structure, dict): - completed = {} - for key, value in structure.items(): - completed[key] = JsonResponseHandler._completeStructure(value) - return completed - elif isinstance(structure, list): - completed = [] - for item in structure: - completed.append(JsonResponseHandler._completeStructure(item)) - return completed - else: - # Primitive value - return as is - return structure - - @staticmethod - def getContentTypeForFragment(fragment_type: str) -> str: - """Map fragment type to content type.""" - mapping = { - "table_rows": "table", - "table_element": "table", - "code_lines": "code_block", - "code_element": "code_block", - "list_items": "bullet_list" - } - return mapping.get(fragment_type, "paragraph") - - @staticmethod - def deepCompare(obj1: Any, obj2: Any, max_depth: int = 10) -> bool: - """ - Deep recursive comparison of two JSON-serializable objects. - Handles nested structures of any depth and complexity. - - Args: - obj1: First object to compare - obj2: Second object to compare - max_depth: Maximum recursion depth to prevent infinite loops - - Returns: - True if objects are deeply equal, False otherwise - """ - if max_depth <= 0: - return False - - # Type check - if type(obj1) != type(obj2): - return False - - # Primitive types - if isinstance(obj1, (str, int, float, bool, type(None))): - return obj1 == obj2 - - # Lists/arrays - compare element by element - if isinstance(obj1, list): - if len(obj1) != len(obj2): - return False - return all(JsonResponseHandler.deepCompare(item1, item2, max_depth - 1) - for item1, item2 in zip(obj1, obj2)) - - # Dicts/objects - compare key by key - if isinstance(obj1, dict): - if set(obj1.keys()) != set(obj2.keys()): - return False - return all(JsonResponseHandler.deepCompare(obj1[key], obj2[key], max_depth - 1) - for key in obj1.keys()) - - # Fallback for other types - return obj1 == obj2 - - @staticmethod - def findLongestCommonSuffix( - existing_list: List[Any], - new_list: List[Any], - min_overlap: int = 1 - ) -> int: - """ - Find the longest common suffix of existing_list that matches a prefix of new_list. - - This handles cases where multiple elements overlap: - - existing: [A, B, C, D] - - new: [C, D, E, F] - - overlap: [C, D] (length 2) - - Returns the length of the overlap (0 if no overlap found). - """ - if not existing_list or not new_list: - return 0 - - max_overlap = min(len(existing_list), len(new_list)) - - # Try all possible overlap lengths (from longest to shortest) - for overlap_len in range(max_overlap, min_overlap - 1, -1): - existing_suffix = existing_list[-overlap_len:] - new_prefix = new_list[:overlap_len] - - # Deep compare suffix and prefix - if all(JsonResponseHandler.deepCompare(existing_suffix[i], new_prefix[i]) - for i in range(overlap_len)): - return overlap_len - - return 0 - - @staticmethod - def findPartialOverlap( - existing_item: Any, - new_item: Any - ) -> Tuple[bool, Optional[Any]]: - """ - Detect if new_item completes an incomplete existing_item. - - Handles cases like: - - existing: ["37643", "37649", "37657", "37663", "37691", "37693", "37699", "37717", "37747", "376"] - - new: ["37643", "37649", ...] - - Returns (is_partial_overlap, merged_item) if partial overlap detected, else (False, None). - """ - # Check if both are lists - if isinstance(existing_item, list) and isinstance(new_item, list): - if not existing_item or not new_item: - return False, None - - # Check if last element of existing is incomplete and matches first of new - last_existing = existing_item[-1] - first_new = new_item[0] - - # If last existing is a string and first new is a string - if isinstance(last_existing, str) and isinstance(first_new, str): - # Check if last existing is incomplete (very short, ends with number, etc.) - if len(last_existing) < 10 and first_new.startswith(last_existing): - # Partial overlap - merge them - merged_last = last_existing + first_new[len(last_existing):] - merged_item = existing_item[:-1] + [merged_last] + new_item[1:] - return True, merged_item - - # Check if last existing is incomplete list and first new completes it - if isinstance(last_existing, list) and isinstance(first_new, list): - if len(last_existing) < len(first_new): - # Check if last existing is prefix of first new - if first_new[:len(last_existing)] == last_existing: - # Merge: replace incomplete last with complete first - merged_item = existing_item[:-1] + [first_new] + new_item[1:] - return True, merged_item - - # Check if existing is incomplete string and new completes it - if isinstance(existing_item, str) and isinstance(new_item, str): - if len(existing_item) < 50 and new_item.startswith(existing_item): - # Partial overlap - merged = existing_item + new_item[len(existing_item):] - return True, merged - - return False, None - - @staticmethod - def mergeRowsWithOverlap( - existing_rows: List[List[str]], - new_rows: List[List[str]], - iteration: int - ) -> List[List[str]]: - """ - Merge table rows with sophisticated overlap detection. - Handles multiple overlapping rows and partial overlaps. - """ - if not new_rows: - return existing_rows - if not existing_rows: - return new_rows - - # Strategy 1: Find longest common suffix/prefix overlap - overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing_rows, new_rows, min_overlap=1) - if overlap_len > 0: - logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping table rows, removing duplicates") - return existing_rows + new_rows[overlap_len:] - - # Strategy 2: Check for partial overlap in last row - if len(existing_rows) > 0 and len(new_rows) > 0: - last_existing = existing_rows[-1] - first_new = new_rows[0] - - is_partial, merged_row = JsonResponseHandler.findPartialOverlap(last_existing, first_new) - if is_partial: - logger.debug(f"Iteration {iteration}: Found partial overlap in table rows, merging") - return existing_rows[:-1] + [merged_row] + new_rows[1:] - - # Strategy 3: Simple first/last comparison (fallback) - if isinstance(existing_rows[-1], list) and isinstance(new_rows[0], list): - if list(existing_rows[-1]) == list(new_rows[0]): - logger.debug(f"Iteration {iteration}: Removed duplicate table row (exact match)") - return existing_rows + new_rows[1:] - - # No overlap detected - append all new rows - return existing_rows + new_rows - - @staticmethod - def mergeItemsWithOverlap( - existing_items: List[str], - new_items: List[str], - iteration: int - ) -> List[str]: - """ - Merge list items with sophisticated overlap detection. - Handles multiple overlapping items and partial overlaps. - """ - if not new_items: - return existing_items - if not existing_items: - return new_items - - # Strategy 1: Find longest common suffix/prefix overlap - overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing_items, new_items, min_overlap=1) - if overlap_len > 0: - logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping list items, removing duplicates") - return existing_items + new_items[overlap_len:] - - # Strategy 2: Check for partial overlap in last item - if len(existing_items) > 0 and len(new_items) > 0: - is_partial, merged_item = JsonResponseHandler.findPartialOverlap(existing_items[-1], new_items[0]) - if is_partial: - logger.debug(f"Iteration {iteration}: Found partial overlap in list items, merging") - return existing_items[:-1] + [merged_item] + new_items[1:] - - # Strategy 3: Simple first/last comparison (fallback) - if existing_items[-1] == new_items[0]: - logger.debug(f"Iteration {iteration}: Removed duplicate list item (exact match)") - return existing_items + new_items[1:] - - # No overlap detected - append all new items - return existing_items + new_items - - @staticmethod - def mergeDeepStructures( - existing: Any, - new: Any, - iteration: int, - path: str = "root" - ) -> Any: - """ - FULLY GENERIC recursive merge for ANY JSON structure of arbitrary depth/complexity. - - Handles ALL cases generically: - 1. Arrays/Lists: Overlap detection (suffix/prefix), partial overlap, no overlap (continuation) - 2. Objects/Dicts: Key-by-key merge with overlap detection for nested structures - 3. Primitives: Equality check, replacement if different - 4. Nested structures: Recursively handles any depth/complexity - - Overlap detection strategies (all generic): - - Array overlap: Finds longest common suffix/prefix, handles partial overlaps - - Object overlap: Detected recursively through key matching and deep comparison - - No overlap: Appends/merges continuation content after cut-off point - - CRITICAL: Fully generic - no specific logic for content types. - Works for ANY JSON structure: arrays, objects, nested, primitives, any combination. - """ - # Type check - if type(existing) != type(new): - # Types don't match - return new (replacement) - logger.debug(f"Iteration {iteration}: Types don't match at {path} ({type(existing).__name__} vs {type(new).__name__}), replacing") - return new - - # Lists/arrays - GENERIC merge with overlap detection - if isinstance(existing, list) and isinstance(new, list): - if not new: - return existing - if not existing: - return new - - # Strategy 1: Find longest common suffix/prefix overlap (handles multiple overlapping elements) - overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing, new, min_overlap=1) - if overlap_len > 0: - logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping elements at {path}, removing duplicates") - return existing + new[overlap_len:] - - # Strategy 2: Check for partial overlap in last element (incomplete element completion) - if len(existing) > 0 and len(new) > 0: - is_partial, merged_item = JsonResponseHandler.findPartialOverlap(existing[-1], new[0]) - if is_partial: - logger.debug(f"Iteration {iteration}: Found partial overlap at {path}, merging incomplete element") - return existing[:-1] + [merged_item] + new[1:] - - # Strategy 3: No overlap detected - continuation after cut-off point - # This handles the case where new data starts exactly after the cut-off - logger.debug(f"Iteration {iteration}: No overlap at {path}, appending continuation content ({len(new)} items)") - return existing + new - - # Dicts/objects - GENERIC merge with recursive overlap detection - if isinstance(existing, dict) and isinstance(new, dict): - merged = existing.copy() - - # Check for object-level overlap: if new object is subset/superset of existing - # This handles cases where same object structure appears in both - existing_keys = set(existing.keys()) - new_keys = set(new.keys()) - - # If new is subset of existing and values match, it's overlap (skip) - if new_keys.issubset(existing_keys): - all_match = True - for key in new_keys: - if not JsonResponseHandler.deepCompare(existing[key], new[key]): - all_match = False - break - if all_match: - logger.debug(f"Iteration {iteration}: Object at {path} is subset overlap, skipping") - return existing - - # Merge key-by-key with recursive overlap detection - for key, new_value in new.items(): - if key in merged: - # Key exists - merge recursively (handles nested overlap detection) - merged[key] = JsonResponseHandler.mergeDeepStructures( - merged[key], - new_value, - iteration, - f"{path}.{key}" - ) - else: - # New key - add it (continuation content) - merged[key] = new_value - logger.debug(f"Iteration {iteration}: Added new key '{key}' at {path} (continuation)") - - return merged - - # Primitives - equality check - if existing == new: - return existing - # Different primitive values - return new (continuation/replacement) - logger.debug(f"Iteration {iteration}: Primitive at {path} differs, using new value") - return new - - @staticmethod - def _mergeFragmentIntoElement( - last_element: Dict[str, Any], - fragment_data: Any, - target_section: Dict[str, Any], - iteration: int, - path: str - ) -> Dict[str, Any]: - """ - GENERIC fragment merging for ALL structure types. - - Automatically detects the structure type and merges fragments accordingly. - Works for: tables, lists, code blocks, paragraphs, images, and any nested structures. - - Strategy: - 1. Analyze last_element structure to determine content location (content.rows, content.items, etc.) - 2. Detect fragment type (array, object, primitive) - 3. Merge fragment into appropriate location using mergeDeepStructures - - Args: - last_element: The existing element to merge into - fragment_data: The fragment data to merge (can be any JSON structure) - target_section: The target section (for content_type detection) - iteration: Current iteration number - path: Path for logging - - Returns: - Merged element - """ - contentType = target_section.get("content_type", "") - elementType = last_element.get("type", "") - - # Determine the content structure path based on element type and content type - # This handles both nested (content.rows) and flat (rows) structures - contentPath = None - fragmentIsArray = isinstance(fragment_data, list) and len(fragment_data) > 0 - - # Detect structure type and determine merge path - if contentType == "table" or elementType == "table": - # Tables: merge into content.rows or rows - if "content" in last_element and isinstance(last_element["content"], dict): - contentPath = "content.rows" - else: - contentPath = "rows" - elif contentType in ["bullet_list", "numbered_list", "list"] or elementType in ["bullet_list", "numbered_list", "list"]: - # Lists: merge into content.items or items - if "content" in last_element and isinstance(last_element["content"], dict): - contentPath = "content.items" - else: - contentPath = "items" - elif contentType == "code_block" or elementType == "code_block": - # Code blocks: merge into content.code or code - if "content" in last_element and isinstance(last_element["content"], dict): - contentPath = "content.code" - else: - contentPath = "code" - elif contentType in ["paragraph", "heading"] or elementType in ["paragraph", "heading"]: - # Text: merge into content.text or text - if "content" in last_element and isinstance(last_element["content"], dict): - contentPath = "content.text" - else: - contentPath = "text" - elif contentType == "image" or elementType == "image": - # Images: merge into base64Data - contentPath = "base64Data" - - # If we have a specific content path, merge into that location - if contentPath: - # Split path (e.g., "content.rows" -> ["content", "rows"]) - pathParts = contentPath.split(".") - - # Ensure nested structure exists - current = last_element - for i, part in enumerate(pathParts[:-1]): - if part not in current: - current[part] = {} - elif not isinstance(current[part], dict): - current[part] = {} - current = current[part] - - # Get existing content at target path - targetKey = pathParts[-1] - existingContent = current.get(targetKey, []) - - # Merge fragment into existing content - # CRITICAL: Handle both array fragments and object fragments generically - if fragmentIsArray: - # Fragment is an array - merge arrays - if isinstance(existingContent, list): - # Check if fragment is array of arrays (e.g., table rows) or array of primitives - if len(fragment_data) > 0 and isinstance(fragment_data[0], list): - # Array of arrays - use rows merge for tables, generic merge for others - if contentPath.endswith(".rows"): - mergedContent = JsonResponseHandler.mergeRowsWithOverlap(existingContent, fragment_data, iteration) - else: - # Generic array-of-arrays merge - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent, - fragment_data, - iteration, - f"{path}.{targetKey}" - ) - else: - # Array of primitives - use items merge for lists, generic merge for others - if contentPath.endswith(".items"): - mergedContent = JsonResponseHandler.mergeItemsWithOverlap(existingContent, fragment_data, iteration) - else: - # Generic array merge using mergeDeepStructures - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent, - fragment_data, - iteration, - f"{path}.{targetKey}" - ) - else: - # Existing content is not a list - replace with fragment - mergedContent = fragment_data - elif isinstance(fragment_data, dict): - # Fragment is an object - check if it contains nested content (e.g., {"content": {"rows": [...]}}) - # If fragment has same structure as target, merge nested content - if "content" in fragment_data and isinstance(fragment_data["content"], dict): - fragmentNested = fragment_data["content"] - # Check if fragment has the same key as our target (e.g., fragment.content.rows) - if targetKey in fragmentNested: - # Fragment has nested content matching our target - merge that content - fragmentNestedContent = fragmentNested[targetKey] - if isinstance(existingContent, list) and isinstance(fragmentNestedContent, list): - # Both are lists - merge them - if contentPath.endswith(".rows"): - mergedContent = JsonResponseHandler.mergeRowsWithOverlap(existingContent, fragmentNestedContent, iteration) - elif contentPath.endswith(".items"): - mergedContent = JsonResponseHandler.mergeItemsWithOverlap(existingContent, fragmentNestedContent, iteration) - else: - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent, - fragmentNestedContent, - iteration, - f"{path}.{targetKey}" - ) - else: - # Use deep merge for nested content - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent if existingContent else {}, - fragmentNestedContent, - iteration, - f"{path}.{targetKey}" - ) - else: - # Fragment has different structure - merge entire fragment object - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent if existingContent else {}, - fragment_data, - iteration, - f"{path}.{targetKey}" - ) - else: - # Fragment is a simple object - use deep merge - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent if existingContent else {}, - fragment_data, - iteration, - f"{path}.{targetKey}" - ) - else: - # Fragment is a primitive or unknown type - use deep merge - mergedContent = JsonResponseHandler.mergeDeepStructures( - existingContent if existingContent else {}, - fragment_data, - iteration, - f"{path}.{targetKey}" - ) - - # Update the merged content - current[targetKey] = mergedContent - - # Ensure type is set - if elementType and "type" not in last_element: - last_element["type"] = elementType - elif contentType and "type" not in last_element: - last_element["type"] = contentType - - logger.info(f"Iteration {iteration}: ✅ Merged fragment into {contentPath} for section '{target_section.get('id')}'") - return last_element - - # No specific content path - use generic deep merge - # This handles any structure type generically - merged_element = JsonResponseHandler.mergeDeepStructures( - last_element, - fragment_data, - iteration, - path - ) - - logger.info(f"Iteration {iteration}: ✅ Merged GENERIC fragment (type: {type(fragment_data).__name__}) into section '{target_section.get('id')}'") - return merged_element - - @staticmethod - def cleanEncodingIssues(jsonString: str) -> str: - """ - GENERIC function to remove problematic encoding parts from JSON string. - - Works for ANY JSON structure - removes problematic characters/bytes. - - Args: - jsonString: JSON string that may have encoding issues - - Returns: - Cleaned JSON string - """ - try: - # Try to decode/encode to detect issues - jsonString.encode('utf-8').decode('utf-8') - return jsonString - except UnicodeError: - # Remove problematic parts - cleaned = jsonString.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') - logger.warning("Removed encoding issues from JSON string") - return cleaned - - @staticmethod - def mergeJsonStringsWithOverlap( - accumulated: str, - newFragment: str - ) -> Tuple[str, bool]: - """ - Merge JSON fragments intelligently using modular parser. - - Uses the new ModularJsonMerger for clean, robust merging. - Falls back to legacy code only if new merger fails completely. - - Args: - accumulated: Previously accumulated JSON string (may be incomplete/fragmented) - newFragment: New fragment string to append (may be incomplete/fragmented) - - Returns: - Tuple of (merged_json_string, has_overlap): - - merged_json_string: Combined JSON string with fragments properly merged - - has_overlap: True if overlap was found (iterations should continue), False if no overlap (iterations should stop) - """ - if not accumulated: - result = newFragment if newFragment else "{}" - return (result, False) # No overlap if no accumulated data - if not newFragment: - return (accumulated, False) # No overlap if no new fragment - - # Use new modular merger - try: - from .subJsonMerger import ModularJsonMerger - result, hasOverlap = ModularJsonMerger.merge(accumulated, newFragment) - # IMPORTANT: ModularJsonMerger returns unclosed JSON if overlap found (with incomplete element at end) - # If no overlap, returns closed JSON (iterations should stop) - if result and result.strip() and result.strip() != "{}": - # Return result with overlap flag - return (result, hasOverlap) - except Exception as e: - logger.debug(f"Modular merger failed, using fallback: {e}") - - # Fallback to legacy merger (simplified) - - accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() - newFragmentExtracted = stripCodeFences(normalizeJsonText(newFragment)).strip() - - # Try simple string merge with repair - try: - # Close structures - accClosed = closeJsonStructures(accumulatedExtracted) if accumulatedExtracted else "{}" - fragClosed = closeJsonStructures(newFragmentExtracted) if newFragmentExtracted else "{}" - - # Try to parse both - accParsed, accErr, _ = tryParseJson(accClosed) - fragParsed, fragErr, _ = tryParseJson(fragClosed) - - # If both parse, merge structurally - if accErr is None and fragErr is None: - merged = JsonResponseHandler._mergeParsedJson(accParsed, fragParsed) - if merged: - result = json.dumps(merged, indent=2, ensure_ascii=False) - return (result, False) # No overlap in fallback - close and stop - - # If only accumulated parses, return it - if accErr is None and accParsed: - result = json.dumps(accParsed, indent=2, ensure_ascii=False) - return (result, False) # No overlap - close and stop - except Exception: - pass - - # Last resort: return accumulated (at least we have that) - close it - if accumulatedExtracted: - try: - closed = closeJsonStructures(accumulatedExtracted) - return (closed, False) # No overlap - close and stop - except Exception: - return (accumulatedExtracted, False) # No overlap - return as-is - - result = accumulated if accumulated else "{}" - return (result, False) # No overlap - return as-is - - @staticmethod - def _mergeParsedJson(accParsed: Any, fragParsed: Any) -> Optional[Dict[str, Any]]: - """Simple merge of two parsed JSON objects.""" - if isinstance(accParsed, dict) and isinstance(fragParsed, dict): - # Merge dicts - merged = accParsed.copy() - - # Merge elements if both have them - if "elements" in accParsed and "elements" in fragParsed: - accElements = accParsed.get("elements", []) - fragElements = fragParsed.get("elements", []) - # Simple merge - append new elements - merged["elements"] = accElements + fragElements - elif "elements" in fragParsed: - merged["elements"] = fragParsed["elements"] - - # Merge other keys - for key, value in fragParsed.items(): - if key != "elements": - if key in merged and isinstance(merged[key], list) and isinstance(value, list): - merged[key] = merged[key] + value - else: - merged[key] = value - - return merged - - return None - - @staticmethod - def _normalizeToElementsStructure( - jsonString: str, - originalString: str - ) -> Optional[Dict[str, Any]]: - """ - Normalize any JSON structure (Dict, List, None, or parse error) to {"elements": [...]} format. - - Handles: - - Dict with "elements" → return as-is - - Dict without "elements" but with "type" → wrap in elements array - - List → wrap in elements structure - - Parse error → try repairBrokenJson - - None → return None - - Args: - jsonString: Extracted JSON string - originalString: Original string (for context) - - Returns: - Normalized Dict with "elements" array, or None if normalization fails - """ - if not jsonString: - return None - - - # Try to parse directly first - try: - parsed = json.loads(jsonString) - parseErr = None - except Exception as e: - parseErr = e - parsed = None - - # If parsing failed, try closing structures first (for incomplete fragments) - if parseErr is not None: - try: - closed = closeJsonStructures(jsonString) - parsed = json.loads(closed) - parseErr = None - except Exception: - pass - - # If still failed, try repairBrokenJson ONLY if it looks like document structure - # For other structures (like section_content), use fragment detection instead - if parseErr is not None: - # Check if this looks like a document structure (has "documents" or "sections") - isDocumentStructure = '"documents"' in jsonString or '"sections"' in jsonString - - if isDocumentStructure: - # Use repairBrokenJson for document structures - repaired = repairBrokenJson(jsonString) - if repaired: - parsed = repaired - parseErr = None - else: - # Still can't parse - try to detect fragment structure - return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) - else: - # For non-document structures, skip repairBrokenJson and go straight to fragment detection - # repairBrokenJson tries to extract "sections" which doesn't work for other structures - return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) - - # Normalize based on type - if parsed is None: - return None - elif isinstance(parsed, dict): - # Already a dict - if "elements" in parsed: - return parsed - elif "type" in parsed: - # Single element - wrap in elements array - return {"elements": [parsed]} - else: - # Unknown dict structure - try to extract elements - return JsonResponseHandler._extractElementsFromDict(parsed) - elif isinstance(parsed, list): - # List - check if it's a list of elements or a fragment - if parsed and isinstance(parsed[0], dict) and "type" in parsed[0]: - # List of elements - return {"elements": parsed} - else: - # Fragment list (e.g., array of rows) - detect structure - return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) - else: - # Primitive type - can't normalize - return None - - @staticmethod - def _detectAndNormalizeFragment( - jsonString: str, - originalString: str - ) -> Optional[Dict[str, Any]]: - """ - Detect fragment structure and normalize it. - - Fragments can be: - - Array of arrays (table rows): `[["row1"], ["row2"]]` or `["1947", "16883"], ["1948", "16889"]` - - Array of strings (list items): `["item1", "item2"]` - - Incomplete structure: `["item1", "item2", ` (ends with comma) - - Partial object: `{"type": "table", "content": {"rows": [["1947"...` (cut mid-string) - - Returns normalized structure or None if detection fails. - """ - jsonStripped = jsonString.strip() - - # Strategy 1: Check if it's an array fragment - if jsonStripped.startswith('['): - # Try to parse as array - - # Close incomplete structures - closed = closeJsonStructures(jsonStripped) - parsed, parseErr, _ = tryParseJson(closed) - - if parseErr is None and isinstance(parsed, list): - # Check structure: array of arrays (table rows) or array of strings (list items) - if parsed and isinstance(parsed[0], list): - # Array of arrays - likely table rows fragment - return { - "elements": [{ - "type": "table", - "content": { - "rows": parsed - } - }] - } - elif parsed and isinstance(parsed[0], str): - # Array of strings - likely list items fragment - return { - "elements": [{ - "type": "bullet_list", - "content": { - "items": parsed - } - }] - } - elif parseErr is not None: - # Can't parse - try regex extraction for table rows - rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) - if rows: - return { - "elements": [{ - "type": "table", - "content": { - "rows": rows - } - }] - } - - # Strategy 2: Check if it's a partial object (cut mid-structure) - # Look for patterns like: {"elements": [...] or {"type": "table"... - if jsonStripped.startswith('{'): - - # Try to close and parse - closed = closeJsonStructures(jsonStripped) - parsed, parseErr, _ = tryParseJson(closed) - - if parseErr is None and isinstance(parsed, dict): - # Successfully parsed - normalize it - return JsonResponseHandler._normalizeToElementsStructure(closed, originalString) - elif parseErr is not None: - # Can't parse - try to extract table rows from the raw string - # This handles cases like: {"elements": [{"type": "table", "content": {"rows": [["1947"... - rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) - if rows: - return { - "elements": [{ - "type": "table", - "content": { - "rows": rows - } - }] - } - - # Try to extract any array patterns that might be table rows - # Look for patterns like: ["1947", "10000"], ["1948", "10100"] - import re - # Pattern: ["value1", "value2"], ["value3", "value4"] - rowPattern = r'\["([^"]*)",\s*"([^"]*)"\]' - matches = re.findall(rowPattern, jsonStripped) - if matches and len(matches) >= 2: - # Found multiple row patterns - likely table rows - rows = [[match[0], match[1]] for match in matches] - return { - "elements": [{ - "type": "table", - "content": { - "rows": rows - } - }] - } - - # Strategy 3: Try to extract rows from any text (even if not starting with [ or {) - rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) - if rows: - return { - "elements": [{ - "type": "table", - "content": { - "rows": rows - } - }] - } - - return None - - @staticmethod - def _extractElementsFromDict(d: Dict[str, Any]) -> Dict[str, Any]: - """ - Try to extract elements from unknown dict structure. - Returns normalized structure or empty elements array. - """ - # Check common patterns - if "sections" in d: - # Document structure with sections - sections = d.get("sections", []) - elements = [] - for section in sections: - if isinstance(section, dict) and "elements" in section: - elements.extend(section.get("elements", [])) - return {"elements": elements} - - # Unknown structure - return empty - return {"elements": []} - - @staticmethod - def _mergeJsonStructuresGeneric( - accumulatedObj: Dict[str, Any], - newFragmentObj: Dict[str, Any], - accumulatedRaw: str, - newFragmentRaw: str, - overlapElements: Optional[List[Dict[str, Any]]] = None - ) -> Optional[Dict[str, Any]]: - """ - GENERIC merge of two JSON structures, handling overlaps and missing parts. - - Strategy: - 1. Extract elements from both structures (both are normalized to {"elements": [...]}) - 2. Use overlap elements if provided to identify merge point - 3. Detect if both have same structure (same content type) - 4. Group elements by type - 5. Merge elements of same type using content-type-specific logic with overlap detection - 6. Handle overlaps and missing parts intelligently - - Args: - accumulatedObj: Normalized accumulated JSON object (guaranteed to have "elements") - newFragmentObj: Normalized new fragment JSON object (guaranteed to have "elements") - accumulatedRaw: Raw accumulated string (for fragment detection) - newFragmentRaw: Raw new fragment string (for fragment detection) - overlapElements: Optional list of overlap elements from continuation response - - Returns: - Merged JSON object or None if merging fails - """ - try: - # Step 1: Extract elements (both are normalized, so this should always work) - accumulatedElements = accumulatedObj.get("elements", []) if isinstance(accumulatedObj, dict) else [] - newFragmentElements = newFragmentObj.get("elements", []) if isinstance(newFragmentObj, dict) else [] - - if not accumulatedElements and not newFragmentElements: - # No elements found - try to extract from raw strings - # Try to extract any valid JSON structure from raw strings - - # Try accumulated first - if accumulatedRaw: - try: - closedAccumulated = closeJsonStructures(accumulatedRaw) - parsed, parseErr, _ = tryParseJson(closedAccumulated) - if parseErr is None and parsed: - normalized = JsonResponseHandler._normalizeToElementsStructure(closedAccumulated, accumulatedRaw) - if normalized: - return normalized - except Exception: - pass - - # Try new fragment - if newFragmentRaw: - try: - closedFragment = closeJsonStructures(newFragmentRaw) - parsed, parseErr, _ = tryParseJson(closedFragment) - if parseErr is None and parsed: - normalized = JsonResponseHandler._normalizeToElementsStructure(closedFragment, newFragmentRaw) - if normalized: - return normalized - except Exception: - pass - - # If still nothing, return empty structure (never None) - return {"elements": []} - - # Step 2: Use overlap elements to identify merge point - # If overlap elements are provided, use them to find where to merge - if overlapElements and isinstance(overlapElements, list) and len(overlapElements) > 0: - # Find overlap in accumulated elements - overlapStartIndex = JsonResponseHandler._findOverlapStartIndex(accumulatedElements, overlapElements) - if overlapStartIndex >= 0: - # Remove overlapping elements from accumulated (they'll be replaced by continuation) - accumulatedElements = accumulatedElements[:overlapStartIndex] - logger.debug(f"Found overlap at index {overlapStartIndex}, removed {len(accumulatedElements) - overlapStartIndex} overlapping elements") - - # Step 3: Detect if newFragment is a continuation fragment - # Check if newFragment starts with array elements (fragment, not full JSON) - isFragment = JsonResponseHandler._isFragment(newFragmentRaw, newFragmentElements) - - # Step 4: Group elements by type for intelligent merging - accumulatedByType = {} - for elem in accumulatedElements: - if isinstance(elem, dict): - elemType = elem.get("type", "unknown") - if elemType not in accumulatedByType: - accumulatedByType[elemType] = [] - accumulatedByType[elemType].append(elem) - - newFragmentByType = {} - for elem in newFragmentElements: - if isinstance(elem, dict): - elemType = elem.get("type", "unknown") - if elemType not in newFragmentByType: - newFragmentByType[elemType] = [] - newFragmentByType[elemType].append(elem) - - # Step 5: Merge elements intelligently - mergedElements = [] - allTypes = set(accumulatedByType.keys()) | set(newFragmentByType.keys()) - - for elemType in allTypes: - accElems = accumulatedByType.get(elemType, []) - fragElems = newFragmentByType.get(elemType, []) - - if not accElems: - # Only in fragment - add all - mergedElements.extend(fragElems) - elif not fragElems: - # Only in accumulated - add all - mergedElements.extend(accElems) - else: - # Both have elements of this type - merge them using content-type-specific logic - mergedElem = JsonResponseHandler._mergeElementsOfSameTypeGeneric( - accElems[0], fragElems[0], elemType, accumulatedRaw, newFragmentRaw, isFragment - ) - if mergedElem: - mergedElements.append(mergedElem) - - # Step 6: Reconstruct base structure - if mergedElements: - return {"elements": mergedElements} - else: - # No merged elements - return accumulated if available (NEVER return None) - if accumulatedElements: - return {"elements": accumulatedElements} - # If no accumulated, return new fragment if available - if newFragmentElements: - return {"elements": newFragmentElements} - # Last resort: return empty structure (never None) - return {"elements": []} - - except Exception as e: - logger.debug(f"Structure-based merge failed: {e}") - import traceback - logger.debug(traceback.format_exc()) - return None - - @staticmethod - def _isFragment(jsonString: str, elements: List[Dict[str, Any]]) -> bool: - """ - Detect if JSON string is a fragment (not a complete JSON object). - - Fragments: - - Start with `[` but not `[{"` (array fragment, not full elements array) - - Start with array elements like `["cell1", "cell2"],` (table rows fragment) - - Don't have full structure (missing outer object with "elements") - - Are continuations of previous structure - """ - jsonStripped = jsonString.strip() - - # Check if it starts with array (fragment) - if jsonStripped.startswith('['): - # Check if it's a full elements array `[{"type": ...}]` or a fragment `["cell1", "cell2"]` - if jsonStripped.startswith('[{"') or jsonStripped.startswith('[{'): - # Could be full structure - check if it has "type" field - if elements and isinstance(elements[0], dict) and "type" in elements[0]: - return False # Full structure - # Otherwise it's a fragment (array of primitives or incomplete) - return True - - # Check if it starts with object but missing "elements" wrapper - if jsonStripped.startswith('{'): - # Check if it has "elements" field - if '"elements"' not in jsonStripped[:200]: # Check first 200 chars - # Might be a single element fragment - return True - - # Check if elements are incomplete (no full structure) - if elements and isinstance(elements[0], dict): - # Check if first element is missing required fields - firstElem = elements[0] - if "type" not in firstElem and "content" not in firstElem: - return True - - return False - - @staticmethod - def _mergeElementsOfSameTypeGeneric( - accumulatedElem: Dict[str, Any], - newFragmentElem: Dict[str, Any], - elemType: str, - accumulatedRaw: str, - newFragmentRaw: str, - isFragment: bool - ) -> Optional[Dict[str, Any]]: - """ - GENERIC merge of two elements of the same type, with content-type-specific optimizations. - - Content-type-specific merging: - - table: Merge rows arrays with overlap detection - - paragraph: Merge text content - - code_block: Merge code strings - - bullet_list/numbered_list: Merge items arrays - - heading: Use new fragment (usually complete) - - image: Use new fragment (usually complete) - - Other: Generic deep merge - - Args: - accumulatedElem: Accumulated element - newFragmentElem: New fragment element - elemType: Content type (table, paragraph, etc.) - accumulatedRaw: Raw accumulated string - newFragmentRaw: Raw new fragment string - isFragment: Whether newFragment is a fragment (continuation) - - Returns: - Merged element or None if merging fails - """ - if elemType == "table": - return JsonResponseHandler._mergeTableElementsGeneric( - accumulatedElem, newFragmentElem, accumulatedRaw, newFragmentRaw, isFragment - ) - elif elemType == "paragraph": - return JsonResponseHandler._mergeParagraphElements( - accumulatedElem, newFragmentElem, isFragment - ) - elif elemType == "code_block": - return JsonResponseHandler._mergeCodeBlockElements( - accumulatedElem, newFragmentElem, isFragment - ) - elif elemType in ["bullet_list", "numbered_list"]: - return JsonResponseHandler._mergeListElements( - accumulatedElem, newFragmentElem, isFragment - ) - elif elemType in ["heading", "image"]: - # Usually complete - use new fragment if it exists, otherwise accumulated - return newFragmentElem if newFragmentElem else accumulatedElem - else: - # Generic merge: use mergeDeepStructures - return JsonResponseHandler.mergeDeepStructures( - accumulatedElem, newFragmentElem, 0, f"element_merge.{elemType}" - ) - - @staticmethod - def _mergeTableElementsGeneric( - accumulatedElem: Dict[str, Any], - newFragmentElem: Dict[str, Any], - accumulatedRaw: str, - newFragmentRaw: str, - isFragment: bool - ) -> Dict[str, Any]: - """ - GENERIC merge of two table elements with content-type-specific optimizations. - - Handles: - - Overlapping rows (detect duplicates by comparing row content) - - Missing headers (complete with existing headers) - - Incomplete rows (complete with null values if needed) - - Fragment rows (if newFragment is a fragment, extract rows from raw string) - - Args: - accumulatedElem: Accumulated table element - newFragmentElem: New fragment table element - accumulatedRaw: Raw accumulated string (for fragment detection) - newFragmentRaw: Raw new fragment string (for fragment extraction) - isFragment: Whether newFragment is a fragment - - Returns: - Merged table element - """ - # Extract content (handle both nested and flat structures) - accContent = accumulatedElem.get("content", {}) - if not accContent and "rows" in accumulatedElem: - accContent = accumulatedElem - - fragContent = newFragmentElem.get("content", {}) - if not fragContent and "rows" in newFragmentElem: - fragContent = newFragmentElem - - # Extract rows - accRows = accContent.get("rows", []) if isinstance(accContent, dict) else [] - - # If fragment, try to extract rows from raw string - fragRows = fragContent.get("rows", []) if isinstance(fragContent, dict) else [] - if isFragment and not fragRows: - fragRows = JsonResponseHandler._extractRowsFromFragment(newFragmentRaw) - - # Extract headers (complete missing with existing) - accHeaders = accContent.get("headers", []) if isinstance(accContent, dict) else [] - fragHeaders = fragContent.get("headers", []) if isinstance(fragContent, dict) else [] - mergedHeaders = accHeaders if accHeaders else fragHeaders - - # Merge rows with overlap detection - mergedRows = JsonResponseHandler._mergeRowsWithOverlapDetection(accRows, fragRows) - - # Reconstruct table element - mergedContent = { - "headers": mergedHeaders, - "rows": mergedRows - } - - # Preserve other fields (caption, etc.) - if isinstance(accContent, dict) and "caption" in accContent: - mergedContent["caption"] = accContent["caption"] - elif isinstance(fragContent, dict) and "caption" in fragContent: - mergedContent["caption"] = fragContent["caption"] - - return { - "type": "table", - "content": mergedContent - } - - @staticmethod - def _extractRowsFromFragment(fragmentRaw: str) -> List[List[str]]: - """ - Extract table rows from fragment string. - - Handles fragments like: - - `["1947", "16883"], ["1948", "16889"], ...` - - `"rows": [["1947", "10000"], ["1948", "10100"]...` - - Incomplete fragments cut mid-string - Also handles fragments with more than 2 columns. - """ - import re - rows = [] - - # Pattern 1: Array of arrays with 2 columns `["cell1", "cell2"], ["cell3", "cell4"]` - # This pattern matches complete arrays: ["value1", "value2"] - pattern2Col = r'\["([^"]*)",\s*"([^"]*)"\]' - matches2Col = re.findall(pattern2Col, fragmentRaw) - - if matches2Col and len(matches2Col) >= 2: # Need at least 2 rows to be confident - for match in matches2Col: - if len(match) == 2: - rows.append([match[0], match[1]]) - if rows: - return rows - - # Pattern 2: Array of arrays with variable columns (more robust) - # Find all array patterns: ["...", "...", ...] - # Use non-greedy matching but ensure we get complete arrays - arrayPattern = r'\[(.*?)\]' - arrayMatches = re.findall(arrayPattern, fragmentRaw) - - # Filter to only arrays that look like table rows (have multiple quoted values) - validArrays = [] - for arrayContent in arrayMatches: - # Extract quoted strings from array content - cellPattern = r'"([^"]*)"' - cells = re.findall(cellPattern, arrayContent) - # Only consider arrays with 2+ cells (likely table rows) - if len(cells) >= 2: - validArrays.append(cells) - - if validArrays and len(validArrays) >= 2: # Need at least 2 rows - return validArrays - - # Pattern 3: Look for "rows": [...] pattern in incomplete JSON - # This handles cases like: "rows": [["1947", "10000"], ["1948", "10100"]... - rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' - rowsMatch = re.search(rowsPattern, fragmentRaw, re.DOTALL) - if rowsMatch: - rowsContent = rowsMatch.group(1) - # Extract all array patterns from rows content - arrayPattern = r'\[(.*?)\]' - arrayMatches = re.findall(arrayPattern, rowsContent) - for arrayContent in arrayMatches: - cellPattern = r'"([^"]*)"' - cells = re.findall(cellPattern, arrayContent) - if len(cells) >= 2: # At least 2 columns - rows.append(cells) - if rows: - return rows - - # Pattern 4: Try to parse as JSON array (handles complete arrays) - - # Try to close incomplete structures - closed = closeJsonStructures(fragmentRaw.strip()) - parsed, parseErr, _ = tryParseJson(closed) - - if parseErr is None and isinstance(parsed, list): - if parsed and isinstance(parsed[0], list): - # Array of arrays - table rows - return parsed - elif parsed and isinstance(parsed[0], str): - # Array of strings - might be single column table - return [[item] for item in parsed] - - # Pattern 5: Last resort - extract any array patterns we can find - # Even if incomplete, try to extract what we can - if not rows: - # Find all patterns like ["value1", "value2"] even if incomplete - # Use a more lenient pattern that handles incomplete strings - incompletePattern = r'\["([^"]*)"(?:,\s*"([^"]*)")?' - incompleteMatches = re.findall(incompletePattern, fragmentRaw) - for match in incompleteMatches: - if match[0]: # First value exists - if match[1]: # Second value exists - rows.append([match[0], match[1]]) - else: - # Only one value - might be incomplete, skip for now - pass - - return rows - - @staticmethod - def _mergeParagraphElements( - accumulatedElem: Dict[str, Any], - newFragmentElem: Dict[str, Any], - isFragment: bool - ) -> Dict[str, Any]: - """Merge two paragraph elements.""" - accContent = accumulatedElem.get("content", {}) - fragContent = newFragmentElem.get("content", {}) - - accText = accContent.get("text", "") if isinstance(accContent, dict) else "" - fragText = fragContent.get("text", "") if isinstance(fragContent, dict) else "" - - # Merge text (remove overlap if fragment) - mergedText = accText + fragText if not isFragment else (accText.rstrip() + " " + fragText.lstrip()) - - return { - "type": "paragraph", - "content": {"text": mergedText} - } - - @staticmethod - def _mergeCodeBlockElements( - accumulatedElem: Dict[str, Any], - newFragmentElem: Dict[str, Any], - isFragment: bool - ) -> Dict[str, Any]: - """Merge two code block elements.""" - accContent = accumulatedElem.get("content", {}) - fragContent = newFragmentElem.get("content", {}) - - accCode = accContent.get("code", "") if isinstance(accContent, dict) else "" - fragCode = fragContent.get("code", "") if isinstance(fragContent, dict) else "" - - accLanguage = accContent.get("language") if isinstance(accContent, dict) else None - fragLanguage = fragContent.get("language") if isinstance(fragContent, dict) else None - - mergedCode = accCode + "\n" + fragCode if fragCode else accCode - mergedLanguage = accLanguage or fragLanguage - - result = { - "type": "code_block", - "content": {"code": mergedCode} - } - if mergedLanguage: - result["content"]["language"] = mergedLanguage - - return result - - @staticmethod - def _mergeListElements( - accumulatedElem: Dict[str, Any], - newFragmentElem: Dict[str, Any], - isFragment: bool - ) -> Dict[str, Any]: - """Merge two list elements (bullet_list or numbered_list).""" - accContent = accumulatedElem.get("content", {}) - fragContent = newFragmentElem.get("content", {}) - - accItems = accContent.get("items", []) if isinstance(accContent, dict) else [] - fragItems = fragContent.get("items", []) if isinstance(fragContent, dict) else [] - - # Merge items with overlap detection - mergedItems = JsonResponseHandler._mergeItemsWithOverlapDetection(accItems, fragItems) - - elemType = accumulatedElem.get("type") or newFragmentElem.get("type") - - return { - "type": elemType, - "content": {"items": mergedItems} - } - - @staticmethod - def _findOverlapStartIndex( - accumulatedElements: List[Dict[str, Any]], - overlapElements: List[Dict[str, Any]] - ) -> int: - """ - Find the start index in accumulatedElements where overlapElements begin. - - This helps identify where to merge continuation elements by matching - the overlap elements with the end of accumulated elements. - - Args: - accumulatedElements: List of accumulated elements - overlapElements: List of overlap elements from continuation response - - Returns: - Index where overlap starts, or -1 if not found - """ - if not overlapElements or not accumulatedElements: - return -1 - - # Try to find overlap by matching element structures - # Start from the end of accumulatedElements and work backwards - overlapLen = len(overlapElements) - accLen = len(accumulatedElements) - - if overlapLen > accLen: - return -1 - - # Try matching from different start positions - for startIdx in range(max(0, accLen - overlapLen), accLen): - # Check if elements from startIdx match overlapElements - matches = True - for i in range(min(overlapLen, accLen - startIdx)): - accElem = accumulatedElements[startIdx + i] - overlapElem = overlapElements[i] - - # Compare element types - if isinstance(accElem, dict) and isinstance(overlapElem, dict): - accType = accElem.get("type") - overlapType = overlapElem.get("type") - if accType != overlapType: - matches = False - break - - # For tables, compare row counts or last rows - if accType == "table": - accRows = accElem.get("rows", []) or (accElem.get("content", {}).get("rows", []) if isinstance(accElem.get("content"), dict) else []) - overlapRows = overlapElem.get("rows", []) or (overlapElem.get("content", {}).get("rows", []) if isinstance(overlapElem.get("content"), dict) else []) - if accRows and overlapRows: - # Check if last rows match - if len(accRows) >= len(overlapRows): - lastAccRows = accRows[-len(overlapRows):] - if lastAccRows != overlapRows: - matches = False - break - # For lists, compare items - elif accType in ["bullet_list", "numbered_list"]: - accItems = accElem.get("items", []) or (accElem.get("content", {}).get("items", []) if isinstance(accElem.get("content"), dict) else []) - overlapItems = overlapElem.get("items", []) or (overlapElem.get("content", {}).get("items", []) if isinstance(overlapElem.get("content"), dict) else []) - if accItems and overlapItems: - if len(accItems) >= len(overlapItems): - lastAccItems = accItems[-len(overlapItems):] - if lastAccItems != overlapItems: - matches = False - break - else: - matches = False - break - - if matches: - return startIdx - - return -1 - - @staticmethod - def _mergeRowsWithOverlapDetection( - accRows: List[List[str]], - fragRows: List[List[str]] - ) -> List[List[str]]: - """ - Merge two row arrays, detecting and removing overlaps. - - Overlap detection: Compare rows to find duplicates. - Missing parts: Complete with null values if needed. - """ - if not accRows: - return fragRows - if not fragRows: - return accRows - - # Find overlap by comparing last rows of accRows with first rows of fragRows - overlapStart = 0 - maxOverlap = min(len(accRows), len(fragRows)) - - # Find the longest overlap - for overlapLen in range(maxOverlap, 0, -1): - accSuffix = accRows[-overlapLen:] - fragPrefix = fragRows[:overlapLen] - - # Compare rows (exact match) - if accSuffix == fragPrefix: - overlapStart = overlapLen - break - - # Merge: accumulated rows + non-overlapping fragment rows - merged = accRows + fragRows[overlapStart:] - - return merged - - @staticmethod - def _mergeItemsWithOverlapDetection( - accItems: List[str], - fragItems: List[str] - ) -> List[str]: - """ - Merge two item arrays (for lists), detecting and removing overlaps. - - Overlap detection: Compare items to find duplicates. - """ - if not accItems: - return fragItems - if not fragItems: - return accItems - - # Find overlap by comparing last items of accItems with first items of fragItems - overlapStart = 0 - maxOverlap = min(len(accItems), len(fragItems)) - - # Find the longest overlap - for overlapLen in range(maxOverlap, 0, -1): - accSuffix = accItems[-overlapLen:] - fragPrefix = fragItems[:overlapLen] - - # Compare items (exact match) - if accSuffix == fragPrefix: - overlapStart = overlapLen - break - - # Merge: accumulated items + non-overlapping fragment items - merged = accItems + fragItems[overlapStart:] - - return merged - - @staticmethod - def _extractOverlapAndContinuation(jsonString: str) -> Tuple[Optional[List[Dict[str, Any]]], Optional[str]]: - """ - Extract overlap and continuation sections from AI response with explicit overlap structure. - - Expected format: - { - "overlap": [...], // Elements to repeat for merging - "continuation": [...] // New elements to add - } - - Or alternative format: - { - "overlap": "...", // Overlap as string - "continuation": "..." // Continuation as string - } - - Args: - jsonString: JSON string that may contain overlap/continuation structure - - Returns: - Tuple of (overlap_elements, continuation_json_string) or (None, None) if not found - """ - if not jsonString: - return None, None - - - # Extract and normalize JSON - extracted = stripCodeFences(normalizeJsonText(jsonString)).strip() - if not extracted: - return None, None - - # Try to parse - try: - closed = closeJsonStructures(extracted) - parsed, parseErr, _ = tryParseJson(closed) - - if parseErr is None and isinstance(parsed, dict): - # Check for overlap/continuation structure - overlap = parsed.get("overlap") - continuation = parsed.get("continuation") - - if overlap is not None and continuation is not None: - # Found explicit overlap structure - overlapElements = None - continuationJson = None - - # Extract overlap elements - if isinstance(overlap, list): - overlapElements = overlap - elif isinstance(overlap, str): - # Overlap is a string - try to parse it - try: - overlapParsed, _, _ = tryParseJson(closeJsonStructures(overlap)) - if isinstance(overlapParsed, list): - overlapElements = overlapParsed - except Exception: - pass - - # Extract continuation JSON - if isinstance(continuation, (dict, list)): - continuationJson = json.dumps(continuation, indent=2, ensure_ascii=False) - elif isinstance(continuation, str): - continuationJson = continuation - - if overlapElements is not None and continuationJson: - return overlapElements, continuationJson - except Exception: - pass - - return None, None - - @staticmethod - def _mergeWithExplicitOverlap( - accumulated: str, - continuationJson: str, - overlapElements: List[Dict[str, Any]] - ) -> str: - """ - Merge accumulated JSON with continuation JSON using explicit overlap information. - - Strategy: - 1. Find overlap in accumulated using overlapElements - 2. Remove overlapping elements from accumulated - 3. Append continuation JSON - - Args: - accumulated: Previously accumulated JSON string - continuationJson: Continuation JSON string (new content) - overlapElements: List of overlap elements from AI response - - Returns: - Merged JSON string - """ - if not accumulated: - return continuationJson - if not continuationJson: - return accumulated - - - # Normalize accumulated - accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() - accumulatedNormalized = JsonResponseHandler._normalizeToElementsStructure( - accumulatedExtracted, accumulated - ) - - # Normalize continuation - continuationExtracted = stripCodeFences(normalizeJsonText(continuationJson)).strip() - continuationNormalized = JsonResponseHandler._normalizeToElementsStructure( - continuationExtracted, continuationJson - ) - - # If both normalized successfully, use structure-based merge with overlap - if accumulatedNormalized and continuationNormalized: - merged = JsonResponseHandler._mergeJsonStructuresGeneric( - accumulatedNormalized, continuationNormalized, accumulatedExtracted, continuationExtracted, - overlapElements=overlapElements - ) - if merged: - return json.dumps(merged, indent=2, ensure_ascii=False) - - # Fallback: use overlap elements to find merge point in accumulated - # Find where overlap elements match in accumulated - if accumulatedNormalized and overlapElements: - accumulatedElements = accumulatedNormalized.get("elements", []) - overlapStartIndex = JsonResponseHandler._findOverlapStartIndex(accumulatedElements, overlapElements) - - if overlapStartIndex >= 0: - # Remove overlapping elements - accumulatedElements = accumulatedElements[:overlapStartIndex] - accumulatedNormalized["elements"] = accumulatedElements - - # Merge continuation - if continuationNormalized: - continuationElements = continuationNormalized.get("elements", []) - accumulatedElements.extend(continuationElements) - accumulatedNormalized["elements"] = accumulatedElements - return json.dumps(accumulatedNormalized, indent=2, ensure_ascii=False) - - # Last resort: simple concatenation - return JsonResponseHandler._mergeJsonStringsWithOverlapFallback(accumulated, continuationJson) - - @staticmethod - def _extractValidJsonPrefix(jsonString: str) -> str: - """ - Extract the longest valid JSON prefix from a string that may be cut randomly. - - Strategy: - 1. Try to find the longest prefix that can be closed and parsed - 2. Handle random cuts (mid-string, mid-number, etc.) - 3. Return the longest valid prefix found - - Args: - jsonString: JSON string that may be cut randomly - - Returns: - Longest valid JSON prefix, or empty string if none found - """ - if not jsonString or not jsonString.strip(): - return "" - - - # Strategy 1: Try progressive truncation to find longest valid JSON - # Use binary search-like approach for efficiency - bestValid = "" - bestLength = 0 - maxLen = len(jsonString) - - # Generate test lengths: full, 95%, 90%, ..., 10% - testLengths = [] - for percent in range(100, 9, -5): - testLen = int(maxLen * percent / 100) - if testLen > bestLength: - testLengths.append(testLen) - - # Also test specific points near the end (common cut points) - for offset in [200, 100, 50, 20, 10, 5, 2, 1]: - if maxLen > offset: - testLen = maxLen - offset - if testLen > bestLength: - testLengths.append(testLen) - - # Sort and deduplicate - testLengths = sorted(set(testLengths), reverse=True) - - for testLen in testLengths: - if testLen <= bestLength: - continue # Already found better - - testStr = jsonString[:testLen] - if not testStr.strip(): - continue - - # Try to close and parse - try: - closed = closeJsonStructures(testStr) - parsed, parseErr, _ = tryParseJson(closed) - - if parseErr is None and parsed is not None: - # Valid JSON found - if testLen > bestLength: - bestValid = closed - bestLength = testLen - except Exception: - continue - - # Strategy 2: If we found valid JSON, return it - if bestValid: - return bestValid - - # Strategy 3: Try to extract balanced JSON (find first complete structure) - jsonStripped = jsonString.strip() - - if jsonStripped.startswith('{') or jsonStripped.startswith('['): - # Try to extract balanced JSON - balanced = extractFirstBalancedJson(jsonStripped) - if balanced and balanced != jsonStripped: - try: - closed = closeJsonStructures(balanced) - parsed, parseErr, _ = tryParseJson(closed) - if parseErr is None: - return closed - except Exception: - pass - - # Strategy 4: Try to repair by removing incomplete trailing structures - # Find the last complete element/item before the cut - try: - # For arrays: find last complete element - if jsonStripped.startswith('['): - # Find last complete array element - lastComma = jsonStripped.rfind(',') - if lastComma > 0: - # Try prefix up to last comma - prefix = jsonStripped[:lastComma].strip() - if prefix.endswith(','): - prefix = prefix[:-1].strip() - if prefix: - closed = closeJsonStructures(prefix + ']') - parsed, parseErr, _ = tryParseJson(closed) - if parseErr is None: - return closed - - # For objects: find last complete key-value pair - elif jsonStripped.startswith('{'): - # Find last complete key-value pair - lastComma = jsonStripped.rfind(',') - if lastComma > 0: - # Try prefix up to last comma - prefix = jsonStripped[:lastComma].strip() - if prefix.endswith(','): - prefix = prefix[:-1].strip() - if prefix: - closed = closeJsonStructures(prefix + '}') - parsed, parseErr, _ = tryParseJson(closed) - if parseErr is None: - return closed - except Exception: - pass - - # Last resort: return empty (caller will handle) - return "" - - @staticmethod - def _smartConcatenate(accumulated: str, newFragment: str) -> str: - """ - Smart concatenation that tries to merge JSON fragments intelligently. - - Strategy: - 1. Extract valid JSON from both fragments - 2. Parse both as JSON objects/arrays - 3. Merge them structurally - 4. Return valid JSON - - Args: - accumulated: Accumulated JSON string - newFragment: New fragment to append - - Returns: - Merged string with valid JSON, or empty if merging not possible - """ - if not accumulated or not newFragment: - return "" - - - # Extract valid JSON prefixes from both - accumulatedValid = JsonResponseHandler._extractValidJsonPrefix(accumulated) - newFragmentValid = JsonResponseHandler._extractValidJsonPrefix(newFragment) - - if not accumulatedValid: - accumulatedValid = accumulated - if not newFragmentValid: - newFragmentValid = newFragment - - # Try to parse both - try: - closedAccumulated = closeJsonStructures(accumulatedValid) - parsedAccumulated, parseErr1, _ = tryParseJson(closedAccumulated) - - closedNewFragment = closeJsonStructures(newFragmentValid) - parsedNewFragment, parseErr2, _ = tryParseJson(closedNewFragment) - - # If both parse successfully, merge structurally - if parseErr1 is None and parseErr2 is None: - # Normalize both to elements structure - accNormalized = JsonResponseHandler._normalizeToElementsStructure(closedAccumulated, accumulated) - newNormalized = JsonResponseHandler._normalizeToElementsStructure(closedNewFragment, newFragment) - - if accNormalized and newNormalized: - merged = JsonResponseHandler._mergeJsonStructuresGeneric( - accNormalized, newNormalized, closedAccumulated, closedNewFragment - ) - if merged: - return json.dumps(merged, indent=2, ensure_ascii=False) - - # If only accumulated parses, return it - if parseErr1 is None and parsedAccumulated: - return json.dumps(parsedAccumulated, indent=2, ensure_ascii=False) - - # If only new fragment parses, return it - if parseErr2 is None and parsedNewFragment: - return json.dumps(parsedNewFragment, indent=2, ensure_ascii=False) - except Exception: - pass - - # Fallback: Try simple string concatenation with repair - accumulatedStripped = accumulated.strip() - newFragmentStripped = newFragment.strip() - - # If accumulated doesn't end with } or ], it might be incomplete - if accumulatedStripped and not accumulatedStripped.endswith(('}', ']')): - try: - closedAccumulated = closeJsonStructures(accumulatedStripped) - - # Check if newFragment starts with continuation - if newFragmentStripped.startswith(','): - # Remove leading comma and append - merged = closedAccumulated.rstrip() + newFragmentStripped.lstrip(',').strip() - elif newFragmentStripped.startswith(('}', ']')): - # Fragment starts with closing - might be completing accumulated - merged = closedAccumulated.rstrip() + newFragmentStripped - else: - # Try to append as continuation - # Check if we need a comma separator - if not closedAccumulated.rstrip().endswith((',', '[', '{')): - merged = closedAccumulated.rstrip() + ',' + newFragmentStripped - else: - merged = closedAccumulated.rstrip() + newFragmentStripped - - # Try to repair and parse the merged result - repaired = closeJsonStructures(merged) - parsed, parseErr, _ = tryParseJson(repaired) - if parseErr is None: - return json.dumps(parsed, indent=2, ensure_ascii=False) - except Exception: - pass - - # If smart concatenation failed, return empty (caller will handle) - return "" - - @staticmethod - def _mergeJsonStringsWithOverlapFallback( - accumulated: str, - newFragment: str - ) -> str: - """ - Fallback overlap detection using string comparison. - Used when both strings are complete JSON structures or fragments. - - CRITICAL: Never returns empty JSON - always returns at least accumulated. - """ - if not accumulated: - return newFragment if newFragment else "{}" - if not newFragment: - return accumulated - - - # Strategy 1: Try to extract valid JSON parts from both fragments - # This handles random cuts better by finding the longest valid prefix/suffix - - # Extract valid JSON from accumulated (find longest valid prefix) - accumulatedValid = JsonResponseHandler._extractValidJsonPrefix(accumulated) - - # Extract valid JSON from newFragment (find longest valid prefix) - newFragmentValid = JsonResponseHandler._extractValidJsonPrefix(newFragment) - - # If we have valid JSON from both, try structure-based merge - if accumulatedValid and newFragmentValid: - try: - parsedAccumulated, parseErr1, _ = tryParseJson(closeJsonStructures(accumulatedValid)) - parsedNewFragment, parseErr2, _ = tryParseJson(closeJsonStructures(newFragmentValid)) - - if parseErr1 is None and parseErr2 is None: - # Both are valid JSON - try structure merge - accNormalized = JsonResponseHandler._normalizeToElementsStructure(accumulatedValid, accumulated) - newNormalized = JsonResponseHandler._normalizeToElementsStructure(newFragmentValid, newFragment) - - if accNormalized and newNormalized: - merged = JsonResponseHandler._mergeJsonStructuresGeneric( - accNormalized, newNormalized, accumulatedValid, newFragmentValid - ) - if merged: - return json.dumps(merged, indent=2, ensure_ascii=False) - except Exception: - pass - - # Strategy 2: Find longest common suffix/prefix match (character-level overlap) - maxOverlapLen = min(len(accumulated), len(newFragment)) - - # Start from maximum possible overlap down to 1 character - # But limit to reasonable overlap (max 50% of shorter string) - maxReasonableOverlap = min(maxOverlapLen, min(len(accumulated), len(newFragment)) // 2) - - for overlapLen in range(maxReasonableOverlap, 0, -1): - accumulatedSuffix = accumulated[-overlapLen:] - newFragmentPrefix = newFragment[:overlapLen] - - if accumulatedSuffix == newFragmentPrefix: - # Found overlap - remove duplicate part - logger.debug(f"Found overlap of {overlapLen} characters, removing duplicate") - merged = accumulated + newFragment[overlapLen:] - # Ensure result is not empty - if merged and merged.strip(): - return merged - - # Strategy 3: No overlap found - try smart concatenation - # Check if we can append newFragment to accumulated without breaking JSON structure - merged = JsonResponseHandler._smartConcatenate(accumulated, newFragment) - if merged and merged.strip(): - return merged - - # Strategy 4: Last resort - simple concatenation (but ensure non-empty and valid JSON) - result = accumulated + newFragment - if not result or result.strip() in ['{}', '[]', '']: - # Return accumulated as fallback (at least we have that) - return accumulated if accumulated else "{}" - - # CRITICAL: Try to repair and validate the merged result - try: - repaired = closeJsonStructures(result) - parsed, parseErr, _ = tryParseJson(repaired) - if parseErr is None: - # Valid JSON - return it - return json.dumps(parsed, indent=2, ensure_ascii=False) - else: - # Still invalid - try to extract valid parts - validPrefix = JsonResponseHandler._extractValidJsonPrefix(result) - if validPrefix: - parsedPrefix, parseErr2, _ = tryParseJson(validPrefix) - if parseErr2 is None: - return json.dumps(parsedPrefix, indent=2, ensure_ascii=False) - except Exception: - pass - - # If repair failed, return accumulated (at least we have that) - if accumulated: - try: - repairedAccumulated = closeJsonStructures(accumulated) - parsedAcc, parseErrAcc, _ = tryParseJson(repairedAccumulated) - if parseErrAcc is None: - return json.dumps(parsedAcc, indent=2, ensure_ascii=False) - except Exception: - pass - return accumulated - - # Last resort: return empty structure - return "{}" - - @staticmethod - def isJsonComplete(parsedJson: Dict[str, Any]) -> bool: - """ - GENERIC function to check if parsed JSON structure is complete. - - Works for ANY JSON structure - no specific logic for content types. - - Completeness checks (all generic): - - All arrays are properly closed - - All objects are properly closed - - No incomplete structures - - Recursive validation of nested structures - - Args: - parsedJson: Parsed JSON object - - Returns: - True if JSON is complete, False otherwise - """ - def _checkStructureComplete(obj: Any, depth: int = 0) -> bool: - """Recursively check if structure is complete.""" - if depth > 50: # Prevent infinite recursion - return True - - if isinstance(obj, dict): - # Check all values recursively - for value in obj.values(): - if not _checkStructureComplete(value, depth + 1): - return False - return True - elif isinstance(obj, list): - # Check all items recursively - for item in obj: - if not _checkStructureComplete(item, depth + 1): - return False - return True - else: - # Primitive value - always complete - return True - - try: - return _checkStructureComplete(parsedJson) - except Exception as e: - logger.debug(f"Error checking JSON completeness: {e}") - return False - - @staticmethod - def finalizeJson(parsedJson: Dict[str, Any]) -> Dict[str, Any]: - """ - GENERIC function to finalize complete JSON by adding missing closing elements and repairing corruption. - - Works for ANY JSON structure - no specific logic for content types. - - Steps (all generic): - 1. Analyze structure for missing closing elements (recursively) - 2. Add closing brackets/braces where needed - 3. Repair any remaining corruption - 4. Validate final structure - - Args: - parsedJson: Parsed JSON object that needs finalization - - Returns: - Finalized JSON object - """ - # For now, just return as-is since parsing succeeded - # If needed, can add logic to check for incomplete structures - # and add closing elements - return parsedJson - - @staticmethod - def extractKpiValuesFromJson( - parsedJson: Dict[str, Any], - kpis: List[Dict[str, Any]] - ) -> List[Dict[str, Any]]: - """ - Extract current KPI values from parsed JSON and update KPI objects. - - Args: - parsedJson: Parsed JSON object - kpis: List of KPI objects (will be updated with currentValue) - - Returns: - Updated list of KPI objects with currentValue set - """ - updatedKpis = [] - - for kpi in kpis: - kpiId = kpi.get("id") - jsonPath = kpi.get("jsonPath") - - if not kpiId or not jsonPath: - continue - - # Create copy of KPI object - updatedKpi = kpi.copy() - - try: - # Extract value using JSON path - # Simple path format: "sections[0].elements[0].items" or "sections[0].elements[0].rows" - value = JsonResponseHandler._extractValueByPath(parsedJson, jsonPath) - - # Handle None (path doesn't exist - incomplete JSON) - if value is None: - updatedKpi["currentValue"] = kpi.get("currentValue", 0) - logger.debug(f"KPI {kpiId} path {jsonPath} not found in JSON (incomplete), keeping current value {updatedKpi['currentValue']}") - # Count items/rows/elements based on type - elif isinstance(value, list): - updatedKpi["currentValue"] = len(value) - logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: list with {len(value)} items") - elif isinstance(value, (int, float)): - updatedKpi["currentValue"] = int(value) - logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: numeric value {int(value)}") - else: - updatedKpi["currentValue"] = 0 - logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: non-list/non-numeric value, set to 0") - - except Exception as e: - logger.warning(f"Error extracting KPI {kpiId} from path {jsonPath}: {e}") - updatedKpi["currentValue"] = kpi.get("currentValue", 0) - - updatedKpis.append(updatedKpi) - - return updatedKpis - - @staticmethod - def extractKpiValuesFromIncompleteJson( - jsonString: str, - kpis: List[Dict[str, Any]] - ) -> List[Dict[str, Any]]: - """ - Extract KPI values from incomplete JSON string. - Uses existing JSON completion function to close incomplete structures, then extracts KPIs. - - Args: - jsonString: Incomplete JSON string - kpis: List of KPI objects - - Returns: - Updated list of KPI objects with currentValue set - """ - updatedKpis = [] - - for kpi in kpis: - kpiId = kpi.get("id") - jsonPath = kpi.get("jsonPath") - - if not kpiId or not jsonPath: - continue - - updatedKpi = kpi.copy() - - try: - # Use existing JSON completion function to close incomplete structures - - # Extract JSON string and complete it with missing closing elements - extracted = extractJsonString(jsonString) - completed = closeJsonStructures(extracted) - - # Parse completed JSON - parsed = json.loads(completed) - - # Extract value using path - value = JsonResponseHandler._extractValueByPath(parsed, jsonPath) - - # Handle None (path doesn't exist - incomplete JSON) - if value is None: - updatedKpi["currentValue"] = kpi.get("currentValue", 0) - logger.debug(f"KPI {kpiId} path {jsonPath} not found in completed JSON (still incomplete), keeping current value {updatedKpi['currentValue']}") - # Count items/rows/elements based on type - elif isinstance(value, list): - updatedKpi["currentValue"] = len(value) - logger.debug(f"Extracted KPI {kpiId} from completed JSON: list with {len(value)} items") - elif isinstance(value, (int, float)): - updatedKpi["currentValue"] = int(value) - logger.debug(f"Extracted KPI {kpiId} from completed JSON: numeric value {int(value)}") - else: - updatedKpi["currentValue"] = 0 - logger.debug(f"Extracted KPI {kpiId} from completed JSON: non-list/non-numeric value, set to 0") - - except Exception as e: - logger.warning(f"Error extracting KPI {kpiId} from incomplete JSON: {e}") - updatedKpi["currentValue"] = kpi.get("currentValue", 0) - - updatedKpis.append(updatedKpi) - - return updatedKpis - - @staticmethod - def _extractValueByPath(obj: Any, path: str) -> Any: - """ - Extract value from object using dot-notation path with array indices. - - Example: "sections[0].elements[0].items" - Returns None if path doesn't exist (for incomplete JSON handling). - """ - parts = path.split('.') - current = obj - - for part in parts: - if '[' in part and ']' in part: - # Handle array access: "sections[0]" - key = part[:part.index('[')] - index = int(part[part.index('[') + 1:part.index(']')]) - - if key: - if isinstance(current, dict): - current = current.get(key) - if current is None: - return None # Key doesn't exist - else: - return None # Can't access key on non-dict - - if isinstance(current, list): - if 0 <= index < len(current): - current = current[index] - else: - # Index out of range - return None for incomplete JSON - return None - else: - # Not a list, can't index - return None - else: - # Handle dict access - if isinstance(current, dict): - current = current.get(part) - if current is None: - return None # Key doesn't exist - else: - return None # Can't access key on non-dict - - return current - - @staticmethod - def validateKpiProgression( - accumulationState: JsonAccumulationState, - updatedKpis: List[Dict[str, Any]] - ) -> Tuple[bool, str]: - """ - Validate KPI progression from parsed JSON. - - Validation rules: - - Proceed if: At least ONE KPI increased - - Stop if: Any KPI went backwards → return (False, "KPI went backwards") - - Stop if: No KPIs progressed → return (False, "No progress") - - Finish if: All KPIs completed OR JSON is complete → return (True, "Complete") - - Args: - accumulationState: Current accumulation state (contains kpis) - updatedKpis: Updated KPI objects with currentValue set - - Returns: - Tuple of (shouldProceed, reason) - """ - if not accumulationState.kpis: - # No KPIs defined - always proceed - return True, "No KPIs defined" - - # Build dict of last values for comparison - lastValues = {kpi.get("id"): kpi.get("currentValue", 0) for kpi in accumulationState.kpis} - logger.debug(f"KPI validation: lastValues = {lastValues}") - logger.debug(f"KPI validation: updatedKpis = {[(kpi.get('id'), kpi.get('currentValue')) for kpi in updatedKpis]}") - - # Check if any KPI went backwards - for updatedKpi in updatedKpis: - kpiId = updatedKpi.get("id") - currentValue = updatedKpi.get("currentValue", 0) - - if kpiId in lastValues: - lastValue = lastValues[kpiId] - if currentValue < lastValue: - logger.warning(f"KPI {kpiId} went BACKWARDS: {lastValue} → {currentValue}") - return False, f"KPI {kpiId} went backwards" - - # Check if all KPIs are completed - allCompleted = True - for updatedKpi in updatedKpis: - targetValue = updatedKpi.get("targetValue", 0) - currentValue = updatedKpi.get("currentValue", 0) - - if currentValue < targetValue: - allCompleted = False - break - - if allCompleted: - logger.info("All KPIs completed") - return True, "All KPIs completed" - - # Check if at least one KPI progressed - atLeastOneProgressed = False - for updatedKpi in updatedKpis: - kpiId = updatedKpi.get("id") - currentValue = updatedKpi.get("currentValue", 0) - - if kpiId in lastValues: - lastValue = lastValues[kpiId] - if currentValue > lastValue: - atLeastOneProgressed = True - logger.info(f"KPI {kpiId} progressed: {lastValue} → {currentValue}") - break - else: - # First time seeing this KPI - if it has a value, it's progress - if currentValue > 0: - atLeastOneProgressed = True - logger.info(f"KPI {kpiId} initialized: {currentValue}") - break - - if not atLeastOneProgressed: - logger.warning(f"No KPIs progressed. Last values: {lastValues}, Current values: {[(kpi.get('id'), kpi.get('currentValue')) for kpi in updatedKpis]}") - return False, "No progress" - - return True, "Progress detected" - - @staticmethod - def accumulateAndParseJsonFragments( - accumulatedJsonString: str, - newFragmentString: str, - allSections: List[Dict[str, Any]], - iteration: int - ) -> Tuple[str, List[Dict[str, Any]], bool, Optional[Dict[str, Any]]]: - """ - Accumulate JSON fragments and parse when complete. - - GENERIC function that handles: - 1. Concatenating JSON strings with overlap detection - 2. Parsing the accumulated string - 3. Extracting sections (partial if incomplete, final if complete) - 4. Determining completion status - - Args: - accumulatedJsonString: Previously accumulated JSON string - newFragmentString: New fragment string from current iteration - allSections: Sections extracted so far (for prompt context) - iteration: Current iteration number - - Returns: - Tuple of: - - accumulatedJsonString: Updated accumulated string - - sections: Extracted sections (partial if incomplete, final if complete) - - isComplete: True if JSON is complete and valid - - parsedResult: Parsed JSON object (if parsing succeeded) - """ - - # Step 1: Clean encoding issues from accumulated string (check end of first delivered part) - cleanedAccumulated = JsonResponseHandler.cleanEncodingIssues(accumulatedJsonString) - - # Step 2: Clean encoding issues from new fragment - cleanedFragment = JsonResponseHandler.cleanEncodingIssues(newFragmentString) - - # Step 3: Concatenate with overlap handling - combinedString, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap( - cleanedAccumulated, - cleanedFragment - ) - # Note: hasOverlap indicates if iterations should continue, but this function - # doesn't control iterations, so we just use the merged string - - # Step 4: Try to parse - try: - extracted = extractJsonString(combinedString) - parsedResult = json.loads(extracted) - - # Step 5: Parsing succeeded - check completeness - isComplete = JsonResponseHandler.isJsonComplete(parsedResult) - - if isComplete: - # Step 6: Complete JSON - finalize - finalizedJson = JsonResponseHandler.finalizeJson(parsedResult) - sections = extractSectionsFromDocument(finalizedJson) - logger.info(f"Iteration {iteration}: JSON accumulation complete, extracted {len(sections)} sections") - return combinedString, sections, True, finalizedJson - else: - # Step 7: Incomplete but parseable - extract partial sections - sections = extractSectionsFromDocument(parsedResult) - logger.info(f"Iteration {iteration}: JSON accumulation incomplete but parseable, extracted {len(sections)} partial sections") - return combinedString, sections, False, parsedResult - - except json.JSONDecodeError: - # Step 8: Still broken - repair and extract partial sections - repaired = repairBrokenJson(combinedString) - if repaired: - sections = extractSectionsFromDocument(repaired) - logger.info(f"Iteration {iteration}: JSON accumulation repaired, extracted {len(sections)} sections") - return combinedString, sections, False, repaired - else: - # Repair failed - continue with data BEFORE merging the problematic piece - # Return previous accumulated string (before adding new fragment) - # This ensures we don't lose previously accumulated data - logger.warning(f"Iteration {iteration}: Repair failed, continuing with previous accumulated data") - return accumulatedJsonString, [], False, None - diff --git a/modules/services/serviceAi/subLoopingUseCases.py b/modules/services/serviceAi/subLoopingUseCases.py deleted file mode 100644 index a2828108..00000000 --- a/modules/services/serviceAi/subLoopingUseCases.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Generic Looping Use Case System - -Provides parametrized looping infrastructure supporting different JSON formats and use cases. -""" - -import logging -from dataclasses import dataclass, field -from typing import Dict, Any, List, Optional, Callable - -logger = logging.getLogger(__name__) - -# Callback functions for use-case-specific logic - -def _handleSectionContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, - debugPrefix: str, services: Any) -> str: - """Handle final result for section_content: return raw result to preserve all JSON blocks.""" - final_json = result # Return raw response to preserve all JSON blocks - # Write final merged result for section_content (overwrites iteration 1 response with complete merged result) - if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): - services.utils.writeDebugFile(final_json, f"{debugPrefix}_response") - return final_json - - -def _handleChapterStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, - debugPrefix: str, services: Any) -> str: - """Handle final result for chapter_structure: format JSON and write debug file.""" - import json - final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) - # Write final result for chapter structure - if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): - services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") - return final_json - - -def _handleCodeStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, - debugPrefix: str, services: Any) -> str: - """Handle final result for code_structure: format JSON and write debug file.""" - import json - final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) - # Write final result for code structure - if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): - services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") - return final_json - - -def _handleCodeContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, - debugPrefix: str, services: Any) -> str: - """Handle final result for code_content: format JSON.""" - import json - final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) - return final_json - - -def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any: - """Normalize JSON structure for section_content use case.""" - # For section_content, expect {"elements": [...]} structure - if isinstance(parsed, list): - # Check if list contains strings (invalid format) or element objects - if parsed and isinstance(parsed[0], str): - # Invalid format - list of strings instead of elements - # Try to convert strings to paragraph elements as fallback - logger.debug(f"Received list of strings instead of elements array, converting to paragraph elements") - elements = [] - for text in parsed: - if isinstance(text, str) and text.strip(): - elements.append({ - "type": "paragraph", - "content": { - "text": text.strip() - } - }) - return {"elements": elements} if elements else {"elements": []} - else: - # Convert plain list of elements to elements structure - return {"elements": parsed} - elif isinstance(parsed, dict): - # If it already has "elements", return as-is - if "elements" in parsed: - return parsed - # If it has "type" and looks like an element, wrap in elements array - elif parsed.get("type"): - return {"elements": [parsed]} - # Otherwise, assume it's already in correct format - else: - return parsed - - # For other use cases, return as-is (they have their own structures) - return parsed - - -def _normalizeDefaultJson(parsed: Any, useCaseId: str) -> Any: - """Default normalizer: return as-is.""" - return parsed - - -@dataclass -class LoopingUseCase: - """Configuration for a specific looping use case.""" - - # Identification - useCaseId: str # "section_content", "chapter_structure", "code_structure", "code_content" - - # JSON Format Detection - jsonTemplate: Dict[str, Any] # Expected JSON structure template - detectionKeys: List[str] # Keys to check for format detection (e.g., ["elements"], ["chapters"], ["files"]) - detectionPath: str # JSONPath to check (e.g., "documents[0].chapters", "files[0].content") - - # Prompt Building - initialPromptBuilder: Optional[Callable] = None # Function to build initial prompt - continuationPromptBuilder: Optional[Callable] = None # Function to build continuation prompt - - # Accumulation & Merging - accumulator: Optional[Callable] = None # Function to accumulate fragments - merger: Optional[Callable] = None # Function to merge accumulated data - - # Continuation Context - continuationContextBuilder: Optional[Callable] = None # Build continuation context for this format - - # Result Building - resultBuilder: Optional[Callable] = None # Build final result from accumulated data - - # Use-case-specific handlers (callbacks to avoid if/elif chains in generic code) - finalResultHandler: Optional[Callable] = None # Handle final result formatting and debug file writing - jsonNormalizer: Optional[Callable] = None # Normalize JSON structure for this use case - - # Metadata - supportsAccumulation: bool = True # Whether this use case supports accumulation - requiresExtraction: bool = False # Whether this requires extraction (like sections) - - -class LoopingUseCaseRegistry: - """Registry of all looping use cases.""" - - def __init__(self): - self.useCases: Dict[str, LoopingUseCase] = {} - self._registerDefaultUseCases() - - def register(self, useCase: LoopingUseCase): - """Register a new use case.""" - self.useCases[useCase.useCaseId] = useCase - logger.debug(f"Registered looping use case: {useCase.useCaseId}") - - def get(self, useCaseId: str) -> Optional[LoopingUseCase]: - """Get use case by ID.""" - return self.useCases.get(useCaseId) - - def detectUseCase(self, parsedJson: Dict[str, Any]) -> Optional[str]: - """Detect which use case matches the JSON structure.""" - for useCaseId, useCase in self.useCases.items(): - if self._matchesFormat(parsedJson, useCase): - return useCaseId - return None - - def _matchesFormat(self, json: Dict[str, Any], useCase: LoopingUseCase) -> bool: - """Check if JSON matches use case format.""" - # Check top-level keys - for key in useCase.detectionKeys: - if key in json: - return True - - # Check nested path using simple dictionary traversal (no jsonpath_ng needed) - if useCase.detectionPath: - try: - # Simple path matching without jsonpath_ng - # Format: "documents[0].chapters" or "files[0].content" - pathParts = useCase.detectionPath.split(".") - current = json - - for part in pathParts: - # Handle array indices like "documents[0]" - if "[" in part and "]" in part: - key = part.split("[")[0] - index = int(part.split("[")[1].split("]")[0]) - if isinstance(current, dict) and key in current: - if isinstance(current[key], list) and 0 <= index < len(current[key]): - current = current[key][index] - else: - return False - else: - return False - else: - # Regular key access - if isinstance(current, dict) and part in current: - current = current[part] - else: - return False - - # If we successfully traversed the path, it matches - return True - except Exception as e: - logger.debug(f"Path matching failed for {useCase.useCaseId}: {e}") - - return False - - def _registerDefaultUseCases(self): - """Register default use cases.""" - - # Use Case 1: Section Content Generation - # Returns JSON with "elements" array directly - self.register(LoopingUseCase( - useCaseId="section_content", - jsonTemplate={"elements": []}, - detectionKeys=["elements"], - detectionPath="", - initialPromptBuilder=None, # Will use default prompt builder - continuationPromptBuilder=None, # Will use default continuation builder - accumulator=None, # Direct return, no accumulation - merger=None, - continuationContextBuilder=None, # Will use default continuation context - resultBuilder=None, # Return JSON directly - finalResultHandler=_handleSectionContentFinalResult, - jsonNormalizer=_normalizeSectionContentJson, - supportsAccumulation=False, - requiresExtraction=False - )) - - # Use Case 2: Chapter Structure Generation - # Returns JSON with "documents[0].chapters" structure - self.register(LoopingUseCase( - useCaseId="chapter_structure", - jsonTemplate={"documents": [{"chapters": []}]}, - detectionKeys=["chapters"], - detectionPath="documents[0].chapters", - initialPromptBuilder=None, - continuationPromptBuilder=None, - accumulator=None, # Direct return, no accumulation - merger=None, - continuationContextBuilder=None, - resultBuilder=None, # Return JSON directly - finalResultHandler=_handleChapterStructureFinalResult, - jsonNormalizer=_normalizeDefaultJson, - supportsAccumulation=False, - requiresExtraction=False - )) - - # Use Case 3: Code Structure Generation - self.register(LoopingUseCase( - useCaseId="code_structure", - jsonTemplate={ - "metadata": { - "language": "", - "projectType": "single_file|multi_file", - "projectName": "" - }, - "files": [ - { - "id": "", - "filename": "", - "fileType": "", - "dependencies": [], - "imports": [], - "functions": [], - "classes": [] - } - ] - }, - detectionKeys=["files"], - detectionPath="files", - initialPromptBuilder=None, - continuationPromptBuilder=None, - accumulator=None, # Direct return - merger=None, - continuationContextBuilder=None, - resultBuilder=None, - finalResultHandler=_handleCodeStructureFinalResult, - jsonNormalizer=_normalizeDefaultJson, - supportsAccumulation=False, - requiresExtraction=False - )) - - # Use Case 5: Code Content Generation (NEW) - self.register(LoopingUseCase( - useCaseId="code_content", - jsonTemplate={"files": [{"content": "", "functions": []}]}, - detectionKeys=["content", "functions"], - detectionPath="files[0].content", - initialPromptBuilder=None, - continuationPromptBuilder=None, - accumulator=None, # Will use default accumulator - merger=None, # Will use default merger - continuationContextBuilder=None, - resultBuilder=None, # Will use default result builder - finalResultHandler=_handleCodeContentFinalResult, - jsonNormalizer=_normalizeDefaultJson, - supportsAccumulation=True, - requiresExtraction=False - )) - - logger.info(f"Registered {len(self.useCases)} default looping use cases") - diff --git a/modules/services/serviceAi/subResponseParsing.py b/modules/services/serviceAi/subResponseParsing.py deleted file mode 100644 index 68c123ac..00000000 --- a/modules/services/serviceAi/subResponseParsing.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Response Parsing Module - -Handles parsing of AI responses, including: -- Section extraction from responses -- JSON completeness detection -- Loop detection -- Document metadata extraction -- Final result building -""" -import json -import logging -from typing import Dict, Any, List, Optional, Tuple - -from modules.shared.jsonUtils import extractJsonString, repairBrokenJson, extractSectionsFromDocument -from .subJsonResponseHandling import JsonResponseHandler -from modules.datamodels.datamodelAi import JsonAccumulationState - -logger = logging.getLogger(__name__) - - -class ResponseParser: - """Handles parsing of AI responses and completion detection.""" - - def __init__(self, services): - """Initialize ResponseParser with service center access.""" - self.services = services - - def extractSectionsFromResponse( - self, - result: str, - iteration: int, - debugPrefix: str, - allSections: List[Dict[str, Any]] = None, - accumulationState: Optional[JsonAccumulationState] = None - ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]: - """ - Extract sections from AI response, handling both valid and broken JSON. - - NEW BEHAVIOR: - - First iteration: Check if complete, if not start accumulation - - Subsequent iterations: Accumulate strings, parse when complete - - Returns: - Tuple of: - - sections: Extracted sections - - wasJsonComplete: True if JSON is complete - - parsedResult: Parsed JSON object - - updatedAccumulationState: Updated accumulation state (None if not in accumulation mode) - """ - if allSections is None: - allSections = [] - - if iteration == 1: - # First iteration - check if complete - parsed = None - try: - extracted = extractJsonString(result) - parsed = json.loads(extracted) - - # Check completeness - if JsonResponseHandler.isJsonComplete(parsed): - # Complete JSON - no accumulation needed - sections = extractSectionsFromDocument(parsed) - logger.info(f"Iteration 1: Complete JSON detected, no accumulation needed") - return sections, True, parsed, None # No accumulation - except Exception: - pass - - # Incomplete - try to extract partial sections from broken JSON - logger.info(f"Iteration 1: Incomplete JSON detected, attempting to extract partial sections") - - partialSections = [] - if parsed: - # Try to extract sections from parsed (even if incomplete) - partialSections = extractSectionsFromDocument(parsed) - else: - # Try to repair broken JSON and extract sections - try: - repaired = repairBrokenJson(result) - if repaired: - partialSections = extractSectionsFromDocument(repaired) - parsed = repaired # Use repaired version for accumulation state - except Exception: - pass # If repair fails, continue with empty sections - - - # Define KPIs (async call - need to handle this) - # For now, create accumulation state without KPIs, will be updated after async call - accumulationState = JsonAccumulationState( - accumulatedJsonString=result, - isAccumulationMode=True, - lastParsedResult=parsed, - allSections=partialSections, - kpis=[] - ) - - # Note: KPI definition will be done in the caller (async context) - return partialSections, False, parsed, accumulationState - - else: - # Subsequent iterations - accumulate - if accumulationState and accumulationState.isAccumulationMode: - accumulated, sections, isComplete, parsedResult = \ - JsonResponseHandler.accumulateAndParseJsonFragments( - accumulationState.accumulatedJsonString, - result, - allSections, - iteration - ) - - # Update accumulation state - accumulationState.accumulatedJsonString = accumulated - accumulationState.lastParsedResult = parsedResult - accumulationState.allSections = allSections + sections if sections else allSections - accumulationState.isAccumulationMode = not isComplete - - # Log accumulated JSON for debugging - if parsedResult: - accumulated_json_str = json.dumps(parsedResult, indent=2, ensure_ascii=False) - self.services.utils.writeDebugFile(accumulated_json_str, f"{debugPrefix}_accumulated_json_iteration_{iteration}.json") - - return sections, isComplete, parsedResult, accumulationState - else: - # No accumulation mode - process normally (shouldn't happen) - logger.warning(f"Iteration {iteration}: No accumulation state but iteration > 1") - return [], False, None, None - - def shouldContinueGeneration( - self, - allSections: List[Dict[str, Any]], - iteration: int, - wasJsonComplete: bool, - rawResponse: str = None - ) -> bool: - """ - Determine if AI generation loop should continue. - - CRITICAL: This is ONLY about AI Loop Completion, NOT Action DoD! - Action DoD is checked AFTER the AI Loop completes in _refineDecide. - - Simple logic: - - If JSON parsing failed or incomplete → continue (needs more content) - - If JSON parses successfully and is complete → stop (all content delivered) - - Loop detection prevents infinite loops - - CRITICAL: JSON completeness is determined by parsing, NOT by last character check! - Returns True if we should continue, False if AI Loop is done. - """ - if len(allSections) == 0: - return True # No sections yet, continue - - # CRITERION 1: If JSON was incomplete/broken (parsing failed or incomplete) - continue to repair/complete - if not wasJsonComplete: - logger.info(f"Iteration {iteration}: JSON incomplete/broken - continuing to complete") - return True - - # CRITERION 2: JSON is complete (parsed successfully) - check for loop detection - if self._isStuckInLoop(allSections, iteration): - logger.warning(f"Iteration {iteration}: Detected potential infinite loop - stopping AI loop") - return False - - # JSON is complete and not stuck in loop - done - logger.info(f"Iteration {iteration}: JSON complete - AI loop done") - return False - - def _isStuckInLoop( - self, - allSections: List[Dict[str, Any]], - iteration: int - ) -> bool: - """ - Detect if we're stuck in a loop (same content being repeated). - - Generic approach: Check if recent iterations are adding minimal or duplicate content. - """ - if iteration < 3: - return False # Need at least 3 iterations to detect a loop - - if len(allSections) == 0: - return False - - # Check if last section is very small (might be stuck) - lastSection = allSections[-1] - elements = lastSection.get("elements", []) - - if isinstance(elements, list) and elements: - lastElem = elements[-1] if elements else {} - else: - lastElem = elements if isinstance(elements, dict) else {} - - # Check content size of last section - lastSectionSize = 0 - if isinstance(lastElem, dict): - for key, value in lastElem.items(): - if isinstance(value, str): - lastSectionSize += len(value) - elif isinstance(value, list): - lastSectionSize += len(str(value)) - - # If last section is very small and we've done many iterations, might be stuck - if lastSectionSize < 100 and iteration > 10: - logger.warning(f"Potential loop detected: iteration {iteration}, last section size {lastSectionSize}") - return True - - return False - - def extractDocumentMetadata( - self, - parsedResult: Dict[str, Any] - ) -> Optional[Dict[str, Any]]: - """ - Extract document metadata (title, filename) from parsed AI response. - Returns dict with 'title' and 'filename' keys if found, None otherwise. - """ - if not isinstance(parsedResult, dict): - return None - - # Try to get from documents array (preferred structure) - if "documents" in parsedResult and isinstance(parsedResult["documents"], list) and len(parsedResult["documents"]) > 0: - firstDoc = parsedResult["documents"][0] - if isinstance(firstDoc, dict): - title = firstDoc.get("title") - filename = firstDoc.get("filename") - if title or filename: - return { - "title": title, - "filename": filename - } - - return None - - def buildFinalResultFromSections( - self, - allSections: List[Dict[str, Any]], - documentMetadata: Optional[Dict[str, Any]] = None - ) -> str: - """ - Build final JSON result from accumulated sections. - Uses AI-provided metadata (title, filename) if available. - """ - if not allSections: - return "" - - # Extract metadata from AI response if available - title = "Generated Document" - filename = "document.json" - if documentMetadata: - if documentMetadata.get("title"): - title = documentMetadata["title"] - if documentMetadata.get("filename"): - filename = documentMetadata["filename"] - - # Build documents structure - # Assuming single document for now - documents = [{ - "id": "doc_1", - "title": title, - "filename": filename, - "sections": allSections - }] - - result = { - "metadata": { - "split_strategy": "single_document", - "source_documents": [], - "extraction_method": "ai_generation" - }, - "documents": documents - } - - return json.dumps(result, indent=2) - diff --git a/modules/services/serviceAi/subStructureFilling.py b/modules/services/serviceAi/subStructureFilling.py deleted file mode 100644 index a96b8353..00000000 --- a/modules/services/serviceAi/subStructureFilling.py +++ /dev/null @@ -1,2593 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Structure Filling Module - -Handles filling document structure with content, including: -- Filling sections with content parts -- Building section generation prompts -- Aggregation logic -""" -import json -import logging -import copy -import asyncio -from typing import Dict, Any, List, Optional, Tuple - -from modules.datamodels.datamodelExtraction import ContentPart -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.workflows.processing.shared.stateTools import checkWorkflowStopped - -logger = logging.getLogger(__name__) - - -class StructureFiller: - """Handles filling document structure with content.""" - - # Default concurrency limit for parallel generation (chapters/sections) - DEFAULT_MAX_CONCURRENT_GENERATION = 16 - - def __init__(self, services, aiService): - """Initialize StructureFiller with service center and AI service access.""" - self.services = services - self.aiService = aiService - - def _getMaxConcurrentGeneration(self, options: Optional[AiCallOptions] = None) -> int: - """Get max concurrent generation limit, configurable via options.""" - if options and hasattr(options, 'maxConcurrentGeneration'): - return options.maxConcurrentGeneration - return self.DEFAULT_MAX_CONCURRENT_GENERATION - - def _getUserLanguage(self) -> str: - """Get user language for document generation""" - try: - if self.services: - # Prefer detected language if available (from user intention analysis) - if hasattr(self.services, 'currentUserLanguage') and self.services.currentUserLanguage: - return self.services.currentUserLanguage - # Fallback to user's preferred language - elif hasattr(self.services, 'user') and self.services.user and hasattr(self.services.user, 'language'): - return self.services.user.language - except Exception: - pass - return 'en' # Default fallback - - def _getDocumentLanguage(self, structure: Dict[str, Any], documentId: str) -> str: - """ - Get language for a specific document from structure. - Falls back to user language if not specified. - - Args: - structure: The document structure with documents array - documentId: The ID of the document to get language for - - Returns: - ISO 639-1 language code (e.g., "de", "en", "fr") - """ - # Try to find document in structure - for doc in structure.get("documents", []): - if doc.get("id") == documentId: - docLanguage = doc.get("language") - if docLanguage: - return docLanguage - - # Fallback to metadata language - metadataLanguage = structure.get("metadata", {}).get("language") - if metadataLanguage: - return metadataLanguage - - # Fallback to user language - return self._getUserLanguage() - - def _extractContentPartInfo(self, chapter: Dict[str, Any]) -> Tuple[List[str], Dict[str, Any]]: - """ - Extract contentPartIds and contentPartInstructions from chapter's contentParts structure. - - Returns: - tuple: (contentPartIds list, contentPartInstructions dict) - """ - contentParts = chapter.get("contentParts", {}) - contentPartIds = list(contentParts.keys()) - # Extract instructions (entries with "instruction" field) and captions (entries with "caption" field) - contentPartInstructions = {} - for partId, partInfo in contentParts.items(): - if isinstance(partInfo, dict): - if "instruction" in partInfo: - contentPartInstructions[partId] = {"instruction": partInfo["instruction"]} - elif "caption" in partInfo: - # For entries with only caption (no instruction), still add to dict so it's available - contentPartInstructions[partId] = {"caption": partInfo["caption"]} - return contentPartIds, contentPartInstructions - - def _getContentPartCaption(self, chapter: Dict[str, Any], partId: str) -> Optional[str]: - """ - Get caption for a contentPart from chapter's contentParts structure. - Returns None if no caption is available. - - Args: - chapter: Chapter dict - partId: ContentPart ID - - Returns: - Caption string or None - """ - if "contentParts" in chapter: - contentParts = chapter.get("contentParts", {}) - partInfo = contentParts.get(partId) - if isinstance(partInfo, dict) and "caption" in partInfo: - return partInfo["caption"] - return None - - async def fillStructure( - self, - structure: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - parentOperationId: str, - language: Optional[str] = None - ) -> Dict[str, Any]: - """ - Phase 5D: Chapter-Content-Generierung (Zwei-Phasen-Ansatz). - - Phase 5D.1: Generiert Sections-Struktur für jedes Chapter - Phase 5D.2: Füllt Sections mit ContentParts - - Args: - structure: Struktur-Dict mit documents und chapters (nicht sections!) - contentParts: Alle vorbereiteten ContentParts - userPrompt: User-Anfrage - parentOperationId: Parent Operation-ID für ChatLog-Hierarchie - language: Language identified from user intention analysis (e.g., "de", "en", "fr") - - Returns: - Gefüllte Struktur mit elements in jeder Section (nach Flattening) - """ - # Erstelle Operation-ID für Struktur-Abfüllen - fillOperationId = f"{parentOperationId}_structure_filling" - - # Validate structure has chapters - hasChapters = False - for doc in structure.get("documents", []): - if "chapters" in doc: - hasChapters = True - break - - if not hasChapters: - error_msg = "Structure must have chapters. Legacy section-based structure is not supported." - logger.error(error_msg) - raise ValueError(error_msg) - - # Get language from services (user intention analysis) or parameter - if language is None: - language = self._getUserLanguage() - logger.debug(f"Using language from services (user intention analysis): {language}") - else: - logger.debug(f"Using provided language parameter: {language}") - - # Starte ChatLog mit Parent-Referenz - chapterCount = sum(len(doc.get("chapters", [])) for doc in structure.get("documents", [])) - self.services.chat.progressLogStart( - fillOperationId, - "Chapter Content Generation", - "Filling", - f"Processing {chapterCount} chapters", - parentOperationId=parentOperationId - ) - - try: - filledStructure = copy.deepcopy(structure) - - # Get options from AI service if available (for concurrency control) - # Default concurrency limit (16) will be used if options is None - options = None - # Note: Options can be passed via fillStructure if needed in the future - - # Phase 5D.1: Sections-Struktur für jedes Chapter generieren - filledStructure = await self._generateChapterSectionsStructure( - filledStructure, contentParts, userPrompt, fillOperationId, language, options - ) - - # Phase 5D.2: Sections mit ContentParts füllen - filledStructure = await self._fillChapterSections( - filledStructure, contentParts, userPrompt, fillOperationId, language, options - ) - - # Flattening: Chapters zu Sections konvertieren - flattenedStructure = self._flattenChaptersToSections(filledStructure) - - # Füge ContentParts-Metadaten zur Struktur hinzu (für Validierung) - flattenedStructure = self._addContentPartsMetadata(flattenedStructure, contentParts) - - # State 4 Validation: Validate and auto-fix filled structure - # Validation 4.1: Filled structure missing 'documents' field - if "documents" not in flattenedStructure: - raise ValueError("Filled structure missing 'documents' field - cannot auto-fix") - - for doc in flattenedStructure["documents"]: - # Validation 4.4: Verify language is preserved from input structure - # Language MUST be preserved from Phase 3 structure (validated in State 3) - if "language" not in doc: - raise ValueError(f"Document {doc.get('id')} missing language in filled structure - should have been preserved from Phase 3") - - # Validate language format - if not isinstance(doc["language"], str) or len(doc["language"]) != 2: - raise ValueError(f"Document {doc.get('id')} has invalid language format in filled structure: {doc['language']} - should be 2-character ISO 639-1 code") - - # CRITICAL: flattenedStructure has sections, not chapters! - # After flattening, chapters are converted to sections, so we need to validate sections directly - for section in doc.get("sections", []): - # Validation 4.2: Section missing 'elements' field - if "elements" not in section: - section["elements"] = [] - logger.info(f"Section {section.get('id')} missing 'elements' - created empty list") - - # Validation 4.3: Section has empty elements list - ALLOW (intentionally empty is OK) - # No action needed - empty elements are allowed - - # ChatLog abschließen - self.services.chat.progressLogFinish(fillOperationId, True) - - return flattenedStructure - - except Exception as e: - self.services.chat.progressLogFinish(fillOperationId, False) - logger.error(f"Error in fillStructure: {str(e)}") - raise - - async def _generateSingleChapterSectionsStructure( - self, - chapter: Dict[str, Any], - chapterIndex: int, - chapterId: str, - chapterLevel: int, - chapterTitle: str, - generationHint: str, - contentPartIds: List[str], - contentPartInstructions: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - language: str, - outputFormat: str, - parentOperationId: str, - totalChapters: int - ) -> None: - """ - Generate sections structure for a single chapter (used for parallel processing). - Modifies chapter dict in place. - """ - try: - # Update progress for chapter structure generation - progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 - self.services.chat.progressLogUpdate( - parentOperationId, - progress, - f"Generating sections for Chapter {chapterIndex}/{totalChapters}: {chapterTitle}" - ) - - chapterPrompt = self._buildChapterSectionsStructurePrompt( - chapterId=chapterId, - chapterLevel=chapterLevel, - chapterTitle=chapterTitle, - generationHint=generationHint, - contentPartIds=contentPartIds, - contentPartInstructions=contentPartInstructions, - contentParts=contentParts, - userPrompt=userPrompt, - language=language, - outputFormat=outputFormat - ) - - # AI-Call für Chapter-Struktur-Generierung - # Note: Debug logging is handled by callAiPlanning - checkWorkflowStopped(self.services) - aiResponse = await self.aiService.callAiPlanning( - prompt=chapterPrompt, - debugType=f"chapter_structure_{chapterId}" - ) - - sectionsStructure = json.loads( - self.services.utils.jsonExtractString(aiResponse) - ) - - chapter["sections"] = sectionsStructure.get("sections", []) - - # Setze useAiCall Flag (falls nicht von AI gesetzt) - # WICHTIG: useAiCall kann nur true sein, wenn mindestens ein ContentPart Format "extracted" hat! - # "object" und "reference" Formate werden direkt als Elemente hinzugefügt, benötigen kein AI. - for section in chapter["sections"]: - if "useAiCall" not in section: - contentType = section.get("content_type", "paragraph") - sectionContentPartIds = section.get("contentPartIds", []) - - # Prüfe ob mindestens ein ContentPart Format "extracted" hat - hasExtractedPart = False - for partId in sectionContentPartIds: - part = self._findContentPartById(partId, contentParts) - if part: - contentFormat = part.metadata.get("contentFormat", "unknown") - if contentFormat == "extracted": - hasExtractedPart = True - break - - # useAiCall kann nur true sein, wenn extracted Parts vorhanden sind - useAiCall = False - if hasExtractedPart: - # Prüfe ob Transformation nötig ist - useAiCall = contentType != "paragraph" - - # Prüfe contentPartInstructions für Transformation - if not useAiCall: - for partId in sectionContentPartIds: - instruction = contentPartInstructions.get(partId, {}).get("instruction", "") - if instruction and instruction.lower() not in ["include full text", "include all content", "use full extracted text"]: - useAiCall = True - break - - section["useAiCall"] = useAiCall - logger.debug(f"Section {section.get('id')}: useAiCall={useAiCall} (hasExtractedPart={hasExtractedPart}, contentType={contentType})") - - # Update progress after chapter completion - progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 - self.services.chat.progressLogUpdate( - parentOperationId, - progress, - f"Chapter {chapterIndex}/{totalChapters} completed: {chapterTitle}" - ) - - except Exception as e: - logger.error(f"Error generating sections structure for chapter {chapterId}: {str(e)}") - # Set empty sections on error - chapter["sections"] = [] - # Update progress even on error - progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 - self.services.chat.progressLogUpdate( - parentOperationId, - progress, - f"Chapter {chapterIndex}/{totalChapters} error: {chapterTitle}" - ) - raise - - async def _generateChapterSectionsStructure( - self, - chapterStructure: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - parentOperationId: str, - language: str, - options: Optional[AiCallOptions] = None - ) -> Dict[str, Any]: - """ - Phase 5D.1: Generiert Sections-Struktur für jedes Chapter (ohne Content) in parallel. - Sections enthalten: content_type, contentPartIds, generationHint, useAiCall - """ - # Count total chapters for progress tracking - totalChapters = sum(len(doc.get("chapters", [])) for doc in chapterStructure.get("documents", [])) - - # Get concurrency limit - maxConcurrent = self._getMaxConcurrentGeneration(options) - semaphore = asyncio.Semaphore(maxConcurrent) - - # Collect all chapters with their indices for parallel processing - chapterTasks = [] - chapterIndex = 0 - - for doc in chapterStructure.get("documents", []): - docId = doc.get("id", "unknown") - # Get language for this specific document - docLanguage = self._getDocumentLanguage(chapterStructure, docId) - # Get output format for this specific document - docFormat = doc.get("outputFormat", "txt") - - for chapter in doc.get("chapters", []): - chapterIndex += 1 - chapterId = chapter.get("id", "unknown") - chapterLevel = chapter.get("level", 1) - chapterTitle = chapter.get("title", "Untitled Chapter") - generationHint = chapter.get("generationHint", "") - contentPartIds, contentPartInstructions = self._extractContentPartInfo(chapter) - - # Create task for parallel processing with semaphore - async def processChapterWithSemaphore(chapter, chapterIndex, chapterId, chapterLevel, chapterTitle, generationHint, contentPartIds, contentPartInstructions, docLanguage, docFormat): - checkWorkflowStopped(self.services) - async with semaphore: - return await self._generateSingleChapterSectionsStructure( - chapter=chapter, - chapterIndex=chapterIndex, - chapterId=chapterId, - chapterLevel=chapterLevel, - chapterTitle=chapterTitle, - generationHint=generationHint, - contentPartIds=contentPartIds, - contentPartInstructions=contentPartInstructions, - contentParts=contentParts, - userPrompt=userPrompt, - language=docLanguage, # Use document-specific language - outputFormat=docFormat, # Use document-specific format - parentOperationId=parentOperationId, - totalChapters=totalChapters - ) - - task = processChapterWithSemaphore( - chapter, chapterIndex, chapterId, chapterLevel, chapterTitle, generationHint, contentPartIds, contentPartInstructions, docLanguage, docFormat - ) - chapterTasks.append((chapterIndex, chapter, task)) - - # Execute all chapter tasks in parallel with concurrency control - if chapterTasks: - # Create list of tasks (without indices for gather) - tasks = [task for _, _, task in chapterTasks] - - # Execute in parallel with error handling - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Process results in order and handle errors - for (originalIndex, originalChapter, _), result in zip(chapterTasks, results): - if isinstance(result, Exception): - logger.error(f"Error processing chapter {originalChapter.get('id')}: {str(result)}") - # Chapter already has empty sections set by _generateSingleChapterSectionsStructure - # Continue processing other chapters - - return chapterStructure - - async def _processAiResponseForSection( - self, - aiResponse: Any, - contentType: str, - operationType: OperationTypeEnum, - sectionId: str, - generationHint: str, - generatedElements: List[Dict[str, Any]], - section: Dict[str, Any] - ) -> List[Dict[str, Any]]: - """ - Helper method to process AI response and extract elements. - Handles both IMAGE_GENERATE and DATA_ANALYSE operation types. - """ - elements = [] - - # Handle IMAGE_GENERATE differently - returns image data directly - if contentType == "image" and operationType == OperationTypeEnum.IMAGE_GENERATE: - import base64 - base64Data = "" - - # Convert image data to base64 string if needed - if isinstance(aiResponse.content, bytes): - base64Data = base64.b64encode(aiResponse.content).decode('utf-8') - elif isinstance(aiResponse.content, str): - # Check if it's already a JSON structure - try: - jsonContent = json.loads(self.services.utils.jsonExtractString(aiResponse.content)) - if isinstance(jsonContent, dict) and jsonContent.get("type") == "image": - elements.append(jsonContent) - logger.debug("AI returned proper JSON image structure") - base64Data = None # Signal that image was already processed - elif isinstance(jsonContent, list) and len(jsonContent) > 0: - if isinstance(jsonContent[0], dict) and jsonContent[0].get("type") == "image": - elements.extend(jsonContent) - logger.debug("AI returned proper JSON image structure in list") - base64Data = None # Signal that image was already processed - else: - base64Data = "" # Continue with normal processing - else: - base64Data = "" # Continue with normal processing - except (json.JSONDecodeError, ValueError, AttributeError): - base64Data = "" # Will be processed below - - # Process base64 if not already handled above - if base64Data is None: - # Already processed as JSON, skip base64 processing - pass - elif aiResponse.content.startswith("data:image/"): - # Extract base64 from data URI - base64Data = aiResponse.content.split(",", 1)[1] - else: - content_stripped = aiResponse.content.strip() - if len(content_stripped) > 100 and all(c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r\t " for c in content_stripped[:200]): - base64Data = content_stripped.replace("\n", "").replace("\r", "").replace("\t", "").replace(" ", "") - else: - base64Data = aiResponse.content - else: - base64Data = "" - - # Always create proper JSON structure for images (if not already processed) - if base64Data is None: - # Image already processed as JSON, skip - pass - elif base64Data: - # Get caption from section if available - caption = section.get("caption") or section.get("metadata", {}).get("caption") or "" - elements.append({ - "type": "image", - "content": { - "base64Data": base64Data, - "altText": generationHint or "Generated image", - "caption": caption # Use caption from section if available - }, - "caption": caption # Also at element level for compatibility - }) - logger.debug(f"Created proper JSON image structure with base64Data length: {len(base64Data)}") - else: - logger.warning(f"IMAGE_GENERATE returned empty or invalid content for section {sectionId}") - elements.append({ - "type": "error", - "message": f"Image generation returned empty or invalid content", - "sectionId": sectionId - }) - else: - # For non-image content: Use already parsed elements from _callAiWithLooping - if generatedElements: - elements.extend(generatedElements) - else: - # Fallback: Try to parse JSON response directly with repair logic - try: - from modules.shared.jsonUtils import tryParseJson, repairBrokenJson - - # Use tryParseJson which handles extraction and basic parsing - fallbackElements, parseError, cleanedStr = tryParseJson(aiResponse.content) - - # If parsing failed, try repair - if parseError and isinstance(aiResponse.content, str): - logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}") - repairedJson = repairBrokenJson(aiResponse.content) - if repairedJson: - fallbackElements = repairedJson - parseError = None - logger.info(f"Successfully repaired JSON for section {sectionId}") - - if parseError: - raise parseError - - if isinstance(fallbackElements, list): - elements.extend(fallbackElements) - elif isinstance(fallbackElements, dict) and "elements" in fallbackElements: - elements.extend(fallbackElements["elements"]) - elif isinstance(fallbackElements, dict) and fallbackElements.get("type"): - elements.append(fallbackElements) - except (json.JSONDecodeError, ValueError) as json_error: - logger.error(f"Error parsing JSON response for section {sectionId}: {str(json_error)}") - elements.append({ - "type": "error", - "message": f"Failed to parse JSON response: {str(json_error)}", - "sectionId": sectionId - }) - - return elements - - async def _processSingleSection( - self, - section: Dict[str, Any], - sectionIndex: int, - totalSections: int, - chapterIndex: int, - totalChapters: int, - chapterId: str, - chapterOperationId: str, - fillOperationId: str, - contentParts: List[ContentPart], - userPrompt: str, - all_sections_list: List[Dict[str, Any]], - language: str, - outputFormat: str = "txt", - calculateOverallProgress: callable = None - ) -> List[Dict[str, Any]]: - """ - Process a single section and return its elements. - Used for parallel processing of sections within a chapter. - """ - sectionId = section.get("id") - sectionTitle = section.get("title", sectionId) - contentPartIds = section.get("contentPartIds", []) - contentFormats = section.get("contentFormats", {}) - generationHint = section.get("generationHint") or section.get("generation_hint") - contentType = section.get("content_type", "paragraph") - useAiCall = section.get("useAiCall", False) - - # Update overall progress at start of section - overallProgress = calculateOverallProgress(chapterIndex - 1, totalChapters, sectionIndex, totalSections) - self.services.chat.progressLogUpdate( - fillOperationId, - overallProgress, - f"Chapter {chapterIndex}/{totalChapters}, Section {sectionIndex + 1}/{totalSections}: {sectionTitle}" - ) - - # WICHTIG: Wenn keine ContentParts vorhanden sind UND kein generationHint, kann kein AI-Call gemacht werden - if len(contentPartIds) == 0 and not generationHint: - useAiCall = False - logger.debug(f"Section {sectionId}: No content parts and no generation hint, setting useAiCall=False") - elif len(contentPartIds) == 0 and generationHint and not useAiCall: - useAiCall = True - logger.info(f"Section {sectionId}: Overriding useAiCall=True (has generationHint but no content parts)") - - elements = [] - - # Prüfe ob Aggregation nötig ist - needsAggregation = self._needsAggregation( - contentType=contentType, - contentPartCount=len(contentPartIds) - ) - - logger.info(f"Processing section {sectionId}: contentType={contentType}, contentPartCount={len(contentPartIds)}, useAiCall={useAiCall}, needsAggregation={needsAggregation}, hasGenerationHint={bool(generationHint)}") - - try: - if needsAggregation and useAiCall: - # Aggregation: Alle Parts zusammen verarbeiten - sectionParts = [ - self._findContentPartById(pid, contentParts) - for pid in contentPartIds - ] - sectionParts = [p for p in sectionParts if p is not None] - - if sectionParts: - # Filtere nur extracted Parts für Aggregation (reference/object werden separat behandelt) - extractedParts = [ - p for p in sectionParts - if contentFormats.get(p.id, p.metadata.get("contentFormat")) == "extracted" - ] - nonExtractedParts = [ - p for p in sectionParts - if contentFormats.get(p.id, p.metadata.get("contentFormat")) != "extracted" - ] - - # Verarbeite non-extracted Parts separat (reference, object) - for part in nonExtractedParts: - contentFormat = contentFormats.get(part.id, part.metadata.get("contentFormat")) - - if contentFormat == "reference": - elements.append({ - "type": "reference", - "documentReference": part.metadata.get("documentReference"), - "label": part.metadata.get("usageHint", part.label) - }) - elif contentFormat == "object": - if part.typeGroup == "image": - # Validate that image data exists - if not part.data: - logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (object format). Skipping image element.") - elements.append({ - "type": "error", - "message": f"Image ContentPart {part.id} has no data", - "sectionId": sectionId - }) - else: - # Get caption from section (priority: section.caption > part.metadata.caption) - caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") - elements.append({ - "type": "image", - "content": { - "base64Data": part.data, - "altText": part.metadata.get("usageHint", part.label), - "caption": caption # Use caption from section - }, - "caption": caption # Also at element level for compatibility - }) - else: - elements.append({ - "type": part.typeGroup, - "content": { - "data": part.data, - "mimeType": part.mimeType, - "label": part.metadata.get("usageHint", part.label) - } - }) - - # Extract images with Vision AI if needed (before aggregation) - processedExtractedParts = [] - for part in extractedParts: - # Check if this is an image that needs Vision AI extraction - if (part.typeGroup == "image" and - part.metadata.get("needsVisionExtraction") == True and - part.metadata.get("intent") == "extract"): - - logger.info(f"Section {sectionId}: Extracting text from image {part.id} using Vision AI") - try: - extractionPrompt = part.metadata.get("extractionPrompt") or "Extract all text content from this image. Return only the extracted text, no additional formatting." - - # Write debug file for image extraction prompt - if self.services and hasattr(self.services, 'utils') and hasattr(self.services.utils, 'writeDebugFile'): - try: - partId = part.id[:8] if part.id else "unknown" - partLabelSafe = (part.label or "image").replace(" ", "_").replace("/", "_").replace("\\", "_")[:30] - debugPrefix = f"extraction_image_{partId}_{partLabelSafe}" - self.services.utils.writeDebugFile(extractionPrompt, f"{debugPrefix}_prompt") - logger.debug(f"Wrote image extraction prompt debug file: {debugPrefix}_prompt") - except Exception as debugError: - logger.warning(f"Failed to write image extraction debug file: {str(debugError)}") - - # Call Vision AI to extract text from image - visionRequest = AiCallRequest( - prompt=extractionPrompt, - context="", - options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), - contentParts=[part] - ) - - checkWorkflowStopped(self.services) - visionResponse = await self.aiService.callAi(visionRequest) - - # Write debug file for image extraction response - if self.services and hasattr(self.services, 'utils') and hasattr(self.services.utils, 'writeDebugFile'): - try: - partId = part.id[:8] if part.id else "unknown" - partLabelSafe = (part.label or "image").replace(" ", "_").replace("/", "_").replace("\\", "_")[:30] - debugPrefix = f"extraction_image_{partId}_{partLabelSafe}" - responseContent = visionResponse.content if visionResponse and visionResponse.content else "" - self.services.utils.writeDebugFile(responseContent, f"{debugPrefix}_response") - logger.debug(f"Wrote image extraction response debug file: {debugPrefix}_response") - except Exception as debugError: - logger.warning(f"Failed to write image extraction response debug file: {str(debugError)}") - - if visionResponse and visionResponse.content: - # Create text part with extracted content - textPart = ContentPart( - id=f"vision_extracted_{part.id}", - label=f"Extracted text from {part.label or 'Image'}", - typeGroup="text", - mimeType="text/plain", - data=visionResponse.content.strip(), - metadata={ - **part.metadata, - "contentFormat": "extracted", - "extractionMethod": "vision", - "sourceImagePartId": part.id, - "needsVisionExtraction": False # Already extracted - } - ) - processedExtractedParts.append(textPart) - logger.info(f"✅ Extracted text from image {part.id}: {len(visionResponse.content)} chars") - else: - logger.warning(f"⚠️ Vision AI extraction returned no content for image {part.id}") - # Keep original image part, but mark extraction as attempted - part.metadata["needsVisionExtraction"] = False - part.metadata["visionExtractionFailed"] = True - processedExtractedParts.append(part) - except Exception as e: - logger.error(f"❌ Vision AI extraction failed for image {part.id}: {str(e)}") - # Keep original image part, but mark extraction as attempted - part.metadata["needsVisionExtraction"] = False - part.metadata["visionExtractionFailed"] = True - processedExtractedParts.append(part) - else: - # Not an image needing extraction, or already processed - processedExtractedParts.append(part) - - # Aggregiere extracted Parts mit AI (now with Vision-extracted text parts) - if processedExtractedParts: - logger.debug(f"Section {sectionId}: Aggregating {len(processedExtractedParts)} extracted parts with AI") - isAggregation = True - generationPrompt, templateStructure = self._buildSectionGenerationPrompt( - section=section, - contentParts=processedExtractedParts, - userPrompt=userPrompt, - generationHint=generationHint, - allSections=all_sections_list, - sectionIndex=sectionIndex, - isAggregation=isAggregation, - language=language, - outputFormat=outputFormat - ) - - sectionOperationId = f"{fillOperationId}_section_{sectionId}" - self.services.chat.progressLogStart( - sectionOperationId, - "Section Generation (Aggregation)", - f"Section {sectionIndex + 1}/{totalSections}", - f"{sectionTitle} ({len(extractedParts)} parts)", - parentOperationId=chapterOperationId - ) - - try: - self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") - - self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") - - operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE - - if operationType == OperationTypeEnum.IMAGE_GENERATE: - maxPromptLength = 4000 - if len(generationPrompt) > maxPromptLength: - logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") - generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] - - # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - generationPrompt, - f"{chapterId}_section_{sectionId}_prompt" - ) - - request = AiCallRequest( - prompt=generationPrompt, - contentParts=[], - options=AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - ) - checkWorkflowStopped(self.services) - aiResponse = await self.aiService.callAi(request) - generatedElements = [] - - # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), - f"{chapterId}_section_{sectionId}_response" - ) - else: - # Use consolidated class method - buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation - - options = AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - - checkWorkflowStopped(self.services) - aiResponseJson = await self.aiService.callAiWithLooping( - prompt=generationPrompt, - options=options, - debugPrefix=f"{chapterId}_section_{sectionId}", - promptBuilder=buildSectionPromptWithContinuation, - promptArgs={ - "section": section, - "contentParts": extractedParts, - "userPrompt": userPrompt, - "generationHint": generationHint, - "allSections": all_sections_list, - "sectionIndex": sectionIndex, - "isAggregation": isAggregation, - "templateStructure": templateStructure, - "basePrompt": generationPrompt - }, - operationId=sectionOperationId, - userPrompt=userPrompt, - contentParts=extractedParts, - useCaseId="section_content" # REQUIRED: Explicit use case ID - ) - - try: - # Use tryParseJson which handles extraction and basic parsing - from modules.shared.jsonUtils import tryParseJson, repairBrokenJson - - # Check if response contains multiple JSON blocks (separated by --- or multiple ```json blocks) - # This can happen when AI returns multiple complete responses - if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1): - logger.info(f"Section {sectionId}: Detected multiple JSON blocks in response, attempting to merge") - generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId) - else: - parsedResponse, parseError, cleanedStr = tryParseJson(aiResponseJson) - - # If parsing failed, try repair - if parseError and isinstance(aiResponseJson, str): - logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}") - repairedJson = repairBrokenJson(aiResponseJson) - if repairedJson: - parsedResponse = repairedJson - parseError = None - logger.info(f"Successfully repaired JSON for section {sectionId}") - - if parseError: - raise parseError - - if isinstance(parsedResponse, list): - generatedElements = parsedResponse - elif isinstance(parsedResponse, dict): - if "elements" in parsedResponse: - generatedElements = parsedResponse["elements"] - elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: - firstSection = parsedResponse["sections"][0] - generatedElements = firstSection.get("elements", []) - elif parsedResponse.get("type"): - generatedElements = [parsedResponse] - else: - generatedElements = [] - else: - generatedElements = [] - - class AiResponse: - def __init__(self, content): - self.content = content - - aiResponse = AiResponse(aiResponseJson) - except Exception as parseError: - logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") - class AiResponse: - def __init__(self, content): - self.content = content - aiResponse = AiResponse(aiResponseJson) - generatedElements = [] - - self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") - # Note: Debug files are written by _callAiWithLooping using debugPrefix - - self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") - - # Process AI response - responseElements = await self._processAiResponseForSection( - aiResponse=aiResponse, - contentType=contentType, - operationType=operationType, - sectionId=sectionId, - generationHint=generationHint, - generatedElements=generatedElements, - section=section - ) - elements.extend(responseElements) - - self.services.chat.progressLogFinish(sectionOperationId, True) - - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed" - ) - - except Exception as e: - self.services.chat.progressLogFinish(sectionOperationId, False) - elements.append({ - "type": "error", - "message": f"Error generating section {sectionId}: {str(e)}", - "sectionId": sectionId - }) - logger.error(f"Error generating section {sectionId}: {str(e)}") - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" - ) - - else: - # Einzelverarbeitung: Jeder Part einzeln ODER Generation ohne ContentParts - if len(contentPartIds) == 0 and useAiCall and generationHint: - # Generate content from scratch using only generationHint - logger.debug(f"Processing section {sectionId}: No content parts, generating from generationHint only") - generationPrompt, templateStructure = self._buildSectionGenerationPrompt( - section=section, - contentParts=[], - userPrompt=userPrompt, - generationHint=generationHint, - allSections=all_sections_list, - sectionIndex=sectionIndex, - isAggregation=False, - language=language, - outputFormat=outputFormat - ) - - sectionOperationId = f"{fillOperationId}_section_{sectionId}" - self.services.chat.progressLogStart( - sectionOperationId, - "Section Generation", - f"Section {sectionIndex + 1}/{totalSections}", - f"{sectionTitle} (from generationHint)", - parentOperationId=chapterOperationId - ) - - try: - self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") - - self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") - - operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE - - if operationType == OperationTypeEnum.IMAGE_GENERATE: - maxPromptLength = 4000 - if len(generationPrompt) > maxPromptLength: - logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") - generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] - - # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - generationPrompt, - f"{chapterId}_section_{sectionId}_prompt" - ) - - request = AiCallRequest( - prompt=generationPrompt, - contentParts=[], - options=AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - ) - aiResponse = await self.aiService.callAi(request) - generatedElements = [] - - # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), - f"{chapterId}_section_{sectionId}_response" - ) - else: - isAggregation = False - - # Use consolidated class method - buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation - - options = AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - - aiResponseJson = await self.aiService.callAiWithLooping( - prompt=generationPrompt, - options=options, - debugPrefix=f"{chapterId}_section_{sectionId}", - promptBuilder=self.buildSectionPromptWithContinuation, - promptArgs={ - "section": section, - "contentParts": [], - "userPrompt": userPrompt, - "generationHint": generationHint, - "allSections": all_sections_list, - "sectionIndex": sectionIndex, - "isAggregation": isAggregation, - "templateStructure": templateStructure, - "basePrompt": generationPrompt, - "language": language - }, - operationId=sectionOperationId, - userPrompt=userPrompt, - contentParts=[], - useCaseId="section_content" # REQUIRED: Explicit use case ID - ) - - try: - parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) - if isinstance(parsedResponse, list): - generatedElements = parsedResponse - elif isinstance(parsedResponse, dict): - if "elements" in parsedResponse: - generatedElements = parsedResponse["elements"] - elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: - firstSection = parsedResponse["sections"][0] - generatedElements = firstSection.get("elements", []) - elif parsedResponse.get("type"): - generatedElements = [parsedResponse] - else: - generatedElements = [] - else: - generatedElements = [] - - class AiResponse: - def __init__(self, content): - self.content = content - - aiResponse = AiResponse(aiResponseJson) - except Exception as parseError: - logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") - class AiResponse: - def __init__(self, content): - self.content = content - aiResponse = AiResponse(aiResponseJson) - generatedElements = [] - - self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") - # Note: Debug files are written by _callAiWithLooping using debugPrefix - - self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") - - responseElements = await self._processAiResponseForSection( - aiResponse=aiResponse, - contentType=contentType, - operationType=operationType, - sectionId=sectionId, - generationHint=generationHint, - generatedElements=generatedElements, - section=section - ) - elements.extend(responseElements) - - self.services.chat.progressLogFinish(sectionOperationId, True) - - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed" - ) - - except Exception as e: - self.services.chat.progressLogFinish(sectionOperationId, False) - elements.append({ - "type": "error", - "message": f"Error generating section {sectionId}: {str(e)}", - "sectionId": sectionId - }) - logger.error(f"Error generating section {sectionId}: {str(e)}") - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" - ) - - # Einzelverarbeitung: Jeder Part einzeln - for partId in contentPartIds: - part = self._findContentPartById(partId, contentParts) - if not part: - continue - - contentFormat = contentFormats.get(partId, part.metadata.get("contentFormat")) - - if contentFormat == "reference": - elements.append({ - "type": "reference", - "documentReference": part.metadata.get("documentReference"), - "label": part.metadata.get("usageHint", part.label) - }) - - elif contentFormat == "object": - if part.typeGroup == "image": - # Validate that image data exists - if not part.data: - logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (object format). Skipping image element.") - elements.append({ - "type": "error", - "message": f"Image ContentPart {part.id} has no data", - "sectionId": sectionId - }) - else: - # Get caption from section (priority: section.caption > part.metadata.caption) - caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") - elements.append({ - "type": "image", - "content": { - "base64Data": part.data, - "altText": part.metadata.get("usageHint", part.label), - "caption": caption # Use caption from section - }, - "caption": caption # Also at element level for compatibility - }) - else: - elements.append({ - "type": part.typeGroup, - "content": { - "data": part.data, - "mimeType": part.mimeType, - "label": part.metadata.get("usageHint", part.label) - } - }) - - elif contentFormat == "extracted": - # CRITICAL: If useAiCall is true, extracted parts are used as input for AI generation - # and should NOT be added as elements. Only add extracted text as element if useAiCall is false. - if useAiCall: - # Extracted part will be used as input for AI call - skip adding as element - logger.debug(f"Section {sectionId}: Skipping extracted part {part.id} as element (useAiCall=true, will be used as AI input)") - # Continue to process this part for AI call, but don't add as element yet - # Check if this is an image that needs Vision AI extraction - originalPartId = part.id - if (part.typeGroup == "image" and - part.metadata.get("needsVisionExtraction") == True and - part.metadata.get("intent") == "extract"): - - logger.info(f"Section {sectionId}: Extracting text from single image {part.id} using Vision AI") - try: - extractionPrompt = part.metadata.get("extractionPrompt") or "Extract all text content from this image. Return only the extracted text, no additional formatting." - - # Call Vision AI to extract text from image - visionRequest = AiCallRequest( - prompt=extractionPrompt, - context="", - options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), - contentParts=[part] - ) - - checkWorkflowStopped(self.services) - visionResponse = await self.aiService.callAi(visionRequest) - - if visionResponse and visionResponse.content: - # Replace image part with text part for further processing - part = ContentPart( - id=f"vision_extracted_{originalPartId}", - label=f"Extracted text from {part.label or 'Image'}", - typeGroup="text", - mimeType="text/plain", - data=visionResponse.content.strip(), - metadata={ - **part.metadata, - "contentFormat": "extracted", - "extractionMethod": "vision", - "sourceImagePartId": originalPartId, - "needsVisionExtraction": False # Already extracted - } - ) - logger.info(f"✅ Extracted text from image {originalPartId}: {len(visionResponse.content)} chars") - else: - logger.warning(f"⚠️ Vision AI extraction returned no content for image {originalPartId}") - part.metadata["needsVisionExtraction"] = False - part.metadata["visionExtractionFailed"] = True - except Exception as e: - logger.error(f"❌ Vision AI extraction failed for image {originalPartId}: {str(e)}") - part.metadata["needsVisionExtraction"] = False - part.metadata["visionExtractionFailed"] = True - - if useAiCall and generationHint: - # AI-Call mit einzelnen ContentPart (now may be text part after Vision extraction) - logger.debug(f"Processing section {sectionId}: Single extracted part with AI call") - generationPrompt, templateStructure = self._buildSectionGenerationPrompt( - section=section, - contentParts=[part], - userPrompt=userPrompt, - generationHint=generationHint, - allSections=all_sections_list, - sectionIndex=sectionIndex, - isAggregation=False, - language=language, - outputFormat=outputFormat - ) - - sectionOperationId = f"{fillOperationId}_section_{sectionId}" - self.services.chat.progressLogStart( - sectionOperationId, - "Section Generation", - f"Section {sectionIndex + 1}/{totalSections}", - f"{sectionTitle} (single part)", - parentOperationId=chapterOperationId - ) - - try: - self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") - - self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") - - operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE - - if operationType == OperationTypeEnum.IMAGE_GENERATE: - maxPromptLength = 4000 - if len(generationPrompt) > maxPromptLength: - logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") - generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] - - # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - generationPrompt, - f"{chapterId}_section_{sectionId}_prompt" - ) - - request = AiCallRequest( - prompt=generationPrompt, - contentParts=[], - options=AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - ) - aiResponse = await self.aiService.callAi(request) - generatedElements = [] - - # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) - self.services.utils.writeDebugFile( - aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), - f"{chapterId}_section_{sectionId}_response" - ) - else: - isAggregation = False - - # Use consolidated class method - buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation - - options = AiCallOptions( - operationType=operationType, - priority=PriorityEnum.BALANCED, - processingMode=ProcessingModeEnum.DETAILED - ) - - aiResponseJson = await self.aiService.callAiWithLooping( - prompt=generationPrompt, - options=options, - debugPrefix=f"{chapterId}_section_{sectionId}", - promptBuilder=self.buildSectionPromptWithContinuation, - promptArgs={ - "section": section, - "contentParts": [part], - "userPrompt": userPrompt, - "generationHint": generationHint, - "allSections": all_sections_list, - "sectionIndex": sectionIndex, - "isAggregation": isAggregation, - "services": self.services, - "templateStructure": templateStructure, - "basePrompt": generationPrompt, - "language": language - }, - operationId=sectionOperationId, - userPrompt=userPrompt, - contentParts=[part], - useCaseId="section_content" # REQUIRED: Explicit use case ID - ) - - try: - parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) - if isinstance(parsedResponse, list): - generatedElements = parsedResponse - elif isinstance(parsedResponse, dict): - if "elements" in parsedResponse: - generatedElements = parsedResponse["elements"] - elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: - firstSection = parsedResponse["sections"][0] - generatedElements = firstSection.get("elements", []) - elif parsedResponse.get("type"): - generatedElements = [parsedResponse] - else: - generatedElements = [] - else: - generatedElements = [] - - class AiResponse: - def __init__(self, content): - self.content = content - - aiResponse = AiResponse(aiResponseJson) - except Exception as parseError: - logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") - class AiResponse: - def __init__(self, content): - self.content = content - aiResponse = AiResponse(aiResponseJson) - generatedElements = [] - - self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") - # Note: Debug files are written by _callAiWithLooping using debugPrefix - - self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") - - responseElements = await self._processAiResponseForSection( - aiResponse=aiResponse, - contentType=contentType, - operationType=operationType, - sectionId=sectionId, - generationHint=generationHint, - generatedElements=generatedElements, - section=section - ) - elements.extend(responseElements) - - self.services.chat.progressLogFinish(sectionOperationId, True) - - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed" - ) - - except Exception as e: - self.services.chat.progressLogFinish(sectionOperationId, False) - elements.append({ - "type": "error", - "message": f"Error generating section {sectionId}: {str(e)}", - "sectionId": sectionId - }) - logger.error(f"Error generating section {sectionId}: {str(e)}") - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" - ) - else: - # Füge extrahierten Content direkt hinzu (kein AI-Call) - # CRITICAL: If content_type is "image", we must render an image, not extracted text - if contentType == "image": - # Section wants to display an image - find the image part - if part.typeGroup == "image": - # Direct image part - use it - logger.debug(f"Processing section {sectionId}: Single extracted IMAGE part WITHOUT AI call") - # Validate that image data exists - if not part.data: - logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (extracted format without AI call). Skipping image element.") - elements.append({ - "type": "error", - "message": f"Image ContentPart {part.id} has no data", - "sectionId": sectionId - }) - else: - # Get caption from section (priority: section.caption > part.metadata.caption) - caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") - elements.append({ - "type": "image", - "content": { - "base64Data": part.data, - "altText": part.metadata.get("usageHint", part.label), - "caption": caption # Use caption from section - }, - "caption": caption # Also at element level for compatibility - }) - elif part.typeGroup == "text" and part.metadata.get("sourceImagePartId"): - # This is a vision-extracted text part - find the original image object part - sourceImagePartId = part.metadata.get("sourceImagePartId") - logger.debug(f"Processing section {sectionId}: Found vision-extracted text part, looking for original image object part: {sourceImagePartId}") - - # Try to find the object part (format: "obj_...") - objectPartId = part.metadata.get("relatedObjectPartId") - objectPart = None - - if objectPartId: - objectPart = self._findContentPartById(objectPartId, contentParts) - - # If not found via metadata, search through all contentParts for object part - if not objectPart: - # Search for object part that references the source image part ID - for candidatePart in contentParts: - if (candidatePart.metadata.get("contentFormat") == "object" and - candidatePart.typeGroup == "image" and - sourceImagePartId in candidatePart.id): - objectPart = candidatePart - objectPartId = candidatePart.id - logger.debug(f"Section {sectionId}: Found object part {objectPartId} by searching all contentParts") - break - - if objectPart and objectPart.typeGroup == "image" and objectPart.data: - logger.info(f"Section {sectionId}: Found object part {objectPartId} for image rendering") - caption = section.get("caption") or section.get("metadata", {}).get("caption") or objectPart.metadata.get("caption", "") - elements.append({ - "type": "image", - "content": { - "base64Data": objectPart.data, - "altText": objectPart.metadata.get("usageHint", objectPart.label), - "caption": caption - }, - "caption": caption - }) - else: - logger.warning(f"Section {sectionId}: No object part found for vision-extracted text part {part.id} (sourceImagePartId={sourceImagePartId}), cannot render image") - elements.append({ - "type": "error", - "message": f"Cannot render image: no object part found for extracted text part (sourceImagePartId={sourceImagePartId})", - "sectionId": sectionId - }) - else: - logger.warning(f"Section {sectionId}: ContentPart {part.id} is not an image (typeGroup={part.typeGroup}), but section content_type is 'image'. Cannot render image.") - elements.append({ - "type": "error", - "message": f"Cannot render image: ContentPart is not an image type", - "sectionId": sectionId - }) - else: - # content_type is not "image" - add extracted text as normal - if part.typeGroup == "image": - logger.debug(f"Processing section {sectionId}: Single extracted IMAGE part WITHOUT AI call") - # Validate that image data exists - if not part.data: - logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (extracted format without AI call). Skipping image element.") - elements.append({ - "type": "error", - "message": f"Image ContentPart {part.id} has no data", - "sectionId": sectionId - }) - else: - # Get caption from section (priority: section.caption > part.metadata.caption) - caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") - elements.append({ - "type": "image", - "content": { - "base64Data": part.data, - "altText": part.metadata.get("usageHint", part.label), - "caption": caption # Use caption from section - }, - "caption": caption # Also at element level for compatibility - }) - else: - logger.debug(f"Processing section {sectionId}: Single extracted TEXT part WITHOUT AI call") - elements.append({ - "type": "extracted_text", - "content": part.data, - "source": part.metadata.get("documentId"), - "extractionPrompt": part.metadata.get("extractionPrompt") - }) - - # Update progress after section completion - chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 - self.services.chat.progressLogUpdate( - chapterOperationId, - chapterProgress, - f"Section {sectionIndex + 1}/{totalSections} completed" - ) - - overallProgress = calculateOverallProgress(chapterIndex - 1, totalChapters, sectionIndex + 1, totalSections) - self.services.chat.progressLogUpdate( - fillOperationId, - overallProgress, - f"Chapter {chapterIndex}/{totalChapters}, Section {sectionIndex + 1}/{totalSections} completed" - ) - - except Exception as e: - logger.error(f"Unexpected error processing section {sectionId}: {str(e)}") - elements.append({ - "type": "error", - "message": f"Unexpected error processing section {sectionId}: {str(e)}", - "sectionId": sectionId - }) - - return elements - - async def _fillChapterSections( - self, - chapterStructure: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - parentOperationId: str, - language: str, - options: Optional[AiCallOptions] = None - ) -> Dict[str, Any]: - """ - Phase 5D.2: Füllt Sections mit ContentParts. - """ - - # Sammle alle Sections für Kontext-Informationen (für alle Sections) - all_sections_list = [] - for doc in chapterStructure.get("documents", []): - for chapter in doc.get("chapters", []): - for section in chapter.get("sections", []): - all_sections_list.append(section) - - # Berechne Gesamtanzahl Chapters für Progress-Tracking - totalChapters = sum(len(doc.get("chapters", [])) for doc in chapterStructure.get("documents", [])) - fillOperationId = parentOperationId - - # Get concurrency limit for sections - maxConcurrent = self._getMaxConcurrentGeneration(options) - sectionSemaphore = asyncio.Semaphore(maxConcurrent) - - # Collect ALL sections from ALL chapters for fully parallel processing - # Each task carries: (docId, chapterId, chapterTitle, sectionIndex, section, docLanguage) - allSectionTasks = [] - totalSections = len(all_sections_list) - completedSections = [0] # Mutable counter for progress tracking - - for doc in chapterStructure.get("documents", []): - docId = doc.get("id", "unknown") - docLanguage = self._getDocumentLanguage(chapterStructure, docId) - docFormat = doc.get("outputFormat", "txt") # Get output format for this document - - for chapter in doc.get("chapters", []): - chapterId = chapter.get("id", "unknown") - chapterTitle = chapter.get("title", "Untitled Chapter") - sections = chapter.get("sections", []) - chapterSectionCount = len(sections) - - for sectionIndex, section in enumerate(sections): - allSectionTasks.append({ - "docId": docId, - "chapterId": chapterId, - "chapterTitle": chapterTitle, - "sectionIndex": sectionIndex, - "chapterSectionCount": chapterSectionCount, - "section": section, - "docLanguage": docLanguage, - "docFormat": docFormat # Include output format - }) - - logger.info(f"Starting FULLY PARALLEL section generation: {totalSections} sections across {totalChapters} chapters") - - # Create task wrapper for each section with progress tracking - async def processSectionWithSemaphore(taskInfo): - checkWorkflowStopped(self.services) - async with sectionSemaphore: - result = await self._processSingleSection( - section=taskInfo["section"], - sectionIndex=taskInfo["sectionIndex"], - totalSections=taskInfo["chapterSectionCount"], - chapterIndex=0, # Not used for sequential logic anymore - totalChapters=totalChapters, - chapterId=taskInfo["chapterId"], - chapterOperationId=fillOperationId, # Use fillOperationId as parent (no chapter-level ops in parallel mode) - fillOperationId=fillOperationId, - contentParts=contentParts, - userPrompt=userPrompt, - all_sections_list=all_sections_list, - language=taskInfo["docLanguage"], - outputFormat=taskInfo.get("docFormat", "txt"), # Pass output format - calculateOverallProgress=lambda *args: completedSections[0] / totalSections if totalSections > 0 else 1.0 - ) - - # Update progress after each section completes - completedSections[0] += 1 - overallProgress = completedSections[0] / totalSections if totalSections > 0 else 1.0 - sectionId = taskInfo["section"].get("id", "unknown") - self.services.chat.progressLogUpdate( - fillOperationId, - overallProgress, - f"Section {completedSections[0]}/{totalSections} completed: {sectionId}" - ) - - return result - - # Create all tasks - tasks = [processSectionWithSemaphore(taskInfo) for taskInfo in allSectionTasks] - - # Execute ALL sections in parallel with concurrency control - if tasks: - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Assign results back to sections - for taskInfo, result in zip(allSectionTasks, results): - section = taskInfo["section"] - if isinstance(result, Exception): - logger.error(f"Error processing section {section.get('id')}: {str(result)}") - section["elements"] = [{ - "type": "error", - "message": f"Error processing section: {str(result)}", - "sectionId": section.get("id") - }] - else: - section["elements"] = result if result is not None else [] - - logger.info(f"Completed FULLY PARALLEL section generation: {totalSections} sections") - - return chapterStructure - - def _addContentPartsMetadata( - self, - structure: Dict[str, Any], - contentParts: List[ContentPart] - ) -> Dict[str, Any]: - """ - Fügt ContentParts-Metadaten zur Struktur hinzu, wenn contentPartIds vorhanden sind. - Dies hilft der Validierung, den Kontext der ContentParts zu verstehen. - """ - # Erstelle Mapping von ContentPart-ID zu Metadaten - contentPartsMap = {} - for part in contentParts: - contentPartsMap[part.id] = { - "id": part.id, - "format": part.metadata.get("contentFormat", "unknown"), - "type": part.typeGroup, - "mimeType": part.mimeType, - "originalFileName": part.metadata.get("originalFileName"), - "usageHint": part.metadata.get("usageHint"), - "documentId": part.metadata.get("documentId"), - "dataSize": len(str(part.data)) if part.data else 0 - } - - # Füge Metadaten zu Sections hinzu, die contentPartIds haben - for doc in structure.get("documents", []): - # Prüfe ob Chapters vorhanden sind (neue Struktur) - if "chapters" in doc: - for chapter in doc.get("chapters", []): - # Füge Metadaten zu Chapter-Level contentPartIds hinzu - chapterContentPartIds, _ = self._extractContentPartInfo(chapter) - if chapterContentPartIds: - chapter["contentPartsMetadata"] = [] - for partId in chapterContentPartIds: - if partId in contentPartsMap: - chapter["contentPartsMetadata"].append(contentPartsMap[partId]) - - # Füge Metadaten zu Sections hinzu - for section in chapter.get("sections", []): - contentPartIds = section.get("contentPartIds", []) - if contentPartIds: - section["contentPartsMetadata"] = [] - for partId in contentPartIds: - if partId in contentPartsMap: - section["contentPartsMetadata"].append(contentPartsMap[partId]) - - return structure - - def _flattenChaptersToSections( - self, - chapterStructure: Dict[str, Any] - ) -> Dict[str, Any]: - """ - Flattening: Konvertiert Chapters zu finaler Section-Struktur. - Jedes Chapter wird zu einer Heading-Section (Level 1) + dessen Sections. - - Chapters are the main structure elements (heading level 1). - All section headings with level < 2 are adjusted to level 2. - """ - result = { - "metadata": chapterStructure.get("metadata", {}), - "documents": [] - } - - for doc in chapterStructure.get("documents", []): - flattened_doc = { - "id": doc.get("id"), - "title": doc.get("title"), - "filename": doc.get("filename"), - "outputFormat": doc.get("outputFormat"), # Preserve from Phase 3 - "language": doc.get("language"), # Preserve from Phase 3 - "sections": [] - } - - for chapter in doc.get("chapters", []): - # 1. Vordefinierte Heading-Section für Chapter-Title (ALWAYS Level 1) - heading_section = { - "id": f"{chapter['id']}_heading", - "content_type": "heading", - "elements": [{ - "type": "heading", - "content": { - "text": chapter.get("title", ""), - "level": 1 # Chapters are always level 1 - } - }] - } - flattened_doc["sections"].append(heading_section) - - # 2. Generierte Sections - adjust heading levels - for section in chapter.get("sections", []): - # CRITICAL: Ensure elements are preserved when flattening - # _adjustSectionHeadingLevels uses deepcopy which should preserve elements, - # but verify that elements exist in the source section - adjusted_section = self._adjustSectionHeadingLevels(section) - # Ensure elements are preserved (deepcopy should handle this, but double-check) - if "elements" in section and "elements" not in adjusted_section: - adjusted_section["elements"] = section["elements"] - flattened_doc["sections"].append(adjusted_section) - - result["documents"].append(flattened_doc) - - return result - - def _adjustSectionHeadingLevels(self, section: Dict[str, Any]) -> Dict[str, Any]: - """ - Adjust heading levels in sections: sections with type heading and level < 2 are changed to level 2. - Only chapter headings have level 1. - """ - adjusted_section = copy.deepcopy(section) - - # Check if this is a heading section - if adjusted_section.get("content_type") == "heading": - elements = adjusted_section.get("elements", []) - for element in elements: - if isinstance(element, dict) and element.get("type") == "heading": - content = element.get("content", {}) - if isinstance(content, dict): - level = content.get("level", 1) - # If level < 2, change to level 2 (only chapters have level 1) - if level < 2: - content["level"] = 2 - - return adjusted_section - - def _buildChapterSectionsStructurePrompt( - self, - chapterId: str, - chapterLevel: int, - chapterTitle: str, - generationHint: str, - contentPartIds: List[str], - contentPartInstructions: Dict[str, Any], - contentParts: List[ContentPart], - userPrompt: str, - language: str = "en", - outputFormat: str = "txt" - ) -> str: - """Baue Prompt für Chapter-Sections-Struktur-Generierung, querying renderer for accepted section types.""" - # Baue ContentParts-Index (nur IDs, keine Previews!) - contentPartsIndex = "" - for partId in contentPartIds: - part = self._findContentPartById(partId, contentParts) - if not part: - # Part not found - try to show info from chapter structure - partInfo = contentPartInstructions.get(partId, {}) - if partInfo: - logger.warning(f"Chapter {chapterId}: ContentPart {partId} not found in contentParts list, but has chapter structure info.") - contentPartsIndex += f"\n- ContentPart ID: {partId}\n" - if "instruction" in partInfo: - contentPartsIndex += f" Instruction: {partInfo['instruction']}\n" - if "caption" in partInfo: - contentPartsIndex += f" Caption: {partInfo['caption']}\n" - contentPartsIndex += f" Note: ContentPart not found in contentParts list (ID may be from nested structure)\n" - continue - - contentFormat = part.metadata.get("contentFormat", "unknown") - partInfo = contentPartInstructions.get(partId, {}) - instruction = partInfo.get("instruction", "Use content as needed") - caption = partInfo.get("caption") - - contentPartsIndex += f"\n- ContentPart ID: {partId}\n" - contentPartsIndex += f" Format: {contentFormat}\n" - contentPartsIndex += f" Type: {part.typeGroup}\n" - if instruction and instruction != "Use content as needed": - contentPartsIndex += f" Instruction: {instruction}\n" - if caption: - contentPartsIndex += f" Caption: {caption}\n" - - if not contentPartsIndex: - contentPartsIndex = "\n(No content parts specified for this chapter)" - - # Query renderer for accepted section types - acceptedSectionTypes = self._getAcceptedSectionTypesForFormat(outputFormat) - - prompt = f"""TASK: Generate Chapter Sections Structure - -LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. - -CHAPTER: {chapterTitle} (Level {chapterLevel}, ID: {chapterId}) -GENERATION HINT: {generationHint} - -## CONTENT EFFICIENCY PRINCIPLES -- Generate COMPACT sections: Focus on essential information only -- AVOID creating too many sections - combine related content where possible -- Each section should serve a clear purpose with meaningful data -- If no relevant data exists for a topic, do NOT create a section for it -- Prefer ONE comprehensive section over multiple sparse sections - -**CRITICAL**: The chapter's generationHint above describes what content this chapter should generate. If the generationHint references documents/images/data, then EACH section that generates content for this chapter MUST assign the relevant ContentParts from AVAILABLE CONTENT PARTS below. - -NOTE: Chapter already has a heading section. Do NOT generate a heading for the chapter title. - -## SECTION INDEPENDENCE -- Each section is independent and self-contained -- One section does NOT have information about another section -- Each section must provide its own context and be understandable alone - -AVAILABLE CONTENT PARTS: -{contentPartsIndex} - -## CONTENT ASSIGNMENT RULE - CRITICAL -If AVAILABLE CONTENT PARTS are listed above, then EVERY section that generates content related to those ContentParts MUST assign them explicitly. - -**Assignment logic:** -- If section generates text content ABOUT a ContentPart → assign "extracted" format ContentPart with appropriate instruction -- If section DISPLAYS a ContentPart → assign "object" format ContentPart -- If section's generationHint or purpose relates to a ContentPart listed above → it MUST have contentPartIds assigned -- If chapter's generationHint references documents/images/data AND section generates content for that chapter → section MUST assign relevant ContentParts -- Empty contentPartIds [] are only allowed if section generates content WITHOUT referencing any available ContentParts AND WITHOUT relating to chapter's generationHint - -## ACCEPTED CONTENT TYPES FOR THIS FORMAT -The document output format ({outputFormat}) accepts only the following content types: -{', '.join(acceptedSectionTypes)} - -**CRITICAL**: Only create sections with content types from this list. Other types will fail. - -useAiCall RULE (simple): -- useAiCall: true → Content needs AI processing (extract, transform, generate, filter, summarize) -- useAiCall: false → Content can be inserted directly without changes (Format is "object" or "reference") - -RETURN JSON: -{{ - "sections": [ - {{ - "id": "section_1", - "content_type": "{acceptedSectionTypes[0]}", - "contentPartIds": ["extracted_part_id"], - "generationHint": "Description of what to extract or generate", - "useAiCall": true, - "elements": [] - }} - ] -}} - -**MANDATORY CONTENT ASSIGNMENT CHECK:** -For each section, verify: -1. Are ContentParts listed in AVAILABLE CONTENT PARTS above? -2. Does this section's generationHint or purpose relate to those ContentParts? -3. If YES to both → section MUST have contentPartIds assigned (cannot be empty []) -4. Assign ContentPart IDs exactly as listed in AVAILABLE CONTENT PARTS above - -IMAGE SECTIONS: -- For image sections, always provide a "caption" field with a descriptive caption for the image. - -Return only valid JSON. Do not include any explanatory text outside the JSON. -""" - return prompt - - def _getContentStructureExample(self, contentType: str) -> str: - """Get the JSON structure example for a specific content type.""" - structures = { - "table": '{{"headers": ["Column1", "Column2"], "rows": [["Value1", "Value2"], ["Value3", "Value4"]]}}', - "bullet_list": '{{"items": ["Item 1", "Item 2", "Item 3"]}}', - "heading": '{{"text": "Section Title", "level": 2}}', - "paragraph": '{{"text": "This is paragraph text."}}', - "code_block": '{{"code": "function example() {{ return true; }}", "language": "javascript"}}', - "image": '{{"base64Data": "", "altText": "Description", "caption": "Optional caption"}}' - } - return structures.get(contentType, '{{"text": ""}}') - - def _buildSectionGenerationPrompt( - self, - section: Dict[str, Any], - contentParts: List[Optional[ContentPart]], - userPrompt: str, - generationHint: str, - allSections: Optional[List[Dict[str, Any]]] = None, - sectionIndex: Optional[int] = None, - isAggregation: bool = False, - language: str = "en", - outputFormat: str = "txt" - ) -> tuple[str, str]: - """Baue Prompt für Section-Generierung mit vollständigem Kontext.""" - # Filtere None-Werte - validParts = [p for p in contentParts if p is not None] - - # Section-Metadaten - sectionId = section.get("id", "unknown") - contentType = section.get("content_type", "paragraph") - - # Baue ContentParts-Beschreibung - contentPartsText = "" - if isAggregation: - # Aggregation: ContentParts werden als Parameter übergeben, keine IDs im Prompt nötig - # Keine ContentPart-Beschreibung nötig - Daten sind bereits im Context verfügbar - contentPartsText = "" - else: - # Einzelverarbeitung: Zeige Previews - for part in validParts: - contentFormat = part.metadata.get("contentFormat", "unknown") - contentPartsText += f"\n- ContentPart {part.id}:\n" - contentPartsText += f" Format: {contentFormat}\n" - contentPartsText += f" Type: {part.typeGroup}\n" - if part.metadata.get("originalFileName"): - contentPartsText += f" Source file: {part.metadata.get('originalFileName')}\n" - - if contentFormat == "extracted": - # CRITICAL: Check if this is binary/image data - NEVER include in text prompt! - isBinaryOrImage = ( - part.typeGroup == "image" or - part.typeGroup == "binary" or - (part.mimeType and ( - part.mimeType.startswith("image/") or - part.mimeType.startswith("video/") or - part.mimeType.startswith("audio/") or - self._isBinaryMimeType(part.mimeType) - )) or - # Heuristic check: if data looks like base64 (long string with base64 chars) - (part.data and isinstance(part.data, str) and - len(part.data) > 100 and - self._looksLikeBase64(part.data)) - ) - - if isBinaryOrImage: - # NEVER include binary/base64 data in text prompt - security risk and token explosion! - dataLength = len(part.data) if part.data else 0 - contentPartsText += f" Type: {part.typeGroup}\n" - contentPartsText += f" MIME type: {part.mimeType or 'unknown'}\n" - contentPartsText += f" Data size: {dataLength} chars (binary/base64 - not shown in prompt)\n" - if part.metadata.get("needsVisionExtraction"): - contentPartsText += f" Note: Will be processed with Vision AI\n" - if part.metadata.get("usageHint"): - contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" - else: - # Only for text data: Show preview - previewLength = 1000 - if part.data: - preview = part.data[:previewLength] + "..." if len(part.data) > previewLength else part.data - contentPartsText += f" Content preview:\n```\n{preview}\n```\n" - else: - contentPartsText += f" Content: (empty)\n" - elif contentFormat == "reference": - contentPartsText += f" Reference: {part.metadata.get('documentReference')}\n" - if part.metadata.get("usageHint"): - contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" - elif contentFormat == "object": - dataLength = len(part.data) if part.data else 0 - contentPartsText += f" Object type: {part.typeGroup}\n" - contentPartsText += f" MIME type: {part.mimeType}\n" - contentPartsText += f" Data size: {dataLength} chars (base64 encoded)\n" - if part.metadata.get("usageHint"): - contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" - - # Baue Section-Kontext (vorherige und nachfolgende Sections) - contextText = "" - if allSections and sectionIndex is not None: - prevSections = [] - nextSections = [] - - if sectionIndex > 0: - for i in range(max(0, sectionIndex - 2), sectionIndex): - prevSection = allSections[i] - prevSections.append({ - "id": prevSection.get("id"), - "content_type": prevSection.get("content_type"), - "generation_hint": prevSection.get("generation_hint", "")[:100] - }) - - if sectionIndex < len(allSections) - 1: - for i in range(sectionIndex + 1, min(len(allSections), sectionIndex + 3)): - nextSection = allSections[i] - nextSections.append({ - "id": nextSection.get("id"), - "content_type": nextSection.get("content_type"), - "generation_hint": nextSection.get("generation_hint", "")[:100] - }) - - if prevSections or nextSections: - contextText = "\n## DOCUMENT CONTEXT\n" - if prevSections: - contextText += "\nPrevious sections:\n" - for prev in prevSections: - contextText += f"- {prev['id']} ({prev['content_type']}): {prev['generation_hint']}\n" - if nextSections: - contextText += "\nFollowing sections:\n" - for next in nextSections: - contextText += f"- {next['id']} ({next['content_type']}): {next['generation_hint']}\n" - - # Get accepted section types for the output format - acceptedTypesAggr = self._getAcceptedSectionTypesForFormat(outputFormat) - - # CRITICAL: If the section's content_type is not supported by the output format, - # use the first accepted type instead. E.g., CSV only supports 'table', so - # even if section says 'code_block', we must output as 'table'. - effectiveContentType = contentType - if contentType not in acceptedTypesAggr and acceptedTypesAggr: - effectiveContentType = acceptedTypesAggr[0] - logger.debug(f"Section {sectionId}: Content type '{contentType}' not supported by format '{outputFormat}', using '{effectiveContentType}' instead") - - contentStructureExample = self._getContentStructureExample(effectiveContentType) - - # Build format note for the prompt - purely dynamic from renderer - # Always show what types are accepted for this format - formatNoteAggr = f"\n- Target Output Format: {outputFormat.upper()} (accepted content types: {', '.join(acceptedTypesAggr)})" - - # Create template structure explicitly (not extracted from prompt) - # This ensures exact identity between initial and continuation prompts - templateStructure = f"""{{ - "elements": [ - {{ - "type": "{effectiveContentType}", - "content": {contentStructureExample} - }} - ] -}}""" - - if isAggregation: - prompt = f"""# TASK: Generate Section Content (Aggregation) - -Return only valid JSON. No explanatory text, no comments, no markdown formatting outside JSON. -If ContentParts have no data, return: {{"elements": [{{"type": "{effectiveContentType}", "content": {{"headers": [], "rows": []}}}}]}} - -LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. - -## SECTION METADATA -- Section ID: {sectionId} -- Content Type: {effectiveContentType} -- Generation Hint: {generationHint}{formatNoteAggr} - -## CONTENT EFFICIENCY PRINCIPLES -- Generate COMPACT content: Focus on essential facts only -- AVOID verbose text, filler phrases, or redundant explanations -- Be CONCISE and direct - every word should add value -- NO introductory phrases like "This section describes..." or "Here we present..." -- Minimize output size for efficient processing - -## INSTRUCTIONS -1. Extract all data from the context provided. Do not skip or omit any data. -2. Extract data only from the provided context. Never invent, create, or generate data that is not in the context. -3. If the context contains no data, return empty structures (empty rows array for tables). -4. Aggregate all data into one element (e.g., one table). -5. For table: Extract all rows from the context. Return {{"headers": [...], "rows": []}} only if no data exists. -6. Format based on content_type ({effectiveContentType}). -7. No HTML/styling: Plain text only, no markup. -8. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. - - -## OUTPUT FORMAT -Return a JSON object with this structure: - -{{ - "elements": [ - {{ - "type": "{effectiveContentType}", - "content": {contentStructureExample} - }} - ] -}} - -Output requirements: -- "content" must be an object (never a string) -- Return only valid JSON - no text before, no text after, no comments, no explanations -- No invented data: Return empty structures if ContentParts have no data -- Extract all data: Process every ContentPart completely and include all extracted data - -## USER REQUEST (for context) -``` -{userPrompt} -``` - -## CONTEXT -{contextText if contextText else ""} -""" - else: - # Determine if we have ContentParts or need to generate from scratch - hasContentParts = len(validParts) > 0 - - if hasContentParts: - # EXTRACT MODE: Extract data from provided ContentParts - prompt = f"""# TASK: Extract Section Content from Provided Data - -LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. - -## SECTION METADATA -- Section ID: {sectionId} -- Content Type: {effectiveContentType} -- Generation Hint: {generationHint}{formatNoteAggr} - -## CONTENT EFFICIENCY PRINCIPLES -- Generate COMPACT content: Focus on essential facts only -- AVOID verbose text, filler phrases, or redundant explanations -- Be CONCISE and direct - every word should add value -- NO introductory phrases like "This section describes..." or "Here we present..." -- Minimize output size for efficient processing - -## AVAILABLE CONTENT FOR THIS SECTION -{contentPartsText} - -## INSTRUCTIONS -1. Extract data only from provided ContentParts. Never invent or generate data. -2. If ContentParts contain no data, return empty structures (empty rows array for tables). -3. Format based on content_type ({effectiveContentType}). -4. Return only valid JSON with "elements" array. -5. No HTML/styling: Plain text only, no markup. -6. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. - -## OUTPUT FORMAT -Return a JSON object with this structure: - -{{ - "elements": [ - {{ - "type": "{effectiveContentType}", - "content": {contentStructureExample} - }} - ] -}} - -Output requirements: -- "content" must be an object (never a string) -- Return only valid JSON - no text before, no text after, no comments, no explanations, no markdown code fences -- Start with {{ and end with }} - return ONLY the JSON object itself -- No invented data: Return empty structures if ContentParts have no data - -## USER REQUEST -``` -{userPrompt} -``` - -## CONTEXT -{contextText if contextText else ""} -""" - else: - # GENERATE MODE: Generate content from scratch based on generationHint - prompt = f"""# TASK: Generate Section Content - -LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. - -## SECTION METADATA -- Section ID: {sectionId} -- Content Type: {effectiveContentType} -- Generation Hint: {generationHint}{formatNoteAggr} - -## CONTENT EFFICIENCY PRINCIPLES -- Generate COMPACT content: Focus on essential facts only -- AVOID verbose text, filler phrases, or redundant explanations -- Be CONCISE and direct - every word should add value -- NO introductory phrases like "This section describes..." or "Here we present..." -- Minimize output size for efficient processing - -## INSTRUCTIONS -1. Generate content based on the Generation Hint above. -2. Create appropriate content that matches the content_type ({effectiveContentType}). -3. The content should be relevant to the USER REQUEST and fit the context of surrounding sections. -4. Return only valid JSON with "elements" array. -5. No HTML/styling: Plain text only, no markup. -6. Keep content CONCISE - focus on substance, not length. - -## OUTPUT FORMAT -Return a JSON object with this structure: - -{{ - "elements": [ - {{ - "type": "{effectiveContentType}", - "content": {contentStructureExample} - }} - ] -}} - -Output requirements: -- "content" must be an object (never a string) -- Return only valid JSON - no text before, no text after, no comments, no explanations, no markdown code fences -- Start with {{ and end with }} - return ONLY the JSON object itself -- Generate meaningful content based on the Generation Hint - -## USER REQUEST -``` -{userPrompt} -``` - -## CONTEXT -{contextText if contextText else ""} -""" - return prompt, templateStructure - - async def buildSectionPromptWithContinuation( - self, - continuationContext: Any, - templateStructure: str, - basePrompt: str - ) -> str: - """Build section prompt with continuation context. Uses unified signature. - - Single unified implementation for all section content generation contexts. - - Note: All initial context (section, contentParts, userPrompt, etc.) is already - contained in basePrompt. This function only adds continuation-specific instructions. - """ - # Extract continuation context fields (only what's needed for continuation) - incompletePart = continuationContext.incomplete_part - lastRawJson = continuationContext.last_raw_json - - # Generate both overlap context and hierarchy context using jsonContinuation - overlapContext = "" - unifiedContext = "" - if lastRawJson: - # Get contexts directly from jsonContinuation - from modules.shared.jsonContinuation import getContexts - contexts = getContexts(lastRawJson) - overlapContext = contexts.overlapContext - unifiedContext = contexts.hierarchyContextForPrompt - elif incompletePart: - unifiedContext = incompletePart - else: - unifiedContext = "Unable to extract context - response was completely broken" - - # Build unified continuation prompt format - continuationPrompt = f"""{basePrompt} - ---- CONTINUATION REQUEST --- -The previous JSON response was incomplete. Continue from where it stopped. - -Context showing structure hierarchy with cut point: -``` -{unifiedContext} -``` - -Overlap Requirement: -To ensure proper merging, your response MUST start EXACTLY with the overlap context shown below, then continue with new content. - -Overlap context (start your response with this exact text): -```json -{overlapContext if overlapContext else "No overlap context available"} -``` - -TASK: -1. Start your response EXACTLY with the overlap context shown above (character by character) -2. Continue seamlessly from where the overlap context ends -3. Complete the remaining content following the JSON structure template above -4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects - -CRITICAL: -- Your response MUST begin with the exact overlap context text (this enables automatic merging) -- Continue seamlessly after the overlap context with new content -- Your response must be valid JSON matching the structure template above""" - return continuationPrompt - - def _extractAndMergeMultipleJsonBlocks(self, responseText: str, contentType: str, sectionId: str) -> List[Dict[str, Any]]: - """ - Extract multiple JSON blocks from response and merge them appropriately. - For tables: Merge all rows into a single table. - For other types: Combine elements. - """ - from modules.shared.jsonUtils import tryParseJson, stripCodeFences, normalizeJsonText, extractFirstBalancedJson - - # Extract all JSON blocks, handling both --- separators and multiple ```json blocks - blocks = [] - - # Strategy: Extract all ```json blocks first (most reliable), then fall back to other methods - # This handles cases where --- separators and ```json blocks are mixed - if "```json" in responseText: - # Extract all ```json blocks regardless of --- separators - jsonParts = responseText.split("```json") - for jsonPart in jsonParts[1:]: # Skip first empty part - jsonPart = "```json" + jsonPart - # Extract just the JSON block (until closing ```) - closingFence = jsonPart.find("```", 7) # Find closing ``` after "```json" - if closingFence != -1: - jsonPart = jsonPart[:closingFence + 3] - jsonPart = jsonPart.strip() - if jsonPart: - blocks.append(jsonPart) - - # If no ```json blocks found, try splitting by --- and extracting JSON - if not blocks and "---" in responseText: - parts = responseText.split("---") - for part in parts: - part = part.strip() - if not part: - continue - - # Try to extract JSON directly from this part - normalized = normalizeJsonText(part) - normalized = stripCodeFences(normalized) - jsonBlock = extractFirstBalancedJson(normalized) - if jsonBlock: - blocks.append(jsonBlock) - elif responseText.count("```json") > 1: - # Split by ```json markers (no --- separator) - parts = responseText.split("```json") - for part in parts[1:]: # Skip first empty part - part = "```json" + part - part = part.strip() - if part: - blocks.append(part) - else: - # Try to find multiple JSON objects/arrays directly - normalized = normalizeJsonText(responseText) - normalized = stripCodeFences(normalized) - - # Find all JSON blocks - start = 0 - while start < len(normalized): - # Find next JSON start - brace = normalized.find('{', start) - bracket = normalized.find('[', start) - jsonStart = -1 - if brace != -1 and (bracket == -1 or brace < bracket): - jsonStart = brace - elif bracket != -1: - jsonStart = bracket - - if jsonStart == -1: - break - - # Extract balanced JSON - jsonBlock = extractFirstBalancedJson(normalized[jsonStart:]) - if jsonBlock: - blocks.append(jsonBlock) - start = jsonStart + len(jsonBlock) - else: - break - - if not blocks: - logger.warning(f"Section {sectionId}: Could not extract multiple JSON blocks") - return [] - - logger.info(f"Section {sectionId}: Extracted {len(blocks)} JSON blocks, merging for contentType={contentType}") - - # Parse all blocks - allElements = [] - for i, block in enumerate(blocks): - parsed, parseError, _ = tryParseJson(block) - if parseError: - logger.warning(f"Section {sectionId}: Failed to parse JSON block {i+1}: {str(parseError)}") - continue - - elementsFromBlock = [] - if isinstance(parsed, dict): - if "elements" in parsed: - elementsFromBlock = parsed["elements"] - allElements.extend(elementsFromBlock) - elif parsed.get("type"): - elementsFromBlock = [parsed] - allElements.append(parsed) - elif isinstance(parsed, list): - elementsFromBlock = parsed - allElements.extend(parsed) - - # Log row count for table elements - if contentType == "table": - tableCount = sum(1 for e in elementsFromBlock if isinstance(e, dict) and e.get("type") == "table") - rowCount = sum( - len(e.get("content", {}).get("rows", [])) - for e in elementsFromBlock - if isinstance(e, dict) and e.get("type") == "table" - ) - if tableCount > 0: - logger.info(f"Section {sectionId}: JSON block {i+1}: {tableCount} table(s) with {rowCount} total rows") - - # Merge elements based on contentType - if contentType == "table" and len(allElements) > 1: - # Find all table elements - tableElements = [e for e in allElements if isinstance(e, dict) and e.get("type") == "table"] - if len(tableElements) > 1: - # Check if tables can be merged (same column counts) - canMerge = self._canMergeTables(tableElements) - if canMerge: - logger.info(f"Section {sectionId}: Merging {len(tableElements)} tables into one") - mergedTable = self._mergeTableElements(tableElements) - # Replace all table elements with merged one - nonTableElements = [e for e in allElements if not (isinstance(e, dict) and e.get("type") == "table")] - return [mergedTable] + nonTableElements - else: - logger.warning(f"Section {sectionId}: Cannot merge {len(tableElements)} tables (incompatible headers/columns). Keeping tables separate.") - # Return all elements as-is (tables remain separate) - return allElements - - return allElements - - def _canMergeTables(self, tableElements: List[Dict[str, Any]]) -> bool: - """Check if tables can be safely merged (same column counts).""" - if len(tableElements) <= 1: - return True - - # Extract column counts from all tables - columnCounts = [] - for table in tableElements: - headers = [] - if isinstance(table.get("content"), dict): - headers = table["content"].get("headers", []) - elif isinstance(table.get("content"), list): - # Old format: content is list of rows - if table["content"] and isinstance(table["content"][0], list): - headers = table["content"][0] - columnCounts.append(len(headers)) - - # Check if all tables have the same column count - firstCount = columnCounts[0] if columnCounts else 0 - return all(count == firstCount for count in columnCounts) - - def _mergeTableElements(self, tableElements: List[Dict[str, Any]]) -> Dict[str, Any]: - """Merge multiple table elements into a single table. - Assumes tables have compatible column counts (checked by _canMergeTables). - """ - if not tableElements: - return {"type": "table", "content": {"headers": [], "rows": []}} - - if len(tableElements) == 1: - return tableElements[0] - - # Extract headers from all tables - allHeaders = [] - for table in tableElements: - headers = [] - if isinstance(table.get("content"), dict): - headers = table["content"].get("headers", []) - elif isinstance(table.get("content"), list): - # Old format: content is list of rows - if table["content"] and isinstance(table["content"][0], list): - headers = table["content"][0] - allHeaders.append(headers) - - # Check header compatibility (same headers or just same column count) - firstHeaders = allHeaders[0] - headersCompatible = all(headers == firstHeaders for headers in allHeaders) - - # If headers differ but column counts match, use first table's headers and log warning - if not headersCompatible: - logger.warning(f"Merging {len(tableElements)} tables with different headers but same column count. Using headers from first table.") - - # Use headers from first table - headers = firstHeaders - - # Collect all rows from all tables, validating column count - allRows = [] - for tableIdx, table in enumerate(tableElements): - rows = [] - if isinstance(table.get("content"), dict): - rows = table["content"].get("rows", []) - elif isinstance(table.get("content"), list): - # Old format: content is list of rows - if table["content"] and isinstance(table["content"][0], list): - rows = table["content"][1:] if len(table["content"]) > 1 else [] - - # Validate row column count matches header count - expectedColCount = len(headers) - validRows = [] - for rowIdx, row in enumerate(rows): - if isinstance(row, list): - if len(row) == expectedColCount: - validRows.append(row) - else: - logger.warning(f"Table {tableIdx+1}, row {rowIdx+1}: column count mismatch ({len(row)} vs {expectedColCount}), skipping row") - elif isinstance(row, dict): - # Convert dict row to list based on header order - rowList = [row.get(h, "") for h in headers] - validRows.append(rowList) - else: - logger.warning(f"Table {tableIdx+1}, row {rowIdx+1}: invalid row format, skipping") - - allRows.extend(validRows) - - # Keep all rows, including duplicates (duplicates may be intentional) - logger.info(f"Merged {len(tableElements)} tables: {len(allRows)} total rows (duplicates preserved)") - - return { - "type": "table", - "content": { - "headers": headers, - "rows": allRows - } - } - - def _isBinaryMimeType(self, mimeType: str) -> bool: - """Check if MIME type is binary.""" - binaryTypes = [ - "application/octet-stream", - "application/pdf", - "application/zip", - "application/x-zip-compressed" - ] - return mimeType in binaryTypes - - def _looksLikeBase64(self, data: str) -> bool: - """ - Heuristic check if string looks like base64-encoded data. - - Base64 contains only: A-Z, a-z, 0-9, +, /, =, and whitespace. - If >95% of characters are base64 chars and no normal text patterns, likely base64. - """ - if not data or len(data) < 100: - return False - - base64Chars = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r\t ") - sample = data[:500] # Check first 500 chars - if not sample: - return False - - base64Ratio = sum(1 for c in sample if c in base64Chars) / len(sample) - - # If >95% base64 chars and no normal text patterns (like spaces between words) → likely base64 - # Base64 typically has very long strings without spaces or punctuation - hasNormalTextPatterns = any( - c in sample[:200] for c in ".,!?;:()[]{}\"'" - ) or " " in sample[:200] # Double spaces suggest text - - return base64Ratio > 0.95 and not hasNormalTextPatterns - - def _findContentPartById(self, partId: str, contentParts: List[ContentPart]) -> Optional[ContentPart]: - """Finde ContentPart nach ID.""" - for part in contentParts: - if part.id == partId: - return part - return None - - def _needsAggregation( - self, - contentType: str, - contentPartCount: int - ) -> bool: - """ - Bestimmt ob mehrere ContentParts aggregiert werden müssen. - - Aggregation nötig wenn: - - content_type erfordert Aggregation (table, bullet_list) - - UND mehrere ContentParts vorhanden sind (> 1) - - Args: - contentType: Section content_type - contentPartCount: Anzahl der ContentParts in dieser Section - - Returns: - True wenn Aggregation nötig, False sonst - """ - aggregationTypes = ["table", "bullet_list"] - - if contentType in aggregationTypes and contentPartCount > 1: - return True - - # Optional: Auch für paragraph wenn mehrere Parts vorhanden - # (z.B. Vergleich mehrerer Dokumente) - # Standard: Keine Aggregation für paragraph - return False - - def _getAcceptedSectionTypesForFormat(self, outputFormat: str) -> List[str]: - """ - Get accepted section types for a given output format by querying the renderer. - - Args: - outputFormat: Format name (e.g., 'csv', 'json', 'pdf') - - Returns: - List of accepted section content types (e.g., ["table", "code_block"]) - - Raises: - ValueError: If renderer not found or doesn't provide accepted types - """ - from modules.services.serviceGeneration.renderers.registry import getRenderer - - # Get document renderer for this format (structure filling is document generation path) - renderer = getRenderer(outputFormat, self.services, outputStyle='document') - - if not renderer: - raise ValueError(f"No renderer found for output format '{outputFormat}'. Check renderer registry.") - - if not hasattr(renderer, 'getAcceptedSectionTypes'): - raise ValueError(f"Renderer for '{outputFormat}' does not implement getAcceptedSectionTypes(). Add this method to the renderer.") - - acceptedTypes = renderer.getAcceptedSectionTypes(outputFormat) - - if not acceptedTypes: - raise ValueError(f"Renderer for '{outputFormat}' returned empty accepted types. Fix getAcceptedSectionTypes() in the renderer.") - - logger.debug(f"Renderer for '{outputFormat}' accepts: {acceptedTypes}") - return acceptedTypes - diff --git a/modules/services/serviceAi/subStructureGeneration.py b/modules/services/serviceAi/subStructureGeneration.py deleted file mode 100644 index 67b045b3..00000000 --- a/modules/services/serviceAi/subStructureGeneration.py +++ /dev/null @@ -1,508 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Structure Generation Module - -Handles document structure generation, including: -- Generating document structure with sections -- Building structure prompts -""" -import json -import logging -from typing import Dict, Any, List, Optional - -from modules.datamodels.datamodelExtraction import ContentPart -from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.workflows.processing.shared.stateTools import checkWorkflowStopped - -logger = logging.getLogger(__name__) - - -class StructureGenerator: - """Handles document structure generation.""" - - def __init__(self, services, aiService): - """Initialize StructureGenerator with service center and AI service access.""" - self.services = services - self.aiService = aiService - - def _getUserLanguage(self) -> str: - """Get user language for document generation""" - try: - if self.services: - # Prefer detected language if available (from user intention analysis) - if hasattr(self.services, 'currentUserLanguage') and self.services.currentUserLanguage: - return self.services.currentUserLanguage - # Fallback to user's preferred language - elif hasattr(self.services, 'user') and self.services.user and hasattr(self.services.user, 'language'): - return self.services.user.language - except Exception: - pass - return 'en' # Default fallback - - async def generateStructure( - self, - userPrompt: str, - contentParts: List[ContentPart], - outputFormat: Optional[str] = None, - parentOperationId: str = None - ) -> Dict[str, Any]: - """ - Phase 5C: Generiert Chapter-Struktur (Table of Contents). - Definiert für jedes Chapter: - - Level, Title - - contentParts (unified object with instruction and/or caption per part) - - generationHint - - Generate document structure with per-document format determination. - Multiple documents can be produced with different formats (e.g., one PDF, one HTML). - AI determines formats per-document from user prompt. The outputFormat parameter is - only a validation fallback - used if AI doesn't return format per document. - - Args: - userPrompt: User-Anfrage - contentParts: Alle vorbereiteten ContentParts mit Metadaten - outputFormat: Optional global format fallback. If omitted, formats are determined - from user prompt by AI. Used as validation fallback if AI doesn't - return format per document. Defaults to "txt" if not provided. - parentOperationId: Parent Operation-ID für ChatLog-Hierarchie - - Returns: - Struktur-Dict mit documents und chapters (nicht sections!) - """ - # If outputFormat not provided, use "txt" as fallback for validation - # AI will determine formats per document from user prompt - if not outputFormat: - outputFormat = "txt" - logger.debug("outputFormat not provided - using 'txt' as validation fallback, formats determined from prompt") - # Erstelle Operation-ID für Struktur-Generierung - structureOperationId = f"{parentOperationId}_structure_generation" - - # Starte ChatLog mit Parent-Referenz - formatDisplay = outputFormat if outputFormat else "auto-determined" - self.services.chat.progressLogStart( - structureOperationId, - "Chapter Structure Generation", - "Structure", - f"Generating chapter structure (format: {formatDisplay})", - parentOperationId=parentOperationId - ) - - try: - # Baue Chapter-Struktur-Prompt mit Content-Index - structurePrompt = self._buildChapterStructurePrompt( - userPrompt=userPrompt, - contentParts=contentParts, - outputFormat=outputFormat - ) - - # AI-Call für Chapter-Struktur-Generierung mit Looping-Unterstützung - # Use _callAiWithLooping instead of callAiPlanning to support continuation if response is cut - options = AiCallOptions( - operationType=OperationTypeEnum.DATA_GENERATE, - priority=PriorityEnum.QUALITY, - processingMode=ProcessingModeEnum.DETAILED, - compressPrompt=False, - compressContext=False, - resultFormat="json" - ) - - structurePrompt, templateStructure = self._buildChapterStructurePrompt( - userPrompt=userPrompt, - contentParts=contentParts, - outputFormat=outputFormat - ) - - # Create prompt builder for continuation support - async def buildChapterStructurePromptWithContinuation( - continuationContext: Any, - templateStructure: str, - basePrompt: str - ) -> str: - """Build chapter structure prompt with continuation context. Uses unified signature. - - Note: All initial context (userPrompt, contentParts, outputFormat, etc.) is already - contained in basePrompt. This function only adds continuation-specific instructions. - """ - # Extract continuation context fields (only what's needed for continuation) - incompletePart = continuationContext.incomplete_part - lastRawJson = continuationContext.last_raw_json - - # Generate both overlap context and hierarchy context using jsonContinuation - overlapContext = "" - unifiedContext = "" - if lastRawJson: - # Get contexts directly from jsonContinuation - from modules.shared.jsonContinuation import getContexts - contexts = getContexts(lastRawJson) - overlapContext = contexts.overlapContext - unifiedContext = contexts.hierarchyContextForPrompt - elif incompletePart: - unifiedContext = incompletePart - else: - unifiedContext = "Unable to extract context - response was completely broken" - - # Build unified continuation prompt format - continuationPrompt = f"""{basePrompt} - ---- CONTINUATION REQUEST --- -The previous JSON response was incomplete. Continue from where it stopped. - -Context showing structure hierarchy with cut point: -``` -{unifiedContext} -``` - -Overlap Requirement: -To ensure proper merging, your response MUST start EXACTLY with the overlap context shown below, then continue with new content. - -Overlap context (start your response with this exact text): -```json -{overlapContext if overlapContext else "No overlap context available"} -``` - -TASK: -1. Start your response EXACTLY with the overlap context shown above (character by character) -2. Continue seamlessly from where the overlap context ends -3. Complete the remaining content following the JSON structure template above -4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects - -CRITICAL: -- Your response MUST begin with the exact overlap context text (this enables automatic merging) -- Continue seamlessly after the overlap context with new content -- Your response must be valid JSON matching the structure template above""" - return continuationPrompt - - # Call AI with looping support - # NOTE: Do NOT pass contentParts here - we only need metadata for structure generation - # The contentParts metadata is already included in the prompt (contentPartsIndex) - # Actual content extraction happens later during section generation - checkWorkflowStopped(self.services) - aiResponseJson = await self.aiService.callAiWithLooping( - prompt=structurePrompt, - options=options, - debugPrefix="chapter_structure_generation", - promptBuilder=buildChapterStructurePromptWithContinuation, - promptArgs={ - "userPrompt": userPrompt, - "outputFormat": outputFormat, - "templateStructure": templateStructure, - "basePrompt": structurePrompt - }, - useCaseId="chapter_structure", # REQUIRED: Explicit use case ID - operationId=structureOperationId, - userPrompt=userPrompt, - contentParts=None # Do not pass ContentParts - only metadata needed, not content extraction - ) - - # Parse the complete JSON response (looping system already handles completion) - extractedJson = self.services.utils.jsonExtractString(aiResponseJson) - parsedJson, parseError, cleanedJson = self.services.utils.jsonTryParse(extractedJson) - - if parseError is not None: - # Even with looping, try repair as fallback - logger.warning(f"JSON parsing failed after looping: {str(parseError)}. Attempting repair...") - from modules.shared import jsonUtils - repairedJson = jsonUtils.repairBrokenJson(extractedJson) - if repairedJson: - parsedJson, parseError, _ = self.services.utils.jsonTryParse(json.dumps(repairedJson)) - if parseError is None: - logger.info("Successfully repaired and parsed JSON structure after looping") - structure = parsedJson - else: - logger.error(f"Failed to parse repaired JSON: {str(parseError)}") - raise ValueError(f"Failed to parse JSON structure after repair: {str(parseError)}") - else: - logger.error(f"Failed to repair JSON. Parse error: {str(parseError)}") - logger.error(f"Cleaned JSON preview (first 500 chars): {cleanedJson[:500]}") - raise ValueError(f"Failed to parse JSON structure: {str(parseError)}") - else: - structure = parsedJson - - # State 3 Validation: Validate and auto-fix structure - # Validation 3.1: Structure missing 'documents' field - if "documents" not in structure: - raise ValueError("Structure missing 'documents' field - cannot auto-fix") - - documents = structure["documents"] - - # Validation 3.2: Structure has no documents - if not isinstance(documents, list) or len(documents) == 0: - raise ValueError("Structure has no documents - cannot generate without documents") - - # Import renderer registry for format validation (existing infrastructure) - from modules.services.serviceGeneration.renderers.registry import getRenderer - - # Validate and fix each document - for doc in documents: - # Validation 3.3 & 3.4: Document outputFormat - # outputFormat parameter is optional - if omitted, formats determined from prompt by AI - # Use as fallback only if AI doesn't return format per document - # Multiple documents can have different formats (e.g., one PDF, one HTML) - globalFormatFallback = outputFormat or "txt" # Fallback for validation - - if "outputFormat" not in doc or not doc["outputFormat"]: - # AI didn't return format or returned empty - use global fallback - doc["outputFormat"] = globalFormatFallback - logger.warning(f"Document {doc.get('id')} missing outputFormat - using fallback: {doc['outputFormat']}") - else: - # AI returned format - validate using existing renderer registry - formatName = str(doc["outputFormat"]).lower().strip() - renderer = getRenderer(formatName) # Uses existing infrastructure - - if not renderer: - # Format doesn't match any renderer - use txt (simple approach) - logger.warning(f"Document {doc.get('id')} has format without renderer: {formatName}, using 'txt'") - doc["outputFormat"] = "txt" - else: - # Valid format with renderer - normalize and keep AI result - doc["outputFormat"] = formatName - logger.debug(f"Document {doc.get('id')} using AI-determined format: {formatName}") - - # Validation 3.5 & 3.6: Document language - # Use validated currentUserLanguage (always valid, validated during user intention analysis) - # Access via _getUserLanguage() which uses self.services.currentUserLanguage - userPromptLanguage = self._getUserLanguage() # Uses validated currentUserLanguage infrastructure - - if "language" not in doc or not isinstance(doc["language"], str) or len(doc["language"]) != 2: - # AI didn't return language or invalid format - use validated currentUserLanguage - doc["language"] = userPromptLanguage - if "language" not in doc: - logger.warning(f"Document {doc.get('id')} missing language - using currentUserLanguage: {userPromptLanguage}") - else: - logger.warning(f"Document {doc.get('id')} has invalid language format from AI: {doc['language']}, using currentUserLanguage") - else: - # AI returned valid language format - normalize - doc["language"] = doc["language"].lower().strip()[:2] - logger.debug(f"Document {doc.get('id')} using AI-determined language: {doc['language']}") - - # Validation 3.7: Document missing 'chapters' field - if "chapters" not in doc: - raise ValueError(f"Document {doc.get('id')} missing 'chapters' field - cannot auto-fix") - - # Validation 3.8: Chapter missing 'contentParts' field - for chapter in doc["chapters"]: - if "contentParts" not in chapter: - raise ValueError(f"Chapter {chapter.get('id')} missing 'contentParts' field - cannot auto-fix") - - # ChatLog abschließen - self.services.chat.progressLogFinish(structureOperationId, True) - - return structure - - except Exception as e: - self.services.chat.progressLogFinish(structureOperationId, False) - logger.error(f"Error in generateStructure: {str(e)}") - raise - - def _buildChapterStructurePrompt( - self, - userPrompt: str, - contentParts: List[ContentPart], - outputFormat: str - ) -> tuple[str, str]: - """Baue Prompt für Chapter-Struktur-Generierung.""" - # Baue ContentParts-Index - filtere leere Parts heraus - contentPartsIndex = "" - validParts = [] - filteredParts = [] - - for part in contentParts: - contentFormat = part.metadata.get("contentFormat", "unknown") - - # WICHTIG: Reference Parts haben absichtlich leere Daten - immer einschließen - if contentFormat == "reference": - validParts.append(part) - logger.debug(f"Including reference ContentPart {part.id} (intentionally empty data)") - continue - - # Überspringe leere Parts (keine Daten oder nur Container ohne Inhalt) - # ABER: Reference Parts wurden bereits oben behandelt - if not part.data or (isinstance(part.data, str) and len(part.data.strip()) == 0): - # Überspringe Container-Parts ohne Daten - if part.typeGroup == "container" and not part.data: - filteredParts.append((part.id, "container without data")) - continue - # Überspringe andere leere Parts (aber nicht Reference, die wurden bereits behandelt) - if not part.data: - filteredParts.append((part.id, f"no data (format: {contentFormat})")) - continue - - validParts.append(part) - logger.debug(f"Including ContentPart {part.id}: format={contentFormat}, type={part.typeGroup}, dataLength={len(str(part.data)) if part.data else 0}") - - if filteredParts: - logger.debug(f"Filtered out {len(filteredParts)} empty ContentParts: {filteredParts}") - - logger.info(f"Building structure prompt with {len(validParts)} valid ContentParts (from {len(contentParts)} total)") - - # Baue Index nur für gültige Parts - for i, part in enumerate(validParts, 1): - contentFormat = part.metadata.get("contentFormat", "unknown") - originalFileName = part.metadata.get('originalFileName', 'N/A') - - contentPartsIndex += f"\n{i}. ContentPart ID: {part.id}\n" - contentPartsIndex += f" Format: {contentFormat}\n" - contentPartsIndex += f" Type: {part.typeGroup}\n" - contentPartsIndex += f" MIME Type: {part.mimeType or 'N/A'}\n" - contentPartsIndex += f" Source: {part.metadata.get('documentId', 'unknown')}\n" - contentPartsIndex += f" Original file name: {originalFileName}\n" - contentPartsIndex += f" Usage hint: {part.metadata.get('usageHint', 'N/A')}\n" - - if not contentPartsIndex: - contentPartsIndex = "\n(No content parts available)" - - # Get language from services (user intention analysis) - language = self._getUserLanguage() - logger.debug(f"Using language from services (user intention analysis) for structure generation: {language}") - - # Create template structure explicitly (not extracted from prompt) - # This ensures exact identity between initial and continuation prompts - templateStructure = f"""{{ - "metadata": {{ - "title": "Document Title", - "language": "{language}" - }}, - "documents": [{{ - "id": "doc_1", - "title": "Document Title", - "filename": "document.{outputFormat}", - "outputFormat": "{outputFormat}", - "language": "{language}", - "chapters": [ - {{ - "id": "chapter_1", - "level": 1, - "title": "Chapter Title", - "contentParts": {{ - "extracted_part_id": {{ - "instruction": "Use extracted content with ALL relevant details from user request" - }} - }}, - "generationHint": "Detailed description including ALL relevant details from user request for this chapter", - "sections": [] - }} - ] - }}] -}}""" - - prompt = f"""# TASK: Plan Document Structure (Documents + Chapters) - -This is a STRUCTURE PLANNING task. You define which documents to create and which chapters each document will have. -Chapter CONTENT will be generated in a later step - here you only plan the STRUCTURE and assign content references. -Return EXACTLY ONE complete JSON object. Do not generate multiple JSON objects, alternatives, or variations. Do not use separators like "---" between JSON objects. - -## USER REQUEST (for context) -``` -{userPrompt} -``` - -## AVAILABLE CONTENT PARTS -{contentPartsIndex} - -## CONTENT ASSIGNMENT RULE - -CRITICAL: Every chapter MUST have contentParts assigned if it relates to documents/images/data from the user request. -If the user request mentions documents/images/data, then EVERY chapter that generates content related to those references MUST assign the relevant ContentParts explicitly. - -Assignment logic: -- If chapter DISPLAYS a document/image → assign "object" format ContentPart with "caption" -- If chapter generates text content ABOUT a document/image/data → assign ContentPart with "instruction": - - Prefer "extracted" format if available (contains analyzed/extracted content) - - If only "object" format is available, use "object" format with "instruction" (to write ABOUT the image/document) -- If chapter's generationHint or purpose relates to a document/image/data mentioned in user request → it MUST have ContentParts assigned -- Multiple chapters might assign the same ContentPart (e.g., one chapter displays image, another writes about it) -- Use ContentPart IDs exactly as listed in AVAILABLE CONTENT PARTS above -- Empty contentParts are only allowed if chapter generates content WITHOUT referencing any documents/images/data from the user request - -CRITICAL RULE: If the user request mentions BOTH: - a) Documents/images/data (listed in AVAILABLE CONTENT PARTS above), AND - b) Generic content types (article text, main content, body text, etc.) -Then chapters that generate those generic content types MUST assign the relevant ContentParts, because the content should relate to or be based on the provided documents/images/data. - -## CONTENT EFFICIENCY PRINCIPLES -- Generate COMPACT content: Focus on essential information only -- AVOID verbose, lengthy, or repetitive text - be concise and direct -- Prioritize FACTS over filler text - no introductions like "In this chapter..." -- Minimize system resources: shorter content = faster processing -- Quality over quantity: precise, meaningful content rather than padding - -## CHAPTER STRUCTURE REQUIREMENTS -- Generate chapters based on USER REQUEST - analyze what structure the user wants -- Create ONLY the minimum chapters needed to cover the user's request - avoid over-structuring -- IMPORTANT: Each chapter MUST have ALL these fields: - - id: Unique identifier (e.g., "chapter_1") - - level: Heading level (1, 2, 3, etc.) - - title: Chapter title - - contentParts: Object mapping ContentPart IDs to usage instructions (MUST assign if chapter relates to documents/data from user request) - - generationHint: Description of what content to generate (including formatting/styling requirements) - - sections: Empty array [] (REQUIRED - sections are generated in next phase) -- contentParts: {{"partId": {{"instruction": "..."}} or {{"caption": "..."}} or both}} - Assign ContentParts as required by CONTENT ASSIGNMENT RULE above -- The "instruction" field for each ContentPart MUST contain ALL relevant details from the USER REQUEST that apply to content extraction for this specific chapter. Include all formatting rules, data requirements, constraints, and specifications mentioned in the user request that are relevant for processing this ContentPart in this chapter. -- generationHint: Keep CONCISE but include relevant details from the USER REQUEST. Focus on WHAT to generate, not HOW to phrase it verbosely. -- The number of chapters depends on the user request - create only what is requested. Do NOT create chapters for topics without available data. - -CRITICAL: Only create chapters for CONTENT sections, not for formatting/styling requirements. Formatting/styling requirements to be included in each generationHint if needed. - -## DOCUMENT STRUCTURE - -For each document, determine: -- outputFormat: From USER REQUEST (explicit mention or infer from purpose/content type). Default: "{outputFormat}". Multiple documents can have different formats. -- language: From USER REQUEST (map to ISO 639-1: de, en, fr, it...). Default: "{language}". Multiple documents can have different languages. -- chapters: Structure appropriately for the format (e.g., pptx=slides, docx=sections, xlsx=worksheets). Match format capabilities and constraints. - -Required JSON fields: -- metadata: {{"title": "...", "language": "..."}} -- documents: Array with id, title, filename, outputFormat, language, chapters[] -- chapters: Array with id, level, title, contentParts, generationHint, sections[] - -EXAMPLE STRUCTURE (for reference only - adapt to user request): -{{ - "metadata": {{ - "title": "Document Title", - "language": "{language}" - }}, - "documents": [{{ - "id": "doc_1", - "title": "Document Title", - "filename": "document.{outputFormat}", - "outputFormat": "{outputFormat}", - "language": "{language}", - "chapters": [ - {{ - "id": "chapter_1", - "level": 1, - "title": "Chapter Title", - "contentParts": {{ - "extracted_part_id": {{ - "instruction": "Use extracted content with ALL relevant details from user request" - }} - }}, - "generationHint": "Detailed description including ALL relevant details from user request for this chapter", - "sections": [] - }} - ] - }}] -}} - -CRITICAL INSTRUCTIONS: -- Generate chapters based on USER REQUEST, NOT based on the example above -- The example shows the JSON structure format, NOT the required chapters -- Create only the chapters that match the user's request -- Adapt chapter titles and structure to match the user's specific request -- Determine outputFormat and language for each document by analyzing the USER REQUEST above -- The example shows placeholders "{outputFormat}" and "{language}" - YOU MUST REPLACE THESE with actual values determined from the USER REQUEST - -MANDATORY CONTENT ASSIGNMENT CHECK: -For each chapter, verify: -1. Does the user request mention documents/images/data? (e.g., "photo", "image", "document", "data", "based on", "about") -2. Does this chapter's generationHint, title, or purpose relate to those documents/images/data mentioned in step 1? - - Examples: "article about the photo", "text describing the image", "analysis of the document", "content based on the data" - - Even if chapter doesn't explicitly say "about the image", if user request mentions both the image AND this chapter's content type → relate them -3. If YES to both → chapter MUST have contentParts assigned (cannot be empty {{}}) -4. If ContentPart is "object" format and chapter needs to write ABOUT it → assign with "instruction" field, not just "caption" - -OUTPUT FORMAT: Start with {{ and end with }}. Do NOT use markdown code fences (```json). Do NOT add explanatory text before or after the JSON. Return ONLY the JSON object itself. -""" - return prompt, templateStructure - diff --git a/modules/services/serviceBilling/__init__.py b/modules/services/serviceBilling/__init__.py deleted file mode 100644 index ab0805d5..00000000 --- a/modules/services/serviceBilling/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Billing service module.""" - -from .mainServiceBilling import BillingService, getService - -__all__ = ["BillingService", "getService"] diff --git a/modules/services/serviceBilling/mainServiceBilling.py b/modules/services/serviceBilling/mainServiceBilling.py deleted file mode 100644 index 8407304c..00000000 --- a/modules/services/serviceBilling/mainServiceBilling.py +++ /dev/null @@ -1,417 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Billing Service - Central service for billing operations. - -Handles: -- Balance checks before AI operations -- Cost recording after AI operations -- Provider permission checks via RBAC -- Price calculation with markup -""" - -import logging -from typing import Dict, Any, List, Optional -from datetime import datetime - -from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelBilling import ( - BillingModelEnum, - BillingCheckResult, - TransactionTypeEnum, - ReferenceTypeEnum, - BillingTransaction, - BillingBalanceResponse, -) -from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface - -logger = logging.getLogger(__name__) - -# Markup percentage for internal pricing (+50% für Infrastruktur und Platform Service + 50% für Währungsrisiko ==> Faktor 2.0) -BILLING_MARKUP_PERCENT = 100 - -# Singleton cache -_billingServices: Dict[str, "BillingService"] = {} - - -def getService(currentUser: User, mandateId: str, featureInstanceId: str = None, featureCode: str = None) -> "BillingService": - """ - Factory function to get or create a BillingService instance. - - Args: - currentUser: Current user object - mandateId: Mandate ID for context - featureInstanceId: Optional feature instance ID - featureCode: Optional feature code (e.g., 'chatplayground', 'automation') - - Returns: - BillingService instance - """ - cacheKey = f"{currentUser.id}_{mandateId}_{featureInstanceId}" - - if cacheKey not in _billingServices: - _billingServices[cacheKey] = BillingService(currentUser, mandateId, featureInstanceId, featureCode) - else: - _billingServices[cacheKey].setContext(currentUser, mandateId, featureInstanceId, featureCode) - - return _billingServices[cacheKey] - - -class BillingService: - """ - Central billing service for AI operations. - - Responsibilities: - - Check balance before operations - - Record usage costs - - Apply pricing markup - - Check provider permissions via RBAC - """ - - def __init__( - self, - currentUser: User, - mandateId: str, - featureInstanceId: str = None, - featureCode: str = None - ): - """ - Initialize the billing service. - - Args: - currentUser: Current user object - mandateId: Mandate ID - featureInstanceId: Optional feature instance ID - featureCode: Optional feature code - """ - self.currentUser = currentUser - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - self.featureCode = featureCode - - # Get billing interface - self._billingInterface = getBillingInterface(currentUser, mandateId) - - # Cache settings - self._settingsCache = None - - def setContext( - self, - currentUser: User, - mandateId: str, - featureInstanceId: str = None, - featureCode: str = None - ): - """Update service context.""" - self.currentUser = currentUser - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - self.featureCode = featureCode - self._billingInterface = getBillingInterface(currentUser, mandateId) - self._settingsCache = None - - def _getSettings(self) -> Optional[Dict[str, Any]]: - """Get billing settings with caching.""" - if self._settingsCache is None: - self._settingsCache = self._billingInterface.getSettings(self.mandateId) - return self._settingsCache - - # ========================================================================= - # Price Calculation - # ========================================================================= - - def calculatePriceWithMarkup(self, basePriceCHF: float) -> float: - """ - Calculate final price with markup. - - The AICore plugins return prices in their original currency (USD). - This method applies the configured markup percentage. - - Args: - basePriceCHF: Base price from AI model (actually USD from provider) - - Returns: - Final price in CHF with markup applied - """ - if basePriceCHF <= 0: - return 0.0 - - # Apply markup (50% = multiply by 1.5) - markup_multiplier = 1 + (BILLING_MARKUP_PERCENT / 100) - return round(basePriceCHF * markup_multiplier, 6) - - # ========================================================================= - # Balance Operations - # ========================================================================= - - def checkBalance(self, estimatedCost: float = 0.0) -> BillingCheckResult: - """ - Check if the current user/mandate has sufficient balance. - - Args: - estimatedCost: Estimated cost of the operation (with markup applied) - - Returns: - BillingCheckResult indicating if operation is allowed - """ - return self._billingInterface.checkBalance( - self.mandateId, - self.currentUser.id, - estimatedCost - ) - - def hasBalance(self, estimatedCost: float = 0.0) -> bool: - """ - Quick check if balance is sufficient. - - Args: - estimatedCost: Estimated cost with markup - - Returns: - True if operation is allowed - """ - result = self.checkBalance(estimatedCost) - return result.allowed - - def getCurrentBalance(self) -> float: - """ - Get current balance for the user/mandate. - - Returns: - Current balance in CHF - """ - result = self.checkBalance(0.0) - return result.currentBalance or 0.0 - - # ========================================================================= - # Usage Recording - # ========================================================================= - - def recordUsage( - self, - priceCHF: float, - workflowId: str = None, - aicoreProvider: str = None, - aicoreModel: str = None, - description: str = None - ) -> Optional[Dict[str, Any]]: - """ - Record AI usage cost as a billing transaction. - - This method: - 1. Applies the pricing markup - 2. Creates a DEBIT transaction - 3. Updates the account balance - - Args: - priceCHF: Base price from AI model (before markup) - workflowId: Optional workflow ID - aicoreProvider: AICore provider name (e.g., 'anthropic', 'openai') - aicoreModel: AICore model name (e.g., 'claude-4-sonnet', 'gpt-4o') - description: Optional description - - Returns: - Created transaction dict or None if not recorded - """ - if priceCHF <= 0: - return None - - # Apply markup - finalPrice = self.calculatePriceWithMarkup(priceCHF) - - if finalPrice <= 0: - return None - - # Build description - if not description: - description = f"AI Usage: {aicoreModel or aicoreProvider or 'unknown'}" - - return self._billingInterface.recordUsage( - mandateId=self.mandateId, - userId=self.currentUser.id, - priceCHF=finalPrice, - workflowId=workflowId, - featureInstanceId=self.featureInstanceId, - featureCode=self.featureCode, - aicoreProvider=aicoreProvider, - aicoreModel=aicoreModel, - description=description - ) - - # ========================================================================= - # Provider Permission Check (via RBAC) - # ========================================================================= - - def isProviderAllowed(self, provider: str) -> bool: - """ - Check if the user has permission to use an AICore provider. - - Uses RBAC to check for resource permission: - resource.aicore.{provider} - - Args: - provider: Provider name (e.g., 'anthropic', 'openai') - - Returns: - True if provider is allowed - """ - try: - from modules.security.rbac import RbacClass - from modules.datamodels.datamodelRbac import AccessRuleContext - from modules.security.rootAccess import getRootDbAppConnector - - # Get database connector via established pattern - dbApp = getRootDbAppConnector() - - rbac = RbacClass(dbApp, dbApp) - resourceKey = f"resource.aicore.{provider}" - - # Check if user has view permission for this resource (view = use for RESOURCE context) - permissions = rbac.getUserPermissions( - self.currentUser, - AccessRuleContext.RESOURCE, - resourceKey, - mandateId=self.mandateId - ) - - return permissions.view - except Exception as e: - logger.warning(f"Error checking provider permission: {e}") - # Default to allowed if RBAC check fails - return True - - def getallowedProviders(self) -> List[str]: - """ - Get list of AICore providers the user is allowed to use. - - Returns: - List of allowed provider names - """ - try: - from modules.aicore.aicoreModelRegistry import modelRegistry - - # Get all available providers - connectors = modelRegistry.discoverConnectors() - allProviders = [c.getConnectorType() for c in connectors] - - # Filter by RBAC permissions - return [p for p in allProviders if self.isProviderAllowed(p)] - except Exception as e: - logger.warning(f"Error getting allowed providers: {e}") - return [] - - # ========================================================================= - # Admin Operations - # ========================================================================= - - def addCredit( - self, - amount: float, - description: str = "Manual credit", - referenceType: ReferenceTypeEnum = ReferenceTypeEnum.ADMIN - ) -> Optional[Dict[str, Any]]: - """ - Add credit to the account (admin operation). - - Args: - amount: Amount to credit (positive) - description: Transaction description - referenceType: Reference type (ADMIN, PAYMENT, SYSTEM) - - Returns: - Created transaction dict or None - """ - if amount <= 0: - return None - - settings = self._getSettings() - if not settings: - logger.warning(f"No billing settings for mandate {self.mandateId}") - return None - - billingModel = BillingModelEnum(settings.get("billingModel", BillingModelEnum.UNLIMITED.value)) - - # Get or create account - if billingModel == BillingModelEnum.PREPAY_USER: - account = self._billingInterface.getOrCreateUserAccount( - self.mandateId, - self.currentUser.id, - initialBalance=0.0 - ) - else: - account = self._billingInterface.getOrCreateMandateAccount( - self.mandateId, - initialBalance=0.0 - ) - - # Create credit transaction - transaction = BillingTransaction( - accountId=account["id"], - transactionType=TransactionTypeEnum.CREDIT, - amount=amount, - description=description, - referenceType=referenceType - ) - - return self._billingInterface.createTransaction(transaction) - - # ========================================================================= - # Statistics & Reporting - # ========================================================================= - - def getBalancesForUser(self) -> List[BillingBalanceResponse]: - """ - Get all billing balances for the current user. - - Returns: - List of balance responses for each mandate - """ - return self._billingInterface.getBalancesForUser(self.currentUser.id) - - def getTransactionHistory(self, limit: int = 100) -> List[Dict[str, Any]]: - """ - Get transaction history for the user across all mandates. - - Args: - limit: Maximum number of transactions - - Returns: - List of transactions - """ - return self._billingInterface.getTransactionsForUser(self.currentUser.id, limit=limit) - - -# ============================================================================ -# Exception Classes -# ============================================================================ - -class InsufficientBalanceException(Exception): - """Raised when there's insufficient balance for an operation.""" - - def __init__(self, currentBalance: float, requiredAmount: float, message: str = None): - self.currentBalance = currentBalance - self.requiredAmount = requiredAmount - self.message = message or f"Insufficient balance. Current: {currentBalance:.2f} CHF, Required: {requiredAmount:.2f} CHF" - super().__init__(self.message) - - -class ProviderNotAllowedException(Exception): - """Raised when a user doesn't have permission to use an AI provider.""" - - def __init__(self, provider: str, message: str = None): - self.provider = provider - self.message = message or f"Provider '{provider}' is not allowed for your role" - super().__init__(self.message) - - -class BillingContextError(Exception): - """Raised when billing context is incomplete (missing mandateId, user, etc.). - - This is a FAIL-SAFE error: AI calls MUST NOT proceed without valid billing context. - Acts like a 0 CHF credit card pre-authorization check - validates that billing - CAN be recorded before any expensive AI operation starts. - """ - - def __init__(self, message: str = None): - self.message = message or "Billing context incomplete - AI call blocked" - super().__init__(self.message) diff --git a/modules/services/serviceBilling/stripeCheckout.py b/modules/services/serviceBilling/stripeCheckout.py deleted file mode 100644 index 692e5087..00000000 --- a/modules/services/serviceBilling/stripeCheckout.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Stripe Checkout service for billing credit top-ups. -Creates Checkout Sessions for redirect-based payment flow. -""" - -import logging -from typing import Optional - -from modules.shared.configuration import APP_CONFIG - -logger = logging.getLogger(__name__) - -# Server-side allowed amounts in CHF - never trust client -ALLOWED_AMOUNTS_CHF = [10, 25, 50, 100, 250, 500] - - -def create_checkout_session( - mandate_id: str, - user_id: Optional[str], - amount_chf: float -) -> str: - """ - Create a Stripe Checkout Session for credit top-up. - - Amount and currency are validated server-side. The client-provided amount - must match an allowed preset. - - Args: - mandate_id: Target mandate ID - user_id: Target user ID (for PREPAY_USER) or None (for mandate pool) - amount_chf: Amount in CHF (must be in ALLOWED_AMOUNTS_CHF) - - Returns: - Stripe Checkout Session URL for redirect - - Raises: - ValueError: If amount is invalid - """ - import stripe - - # Validate amount server-side - if amount_chf not in ALLOWED_AMOUNTS_CHF: - raise ValueError( - f"Invalid amount {amount_chf} CHF. Allowed: {ALLOWED_AMOUNTS_CHF}" - ) - - # Pin API version from config (match Stripe Dashboard) - api_version = APP_CONFIG.get("STRIPE_API_VERSION") - if api_version: - stripe.api_version = api_version - - # Get secrets - secret_key = APP_CONFIG.get("STRIPE_SECRET_KEY_SECRET") or APP_CONFIG.get("STRIPE_SECRET_KEY") - if not secret_key: - raise ValueError("STRIPE_SECRET_KEY_SECRET not configured") - - stripe.api_key = secret_key - - frontend_url = APP_CONFIG.get("APP_FRONTEND_URL", "https://nyla-int.poweron-center.net") - base_path = "/admin/billing" - success_url = f"{frontend_url.rstrip('/')}{base_path}?success=true&session_id={{CHECKOUT_SESSION_ID}}" - cancel_url = f"{frontend_url.rstrip('/')}{base_path}?canceled=true" - - # Amount in cents for Stripe (CHF uses 2 decimal places) - amount_cents = int(round(amount_chf * 100)) - - metadata = { - "mandateId": mandate_id, - "amountChf": str(amount_chf), - } - if user_id: - metadata["userId"] = user_id - - session = stripe.checkout.Session.create( - mode="payment", - line_items=[ - { - "price_data": { - "currency": "chf", - "unit_amount": amount_cents, - "product_data": { - "name": "Guthaben aufladen", - "description": "AI Service Guthaben (CHF)", - }, - }, - "quantity": 1, - } - ], - success_url=success_url, - cancel_url=cancel_url, - metadata=metadata, - ) - - if not session or not session.url: - raise ValueError("Stripe Checkout Session creation failed") - - logger.info( - f"Created Stripe Checkout Session {session.id} for mandate {mandate_id}, " - f"amount {amount_chf} CHF" - ) - - return session.url diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py deleted file mode 100644 index b1e4a7ae..00000000 --- a/modules/services/serviceChat/mainServiceChat.py +++ /dev/null @@ -1,1061 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -import logging -from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelUam import User, UserConnection -from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog -from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.shared.progressLogger import ProgressLogger - -logger = logging.getLogger(__name__) - -class ChatService: - """Service class containing methods for document processing, chat operations, and workflow management""" - - def __init__(self, serviceCenter): - self.services = serviceCenter - self.user = serviceCenter.user - # self.services.workflow is now the ChatWorkflow object (stable during workflow execution) - self.interfaceDbChat = serviceCenter.interfaceDbChat - self.interfaceDbComponent = serviceCenter.interfaceDbComponent - self.interfaceDbApp = serviceCenter.interfaceDbApp - self._progressLogger = None - - def getChatDocumentsFromDocumentList(self, documentList) -> List[ChatDocument]: - """Get ChatDocuments from a DocumentReferenceList. - - Args: - documentList: DocumentReferenceList (required) - - Returns: - List[ChatDocument]: List of ChatDocument objects - """ - from modules.datamodels.datamodelDocref import DocumentReferenceList - - if not isinstance(documentList, DocumentReferenceList): - logger.error(f"getChatDocumentsFromDocumentList: Invalid documentList type: {type(documentList)}. Expected DocumentReferenceList.") - return [] - - # Convert to string list for processing - stringRefs = documentList.to_string_list() - - try: - # Use self.services.workflow which is the ChatWorkflow object (stable during workflow execution) - workflow = self.services.workflow - if not workflow: - logger.error("getChatDocumentsFromDocumentList: No workflow available (self.services.workflow is not set)") - return [] - - workflowId = workflow.id if hasattr(workflow, 'id') else 'NO_ID' - workflowObjId = id(workflow) - logger.debug(f"getChatDocumentsFromDocumentList: input documentList = {stringRefs}") - logger.debug(f"getChatDocumentsFromDocumentList: using workflow.id = {workflowId}, workflow object id = {workflowObjId}") - - # Root cause analysis: Verify workflow.messages integrity and detect workflow changes - self._verifyWorkflowMessagesIntegrity(workflow, workflowId) - - # Debug: list available messages with their labels and document names (filtered by workflowId) - try: - if workflow and hasattr(workflow, 'messages') and workflow.messages: - msgLines = [] - messagesFromOtherWorkflows = [] - for message in workflow.messages: - msgWorkflowId = getattr(message, 'workflowId', None) - # Only include messages that belong to this workflow - if msgWorkflowId and msgWorkflowId != workflowId: - messagesFromOtherWorkflows.append(f"id={getattr(message, 'id', None)}, label={getattr(message, 'documentsLabel', None)}, workflowId={msgWorkflowId}") - continue - # Also skip messages without workflowId (shouldn't happen, but be safe) - if not msgWorkflowId: - messagesFromOtherWorkflows.append(f"id={getattr(message, 'id', None)}, label={getattr(message, 'documentsLabel', None)}, workflowId=Missing") - continue - - label = getattr(message, 'documentsLabel', None) - docNames = [] - if getattr(message, 'documents', None): - for doc in message.documents: - name = getattr(doc, 'fileName', None) or getattr(doc, 'documentName', None) or 'Unnamed' - docNames.append(name) - msgLines.append( - f"- id={getattr(message, 'id', None)}, label={label}, workflowId={msgWorkflowId}, docs={docNames}" - ) - if msgLines: - logger.debug("getChatDocumentsFromDocumentList: available messages (filtered for workflow):\n" + "\n".join(msgLines)) - if messagesFromOtherWorkflows: - logger.warning(f"getChatDocumentsFromDocumentList: Found {len(messagesFromOtherWorkflows)} messages from other workflows in workflow.messages list:\n" + "\n".join(messagesFromOtherWorkflows)) - else: - logger.debug("getChatDocumentsFromDocumentList: no messages available on current workflow") - except Exception as e: - logger.debug(f"getChatDocumentsFromDocumentList: unable to enumerate messages for debug: {e}") - - allDocuments = [] - for docRef in stringRefs: - if docRef.startswith("docItem:"): - # docItem:: or docItem: (filename is optional) - # ALWAYS try to match by documentId first (parts[1] is always the documentId when format is correct) - # Both formats are supported: docItem: and docItem:: - parts = docRef.split(':') - if len(parts) >= 2: - docId = parts[1] # This should be the documentId (UUID) - docFound = False - - # ALWAYS try to match by documentId first (regardless of number of parts) - # This handles both formats: - # - docItem: (without filename - still works) - # - docItem:: (with filename - preferred) - for message in workflow.messages: - # Validate message belongs to this workflow - msgWorkflowId = getattr(message, 'workflowId', None) - if not msgWorkflowId or msgWorkflowId != workflowId: - continue - - if message.documents: - for doc in message.documents: - if doc.id == docId: - allDocuments.append(doc) - docFound = True - logger.debug(f"Matched document reference '{docRef}' to document {doc.id} (fileName: {getattr(doc, 'fileName', 'unknown')}) by documentId") - break - if docFound: - break - - # Fallback: If not found by documentId and it looks like a filename (has file extension), try filename matching - # This handles cases where AI incorrectly generates docItem:filename.docx - if not docFound and '.' in docId and len(parts) == 2: - # Format: docItem:filename (AI generated wrong format) - try to match by filename - filename = parts[1] - logger.warning(f"Document reference '{docRef}' not found by documentId, attempting to match by filename: {filename}") - - for message in workflow.messages: - # Validate message belongs to this workflow - msgWorkflowId = getattr(message, 'workflowId', None) - if not msgWorkflowId or msgWorkflowId != workflowId: - continue - - if message.documents: - for doc in message.documents: - docFileName = getattr(doc, 'fileName', '') - # Match filename exactly or by base name (without path) - if docFileName == filename or docFileName.endswith(filename): - allDocuments.append(doc) - docFound = True - logger.info(f"Matched document reference '{docRef}' to document {doc.id} by filename {docFileName}") - break - if docFound: - break - - if not docFound: - logger.error(f"Could not resolve document reference '{docRef}' - no document found with filename '{filename}'") - elif not docFound: - logger.error(f"Could not resolve document reference '{docRef}' - no document found with documentId '{docId}'") - elif docRef.startswith("docList:"): - # docList::