From d3dbca7289daa97f054fe873af651206edfccdb2 Mon Sep 17 00:00:00 2001 From: Ida Dittrich Date: Mon, 23 Feb 2026 07:32:40 +0100 Subject: [PATCH] nicht fertig; Stand Kessler Demo --- modules/features/chatbot/bridges/ai.py | 187 +++- modules/features/chatbot/bridges/memory.py | 51 +- modules/features/chatbot/chatbot.py | 498 +++++++-- modules/features/chatbot/chatbotConstants.py | 12 +- modules/features/chatbot/config.py | 16 +- .../chatbot/interfaceFeatureChatbot.py | 976 +++++++----------- modules/features/chatbot/mainChatbot.py | 15 +- .../features/chatbot/routeFeatureChatbot.py | 261 +++-- modules/features/chatbot/service.py | 302 +++++- modules/features/chatbotV2/__init__.py | 3 + .../features/chatbotV2/bridges/__init__.py | 8 + modules/features/chatbotV2/bridges/memory.py | 187 ++++ modules/features/chatbotV2/chatbotV2.py | 210 ++++ modules/features/chatbotV2/config.py | 98 ++ .../chatbotV2/contextChunkRetrieval.py | 272 +++++ .../chatbotV2/contextExtractionLangGraph.py | 160 +++ .../chatbotV2/datamodelFeatureChatbotV2.py | 85 ++ .../chatbotV2/interfaceFeatureChatbotV2.py | 440 ++++++++ modules/features/chatbotV2/mainChatbotV2.py | 250 +++++ .../chatbotV2/routeFeatureChatbotV2.py | 350 +++++++ .../features/chatbotV2/serviceChatbotV2.py | 258 +++++ modules/interfaces/interfaceDbManagement.py | 14 +- modules/interfaces/interfaceRbac.py | 7 +- modules/routes/routeAdminFeatures.py | 5 + modules/routes/routeSystem.py | 5 + modules/services/serviceAi/mainServiceAi.py | 10 +- modules/system/registry.py | 15 +- scripts/script_db_export_migration.py | 6 +- scripts/script_db_init_chatbot.py | 101 ++ scripts/script_db_init_chatbotv2.py | 102 ++ 30 files changed, 3957 insertions(+), 947 deletions(-) create mode 100644 modules/features/chatbotV2/__init__.py create mode 100644 modules/features/chatbotV2/bridges/__init__.py create mode 100644 modules/features/chatbotV2/bridges/memory.py create mode 100644 modules/features/chatbotV2/chatbotV2.py create mode 100644 modules/features/chatbotV2/config.py create mode 100644 modules/features/chatbotV2/contextChunkRetrieval.py create mode 100644 modules/features/chatbotV2/contextExtractionLangGraph.py create mode 100644 modules/features/chatbotV2/datamodelFeatureChatbotV2.py create mode 100644 modules/features/chatbotV2/interfaceFeatureChatbotV2.py create mode 100644 modules/features/chatbotV2/mainChatbotV2.py create mode 100644 modules/features/chatbotV2/routeFeatureChatbotV2.py create mode 100644 modules/features/chatbotV2/serviceChatbotV2.py create mode 100644 scripts/script_db_init_chatbot.py create mode 100644 scripts/script_db_init_chatbotv2.py diff --git a/modules/features/chatbot/bridges/ai.py b/modules/features/chatbot/bridges/ai.py index 98871b75..283a9e4e 100644 --- a/modules/features/chatbot/bridges/ai.py +++ b/modules/features/chatbot/bridges/ai.py @@ -7,7 +7,8 @@ Implements LangChain BaseChatModel interface using AI center models. import logging import asyncio -from typing import Any, AsyncIterator, Dict, List, Optional +import time +from typing import Any, AsyncIterator, Callable, Dict, List, Optional from datetime import datetime from langchain_core.language_models.chat_models import BaseChatModel @@ -28,6 +29,7 @@ from modules.datamodels.datamodelAi import ( AiModel, AiModelCall, AiModelResponse, + AiCallResponse, AiCallOptions, OperationTypeEnum, ProcessingModeEnum, @@ -36,6 +38,15 @@ from modules.datamodels.datamodelUam import User logger = logging.getLogger(__name__) +# Workflow-level store for allowed_providers (survives LangGraph/bind_tools execution context +# where instance attributes may be lost when model is wrapped or serialized) +_workflow_allowed_providers: Dict[str, List[str]] = {} + + +def clear_workflow_allowed_providers(workflow_id: str) -> None: + """Remove workflow from registry when stream completes to avoid memory growth.""" + _workflow_allowed_providers.pop(workflow_id, None) + class AICenterChatModel(BaseChatModel): """ @@ -48,6 +59,9 @@ class AICenterChatModel(BaseChatModel): user: User, operation_type: OperationTypeEnum = OperationTypeEnum.DATA_ANALYSE, processing_mode: ProcessingModeEnum = ProcessingModeEnum.DETAILED, + billing_callback: Optional[Callable[[AiCallResponse], None]] = None, + workflow_id: Optional[str] = None, + allowed_providers: Optional[List[str]] = None, **kwargs ): """ @@ -57,6 +71,9 @@ class AICenterChatModel(BaseChatModel): user: Current user for RBAC and model selection operation_type: Operation type for model selection processing_mode: Processing mode for model selection + billing_callback: Optional callback invoked after each _agenerate with AiCallResponse for billing + workflow_id: Optional workflow/conversation ID for billing context + allowed_providers: Optional list of allowed provider connector types (empty/None = all) **kwargs: Additional arguments passed to BaseChatModel """ super().__init__(**kwargs) @@ -65,6 +82,12 @@ class AICenterChatModel(BaseChatModel): object.__setattr__(self, "operation_type", operation_type) object.__setattr__(self, "processing_mode", processing_mode) object.__setattr__(self, "_selected_model", None) + object.__setattr__(self, "_billing_callback", billing_callback) + object.__setattr__(self, "_workflow_id", workflow_id) + object.__setattr__(self, "_allowed_providers", allowed_providers or []) + # Store in workflow-level registry so it survives when instance attrs are lost (e.g. bind_tools) + if workflow_id and allowed_providers: + _workflow_allowed_providers[workflow_id] = list(allowed_providers) @property def _llm_type(self) -> str: @@ -115,10 +138,24 @@ class AICenterChatModel(BaseChatModel): rbacInstance=rbac_instance ) - # Create options for model selector + # Allowed providers: instance attr or workflow store (lost in LangGraph/bind_tools context) + workflow_id = getattr(self, '_workflow_id', None) + allowed = ( + (_workflow_allowed_providers.get(workflow_id) if workflow_id else None) + or getattr(self, '_allowed_providers', None) + or [] + ) + if allowed: + logger.info(f"AICenterChatModel _select_model: applying allowedProviders={allowed}") + filtered = [m for m in available_models if m.connectorType in allowed] + if filtered: + available_models = filtered + else: + logger.warning(f"No models match allowedProviders {allowed}, using all RBAC-permitted models") options = AiCallOptions( operationType=self.operation_type, - processingMode=self.processing_mode + processingMode=self.processing_mode, + allowedProviders=allowed if allowed else None ) # Select model @@ -238,7 +275,15 @@ class AICenterChatModel(BaseChatModel): # Convert messages to AI center format ai_messages = self._convert_messages_to_ai_format(messages) - + + # Compute input bytes for billing (sum of message content lengths) + input_bytes = sum( + len((m.get("content") or "").encode("utf-8")) + for m in ai_messages + if isinstance(m.get("content"), str) + ) + start_time = time.time() + # If tools are bound, add tool definitions to the system message # This ensures the model knows about available tools # Some models need explicit tool definitions to enable tool calling @@ -282,12 +327,10 @@ class AICenterChatModel(BaseChatModel): "content": tools_note.strip() }) - # Convert LangChain tools to OpenAI tool format for potential use - # Note: The actual tool calling is handled by the connector if it supports it - # This conversion is kept for potential future use or connector support + # Convert LangChain tools to OpenAI/function-calling format (used by OpenAI and Ollama-compatible APIs) openai_tools = None - if tools and self._selected_model.connectorType == "openai": - # Convert LangChain tools to OpenAI tool format + if tools and self._selected_model.connectorType in ("openai", "privatellm"): + # Build tool schema in OpenAI format (Ollama uses same format for tool calling) openai_tools = [] for tool in tools: if hasattr(tool, "name") and hasattr(tool, "description"): @@ -355,51 +398,66 @@ class AICenterChatModel(BaseChatModel): ) ) - # If tools are bound and this is an OpenAI model, we need to call the API directly - # with tools included, since the connector interface doesn't support tools - if openai_tools and self._selected_model.connectorType == "openai": - # Call OpenAI API directly with tools (like legacy ChatAnthropic does) + # If tools are bound, use OpenAI-compatible API (OpenAI or Private-LLM Ollama endpoint) + if openai_tools and self._selected_model.connectorType in ("openai", "privatellm"): import httpx + import json as _json from modules.shared.configuration import APP_CONFIG - - api_key = APP_CONFIG.get('Connector_AiOpenai_API_SECRET') - if not api_key: - raise ValueError("OpenAI API key not configured") - + + if self._selected_model.connectorType == "openai": + api_url = self._selected_model.apiUrl + api_key = APP_CONFIG.get("Connector_AiOpenai_API_SECRET") + if not api_key: + raise ValueError("OpenAI API key not configured") + headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + ollama_model = self._selected_model.name + else: + # privatellm: use Ollama OpenAI-compatible /v1/chat/completions (same service, same provider) + base_url = self._selected_model.apiUrl.replace("/api/analyze", "") + api_url = f"{base_url}/v1/chat/completions" + api_key = APP_CONFIG.get("Connector_AiPrivateLlm_API_SECRET") + headers = {"Content-Type": "application/json"} + if api_key: + headers["X-API-Key"] = api_key + # Ollama needs the underlying model name (e.g. qwen2.5:7b), not poweron-text-general + ollama_model = getattr(self._selected_model, "version", None) or self._selected_model.name + payload = { - "model": self._selected_model.name, + "model": ollama_model, "messages": ai_messages, "tools": openai_tools, - "tool_choice": "auto", # Let model decide when to use tools + "tool_choice": "auto", "temperature": self._selected_model.temperature, - "max_tokens": self._selected_model.maxTokens + "max_tokens": self._selected_model.maxTokens, } - + + use_connector_fallback = False async with httpx.AsyncClient(timeout=600.0) as client: - response_obj = await client.post( - self._selected_model.apiUrl, - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json" - }, - json=payload - ) - - if response_obj.status_code != 200: - error_msg = f"OpenAI API error: {response_obj.status_code} - {response_obj.text}" + response_obj = await client.post(api_url, headers=headers, json=payload) + + if response_obj.status_code == 404 and self._selected_model.connectorType == "privatellm": + logger.warning( + "Private-LLM /v1/chat/completions not found (404). Falling back to /api/analyze. " + "Tool calling will not work until the service exposes an OpenAI-compatible endpoint." + ) + use_connector_fallback = True + elif response_obj.status_code != 200: + error_msg = f"AI API error ({self._selected_model.connectorType}): {response_obj.status_code} - {response_obj.text}" logger.error(error_msg) raise ValueError(error_msg) - + + if use_connector_fallback: + if not self._selected_model.functionCall: + raise ValueError(f"Model {self._selected_model.displayName} has no functionCall defined") + response = await self._selected_model.functionCall(model_call) + else: response_json = response_obj.json() choice = response_json["choices"][0] message = choice["message"] - - # Extract content and tool calls + content = message.get("content", "") tool_calls_raw = message.get("tool_calls") - - # Convert OpenAI tool_calls format to LangChain format - # LangChain expects: [{"id": "...", "name": "...", "args": {...}}] + tool_calls = None if tool_calls_raw: tool_calls = [] @@ -407,29 +465,24 @@ class AICenterChatModel(BaseChatModel): func_data = tc.get("function", {}) func_name = func_data.get("name") func_args_str = func_data.get("arguments", "{}") - - # Parse JSON arguments string to dict - import json try: - func_args = json.loads(func_args_str) if isinstance(func_args_str, str) else func_args_str - except: + func_args = _json.loads(func_args_str) if isinstance(func_args_str, str) else func_args_str + except Exception: func_args = {} - tool_calls.append({ "id": tc.get("id"), "name": func_name, - "args": func_args + "args": func_args, }) - - # Create response object + response = AiModelResponse( content=content or "", success=True, modelId=self._selected_model.name, metadata={ "response_id": response_json.get("id", ""), - "tool_calls": tool_calls - } + "tool_calls": tool_calls, + }, ) else: # No tools or not OpenAI - use connector normally @@ -440,7 +493,35 @@ class AICenterChatModel(BaseChatModel): if not response.success: raise ValueError(f"AI model call failed: {response.error or 'Unknown error'}") - + + # Billing: compute price and invoke callback + output_bytes = len((response.content or "").encode("utf-8")) + processing_time = time.time() - start_time + price_chf = 0.0 + if getattr(self._selected_model, "calculatepriceCHF", None): + try: + price_chf = self._selected_model.calculatepriceCHF( + processing_time, input_bytes, output_bytes + ) + except Exception as e: + logger.warning(f"Billing: price calculation failed: {e}") + billing_callback = getattr(self, "_billing_callback", None) + if billing_callback: + try: + ai_response = AiCallResponse( + content=response.content or "", + modelName=self._selected_model.name, + provider=getattr(self._selected_model, "connectorType", "unknown") or "unknown", + priceCHF=price_chf, + processingTime=processing_time, + bytesSent=input_bytes, + bytesReceived=output_bytes, + errorCount=0, + ) + billing_callback(ai_response) + except Exception as e: + logger.error(f"Billing callback error: {e}") + # Extract tool calls from response metadata if present tool_calls = None if response.metadata: @@ -471,7 +552,9 @@ class AICenterChatModel(BaseChatModel): bound_model = AICenterChatModel( user=self.user, operation_type=self.operation_type, - processing_mode=self.processing_mode + processing_mode=self.processing_mode, + billing_callback=getattr(self, "_billing_callback", None), + workflow_id=getattr(self, "_workflow_id", None), ) object.__setattr__(bound_model, "_selected_model", self._selected_model) # Store tools for potential use in message conversion diff --git a/modules/features/chatbot/bridges/memory.py b/modules/features/chatbot/bridges/memory.py index ea74db4e..234dc041 100644 --- a/modules/features/chatbot/bridges/memory.py +++ b/modules/features/chatbot/bridges/memory.py @@ -23,31 +23,43 @@ class CheckpointTuple(NamedTuple): pending_writes: Optional[List[Tuple[str, Any]]] = None from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage, ToolMessage -from modules.interfaces.interfaceDbChat import getInterface -from modules.datamodels.datamodelChat import ChatMessage, ChatWorkflow +from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface +from modules.features.chatbot.interfaceFeatureChatbot import ChatbotMessage from modules.datamodels.datamodelUam import User from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) +def _sanitize_llm_response(text: str) -> str: + """Strip chat template tokens and trailing junk that some models leak.""" + if not text or not isinstance(text, str): + return text or "" + for sentinel in ("<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|user|>", "<|assistant|>"): + if sentinel in text: + text = text.split(sentinel)[0] + return text.strip() + + class DatabaseCheckpointer(BaseCheckpointSaver): """ - Custom LangGraph checkpointer that uses the existing database interface. - Maps LangGraph thread_id to workflow.id and stores messages in the existing format. + Custom LangGraph checkpointer that uses the chatbot's own database interface. + Maps LangGraph thread_id to conversation.id; stores messages via interface (workflowId maps to conversationId). """ - def __init__(self, user: User, workflow_id: str): + def __init__(self, user: User, workflow_id: str, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): """ Initialize the database checkpointer. Args: user: Current user for database access workflow_id: Workflow ID (maps to LangGraph thread_id) + mandateId: Mandate ID for proper data isolation + featureInstanceId: Feature instance ID for proper data isolation """ self.user = user self.workflow_id = workflow_id - self.interface = getInterface(user) + self.interface = getChatbotInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) def _convert_langchain_to_db_message( self, @@ -101,7 +113,7 @@ class DatabaseCheckpointer(BaseCheckpointSaver): def _convert_db_to_langchain_messages( self, - messages: List[ChatMessage] + messages: List[ChatbotMessage] ) -> List[BaseMessage]: """ Convert database messages to LangChain messages. @@ -183,37 +195,40 @@ class DatabaseCheckpointer(BaseCheckpointSaver): elif isinstance(msg, AIMessage): # Check if this message has tool_calls tool_calls = getattr(msg, "tool_calls", None) - - # Skip messages with tool_calls - these are intermediate tool call requests if tool_calls and len(tool_calls) > 0: logger.debug(f"Skipping intermediate AIMessage with tool_calls for workflow {thread_id}") continue - - # Store all other AIMessages (final answers) + # Skip agent_sql_plan output (raw SQL block) - only store agent_formulate final answer + content = msg.content if isinstance(msg.content, str) else str(msg.content) + cu = (content or "").strip().upper() + if content and ( + content.strip().startswith("```") + or (cu.startswith("SELECT") and ("FROM" in cu or "JOIN" in cu)) + ): + logger.debug(f"Skipping intermediate SQL AIMessage for workflow {thread_id}") + continue checkpoint_user_assistant_messages.append(msg) # Only store new messages that aren't already in the database new_messages_to_store = [] for msg in checkpoint_user_assistant_messages: - # Determine role role = "user" if isinstance(msg, HumanMessage) else "assistant" content = msg.content if isinstance(msg.content, str) else str(msg.content) - - # Skip empty messages (they might be status updates) + if isinstance(msg, AIMessage): + content = _sanitize_llm_response(content) if not content or not content.strip(): continue - - # Check if this message already exists content_key = (role, content) if content_key not in existing_content_set: + if isinstance(msg, AIMessage) and msg.content != content: + msg = AIMessage(content=content) new_messages_to_store.append(msg) - existing_content_set.add(content_key) # Mark as seen to avoid duplicates in this batch + existing_content_set.add(content_key) # Store only the new messages if new_messages_to_store: for i, msg in enumerate(new_messages_to_store, 1): sequence_nr = existing_count + i - # Convert to database format db_message_data = self._convert_langchain_to_db_message( msg, diff --git a/modules/features/chatbot/chatbot.py b/modules/features/chatbot/chatbot.py index 436f54e5..7bdbb183 100644 --- a/modules/features/chatbot/chatbot.py +++ b/modules/features/chatbot/chatbot.py @@ -2,12 +2,14 @@ # All rights reserved. """Chatbot domain logic.""" +import re import logging from dataclasses import dataclass, field from typing import Annotated, AsyncIterator, Any, List, Optional, TYPE_CHECKING from pydantic import BaseModel from langchain_core.messages import ( + AIMessage, BaseMessage, HumanMessage, SystemMessage, @@ -17,7 +19,6 @@ from langchain_core.messages import ( from langgraph.graph.message import add_messages from langgraph.graph import StateGraph, START, END from langgraph.graph.state import CompiledStateGraph -from langgraph.prebuilt import ToolNode from modules.features.chatbot.bridges.ai import AICenterChatModel from modules.features.chatbot.bridges.memory import DatabaseCheckpointer @@ -36,10 +37,136 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _tool_output_to_markdown_table(raw: str) -> str: + """ + Convert sqlite_query tool output to a markdown table for deterministic display. + Reduces model hallucination by providing a ready-to-copy table. + Format: "Query returned N rows:\\nColumns: A, B, C\\n1. A: x, B: y, C: z\\n..." + """ + if not raw or not raw.strip(): + return raw + lines = [ln.strip() for ln in raw.strip().split("\n") if ln.strip()] + if len(lines) < 2: + return raw + # Parse header + row_count_line = lines[0] # "Query returned 20 rows:" + cols_line = next((ln for ln in lines if ln.lower().startswith("columns:")), None) + if not cols_line: + return raw + headers = [h.strip() for h in cols_line.replace("Columns:", "").split(",")] + if not headers: + return raw + # Parse data rows (1. Col: val, Col: val) + rows = [] + for ln in lines: + if re.match(r"^\d+\.\s+", ln): + rest = re.sub(r"^\d+\.\s+", "", ln) + row = {} + for part in rest.split(", "): + if ": " in part: + k, v = part.split(": ", 1) + row[k.strip()] = str(v).strip() + if row: + rows.append([row.get(h, "") for h in headers]) + if not rows: + return raw + # Build markdown table + sep = " | " + header_row = sep.join(headers) + div_row = sep.join(["---"] * len(headers)) + data_rows = [sep.join(str(c) for c in r) for r in rows] + table = "\n".join([header_row, div_row] + data_rows) + suffix = "" + if "(Showing first" in raw or "of " in raw: + m = re.search(r"\(Showing first (\d+) of (\d+) rows\)", raw) + if m: + suffix = f"\n\nZeige {m.group(1)} von {m.group(2)} Artikeln." + return f"{row_count_line}\n\n{table}{suffix}" + + +def _sanitize_llm_response(text: str) -> str: + """Strip chat template tokens and trailing junk that some models leak.""" + if not text or not isinstance(text, str): + return text or "" + for sentinel in ("<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|user|>", "<|assistant|>"): + if sentinel in text: + text = text.split(sentinel)[0] + return text.strip() + + +# Natural language markers to split system prompt into context sections +_SPLIT_MARKERS = { + "schema_start": "Die Datenbank enthält", + "schema_start_alt": "Die Datenbank enthält die Tabellen", + "response_structure_start": "Antwortstruktur ist strikt", + "response_structure_alt": "Antwortstruktur:", + "response_structure_fallback": "Antwortstruktur", +} + + +def _split_system_prompt(prompt: str) -> dict: + """ + Split system prompt by natural language section markers. + Returns: {intro, schema, response_structure} + - intro: Role, tools, general instructions (before schema) + - schema: Database tables, SQL rules, column definitions (for SQL generation) + - response_structure: Mandatory answer format (Einleitungssatz, Tabelle, etc.) + """ + if not prompt or not isinstance(prompt, str): + return {"intro": "", "schema": "", "response_structure": ""} + + text = prompt.strip() + intro_end = len(text) + schema_start_idx = -1 + schema_end = len(text) + response_start_idx = -1 + + # Find schema start + for marker in (_SPLIT_MARKERS["schema_start"], _SPLIT_MARKERS["schema_start_alt"]): + idx = text.find(marker) + if idx >= 0: + schema_start_idx = idx + intro_end = idx + break + + # Find response structure start + for marker in ( + _SPLIT_MARKERS["response_structure_start"], + _SPLIT_MARKERS["response_structure_alt"], + _SPLIT_MARKERS["response_structure_fallback"], + ): + idx = text.find(marker) + if idx >= 0: + response_start_idx = idx + schema_end = idx if schema_start_idx >= 0 else len(text) + break + + intro = text[:intro_end].strip() if intro_end > 0 else "" + schema = ( + text[schema_start_idx:schema_end].strip() + if schema_start_idx >= 0 and schema_end > schema_start_idx + else "" + ) + response_structure = ( + text[response_start_idx:].strip() + if response_start_idx >= 0 + else "" + ) + + # Fallback: if no markers found, use full prompt for intro + if not intro and not schema and not response_structure: + intro = text + elif not response_structure and intro: + response_structure = intro # Use intro's format hints as fallback + + return {"intro": intro, "schema": schema, "response_structure": response_structure} + + class ChatState(BaseModel): """Represents the state of a chat session.""" messages: Annotated[List[BaseMessage], add_messages] + plan: Optional[str] = None # Planner routing: "SQL", "TAVILY", "BOTH", "NONE" @dataclass @@ -135,6 +262,8 @@ class Chatbot: ) -> CompiledStateGraph[ChatState, None, ChatState, ChatState]: """Builds the chatbot application workflow using LangGraph. + Supports small context windows via planning phase and tiered prompts. + Args: memory: The chat memory to use. tools: The list of tools the chatbot can use. @@ -142,83 +271,303 @@ class Chatbot: Returns: A compiled state graph representing the chatbot application. """ - llm_with_tools = self.model.bind_tools(tools=tools) + # Build tool subsets per agent type + tools_by_name = {t.name: t for t in tools} + sql_tool = tools_by_name.get("sqlite_query") + tavily_tool = tools_by_name.get("tavily_search") + streaming_tool = tools_by_name.get("send_streaming_message") + tools_sql = [t for t in [sql_tool, tavily_tool, streaming_tool] if t is not None] + tools_tavily = [t for t in [tavily_tool, streaming_tool] if t is not None] + llm_plain = self.model + # SQL path uses structured prompts + parse/execute (no native tool calling) - fits /api/analyze + llm_tavily = self.model.bind_tools(tools=tools_tavily) if tools_tavily else self.model - def select_window(msgs: List[BaseMessage]) -> List[BaseMessage]: - """Selects a window of messages that fit within the context window size. + # Minimal planner prompt (~250 tokens) - fits any 8K+ model + # Explicit: Lager, Bestand, Artikel, wie viele = SQL (Datenbank) + PLANNER_SYSTEM = ( + "Du bist ein Assistent. Antworte NUR mit einem Wort: SQL, TAVILY, BOTH oder NONE.\n" + "SQL = Fragen zu Lager, Bestand, Artikel, Preisen, wie viele, Anzahl (Datenbankabfrage).\n" + "TAVILY = Internetsuche, Produktinfos außerhalb der DB, Markttrends.\n" + "BOTH = beides nötig. NONE = nur Begrüßung oder Danksagung, keine Daten nötig.\n" + "Beispiele: 'wie viele X auf Lager' -> SQL, 'Infos zu Produkt Y' -> TAVILY." + ) - Args: - msgs: The list of messages to select from. + # Truncation suffix for schema when prompt is cut + SCHEMA_TRUNCATION_SUFFIX = ( + "\n\n[... Schema gekürzt. Wichtige Tabellen: Artikel, Lagerplatz_Artikel, Einkaufspreis, Lagerplatz. " + "Artikel-Spalte: a.\"Artikelbezeichnung\". " + "JOIN: Artikel a, Lagerplatz_Artikel l ON a.I_ID = l.R_ARTIKEL, Lagerplatz lp ON l.R_LAGERPLATZ = lp.I_ID.]" + ) - Returns: - A list of messages that fit within the context window size. - """ + # Structured output for /api/analyze (no tool calls): model outputs SQL in code block, we parse and execute + SQL_PLAN_SUFFIX = ( + "\n\n--- AUSGABEFORMAT (PFLICHT) ---\n" + "Antworte NUR mit einer SQL SELECT-Abfrage in diesem Format:\n" + "```sql\nDEINE_SQL_QUERY\n```\n" + "KRITISCH bei 'wie viele X auf Lager': Liefere ARTIKELZEILEN (Artikelnummer, Artikelbezeichnung, Bestand) " + "mit LIMIT 20, NICHT nur SELECT COUNT(*). Ohne Detailzeilen kann keine Tabelle angezeigt werden. " + "Gesamtanzahl optional via Unterabfrage im SELECT." + ) + + bytes_per_token = 3 # Balanced estimate for mixed content + reserved_tokens = 3000 # Tools block + conversation overhead + + def _get_context_length() -> int: + """Get selected model's context length; pre-select if needed.""" + if hasattr(self.model, "_selected_model") and self.model._selected_model: + return getattr(self.model._selected_model, "contextLength", 128000) + return 128000 + + def _truncate_system_prompt(full_prompt: str, max_chars: int, suffix: str = "") -> str: + """Truncate system prompt to fit context budget.""" + if len(full_prompt) <= max_chars: + return full_prompt + return full_prompt[: max_chars - len(suffix)] + suffix + + # Split system prompt by natural language sections for targeted context + _prompt_sections = _split_system_prompt(self.system_prompt) + + def select_window(msgs: List[BaseMessage], max_tokens_override: Optional[int] = None) -> List[BaseMessage]: + """Selects a window of messages that fit within the context window size.""" def approx_counter(items: List[BaseMessage]) -> int: - """Approximate token counter for messages. - - Args: - items: List of messages to count tokens for. - - Returns: - Approximate number of tokens in the messages. - """ return sum(len(getattr(m, "content", "") or "") for m in items) - # Use model's context length if available, otherwise default - max_tokens = getattr(self.model._selected_model, "contextLength", 128000) if hasattr(self.model, "_selected_model") and self.model._selected_model else 128000 - + max_tokens = max_tokens_override or _get_context_length() return trim_messages( msgs, strategy="last", token_counter=approx_counter, - max_tokens=int(max_tokens * 0.8), # Use 80% of context window + max_tokens=int(max_tokens * 0.8), start_on="human", end_on=("human", "tool"), include_system=True, ) - async def agent_node(state: ChatState) -> dict: - """Agent node for the chatbot workflow. + async def planner_node(state: ChatState) -> dict: + """Planner: minimal prompt, no tools. Outputs SQL/TAVILY/BOTH/NONE. + Does NOT add planner message to chat - only sets state.plan for routing.""" + human_msgs = [m for m in state.messages if isinstance(m, HumanMessage)] + last_human = human_msgs[-1].content if human_msgs else "" + window = [ + SystemMessage(content=PLANNER_SYSTEM), + HumanMessage(content=last_human), + ] + plan = "SQL" + try: + response = await llm_plain.ainvoke(window) + except ValueError as exc: + if "No suitable model found" in str(exc): + logger.warning(f"Planner model selection failed: {exc}") + return {"plan": plan} + raise + content = (response.content or "").strip().upper() + for keyword in ("SQL", "TAVILY", "BOTH", "NONE"): + if keyword in content: + plan = keyword + break + return {"plan": plan} - Args: - state: The current chat state. + # Keywords that indicate database/inventory query - override NONE to SQL + _SQL_KEYWORDS = ( + "lager", "bestand", "artikel", "wie viele", "anzahl", "preis", + "lieferant", "lieferanten", "bestellen", "verfügbar", "inventar" + ) - Returns: - The updated chat state after processing. - """ - # Select the message window to fit in context (trim if needed) - window = select_window(state.messages) + def route_by_plan(state: ChatState) -> str: + """Route from planner to agent_sql_plan, agent_tavily, or agent_answer.""" + plan = (state.plan or "SQL").upper() + # Override NONE when user clearly asks for inventory/data (e.g. "wie viele LEDs auf Lager") + if plan == "NONE" and sql_tool: + last_user = "" + for m in reversed(state.messages): + if isinstance(m, HumanMessage): + last_user = (m.content or "").lower() + break + if any(kw in last_user for kw in _SQL_KEYWORDS): + logger.info("Planner returned NONE but user asked inventory question - routing to SQL") + plan = "SQL" + if plan in ("SQL", "BOTH") and sql_tool: + return "agent_sql_plan" + if plan == "TAVILY" and tavily_tool: + return "agent_tavily" + return "agent_answer" - # Ensure the system prompt is present at the start - if not window or not isinstance(window[0], SystemMessage): - window = [SystemMessage(content=self.system_prompt)] + window - - # Call the LLM with tools (use ainvoke for async) - response = await llm_with_tools.ainvoke(window) - - # Return the new state + async def _agent_common( + state: ChatState, + system_content: str, + llm, + node_name: str, + ) -> dict: + """Shared logic for agent nodes.""" + msgs = select_window(state.messages) + if not msgs or not isinstance(msgs[0], SystemMessage): + window = [SystemMessage(content=system_content)] + msgs + else: + window = [SystemMessage(content=system_content)] + [m for m in msgs if not isinstance(m, SystemMessage)] + try: + response = await llm.ainvoke(window) + except ValueError as exc: + if "No suitable model found" in str(exc): + logger.warning(f"{node_name} model selection failed: {exc}") + response = AIMessage( + content=( + "Es tut mir leid, derzeit steht kein passendes KI-Modell für diese Anfrage zur Verfügung. " + "Bitte versuchen Sie es später erneut oder wenden Sie sich an den Administrator." + ) + ) + else: + raise return {"messages": [response]} - def should_continue(state: ChatState) -> str: - """Determines whether to continue the workflow or end it. + async def agent_sql_plan_node(state: ChatState) -> dict: + """Generate SQL. Uses schema section + minimal intro. Output: ```sql...``` for parse/execute.""" + ctx_len = _get_context_length() + max_system_chars = max(1000, int(ctx_len * 0.8 - reserved_tokens) * bytes_per_token) - len(SQL_PLAN_SUFFIX) + # Prefer schema section; add short intro if space allows + schema_part = _prompt_sections["schema"] or _prompt_sections["intro"] + intro_part = _prompt_sections["intro"][:400] if _prompt_sections["intro"] else "" + combined = f"{intro_part}\n\n{schema_part}" if intro_part else schema_part + system_content = _truncate_system_prompt( + combined, max_system_chars, SCHEMA_TRUNCATION_SUFFIX + ) + SQL_PLAN_SUFFIX + return await _agent_common(state, system_content, llm_plain, "agent_sql_plan") - This conditional edge is called after the agent node to decide - whether to continue to the tools node (if the last message contains - tool calls) or to end the workflow (if no tool calls are present). + def _parse_sql_from_content(content: str) -> Optional[str]: + """Extract SQL from ```sql...``` or ```...``` code block. Only allows SELECT.""" + if not content: + return None + match = re.search(r"```(?:sql)?\s*([\s\S]*?)```", content) + if match: + sql = match.group(1).strip() + if sql and sql.upper().strip().startswith("SELECT"): + return sql + # Fallback: find line starting with SELECT + for line in content.split("\n"): + line = line.strip() + if line.upper().startswith("SELECT"): + return line + return None - Args: - state: The current chat state. + def _sanitize_sql_typos(sql: str) -> str: + """Fix common LLM SQL typos that cause syntax errors.""" + if not sql: + return sql + # Fix "CASE WHENLAGerplatz" - missing space after WHEN when followed directly by identifier + sql = re.sub(r"WHEN([A-Za-z_][A-Za-z0-9_.\"]*)", r"WHEN \1", sql, flags=re.IGNORECASE) + # Fix "LAGerplatz_Artikel" / "LAGerplatz" -> correct casing + sql = re.sub(r"\bLAGerplatz_Artikel\b", "Lagerplatz_Artikel", sql) + sql = re.sub(r"\bLAGerplatz\b", "Lagerplatz", sql) + # Preprocessor uses Einkaufspreis (not Einkaufspreis_neu) and m_Artikel (not ARTIKEL) + sql = sql.replace('"Einkaufspreis_neu"', '"Einkaufspreis"') + sql = sql.replace("Einkaufspreis_neu.", "Einkaufspreis.") + sql = re.sub( + r'"Einkaufspreis"\."ARTIKEL"', + '"Einkaufspreis"."m_Artikel"', + sql, + ) + return sql - Returns: - The next node to transition to ("tools" or END). - """ - # Get the last message - last_message = state.messages[-1] + async def parse_execute_sql_node(state: ChatState) -> dict: + """Parse SQL from last AIMessage, execute via preprocessor, add ToolMessage.""" + last_msg = state.messages[-1] if state.messages else None + if not isinstance(last_msg, AIMessage): + return {"messages": [ToolMessage(content="Fehler: Keine AI-Antwort zum Parsen.", tool_call_id="parse_0", name="sqlite_query")]} + sql = _parse_sql_from_content(last_msg.content or "") + if not sql or not sql_tool: + return {"messages": [ToolMessage(content="Konnte keine SQL-Abfrage aus der Antwort extrahieren.", tool_call_id="parse_0", name="sqlite_query")]} + sql = _sanitize_sql_typos(sql) + try: + result = await sql_tool.ainvoke({"query": sql}) + except Exception as e: + logger.error(f"SQL execution failed: {e}") + result = f"Fehler bei der Ausführung: {e}" + return {"messages": [ToolMessage(content=str(result), tool_call_id="parse_0", name="sqlite_query")]} - # Check if the last message contains tool calls - # If so, continue to the tools node; otherwise, end the workflow - return "tools" if getattr(last_message, "tool_calls", None) else END + FORMULATE_TASK = ( + "\n\n--- AKTUELLE AUFGABE ---\n" + "Du erhältst eine Benutzerfrage und die exakten Datenbankergebnisse. " + "KRITISCH: Nutze NUR die gelieferten Daten. Erfinde NIEMALS Daten (keine LED-A01, LED Rot, etc.). " + "Wenn die Ergebnisse NUR eine Zahl enthalten (z.B. '1. COUNT(*): 806'): Reportiere NUR diese Zahl, KEINE erfundene Tabelle. " + "Eine Tabelle darf NUR erstellt werden, wenn echte Zeilen '1. Spalte: Wert, ...' in den Daten stehen. " + "Beachte die obige ANTWORTSTRUKTUR." + ) + + async def agent_formulate_node(state: ChatState) -> dict: + """Formulate final answer. Uses intro + response_structure sections (not schema).""" + human_content = "" + tool_content = "" + for m in state.messages: + if isinstance(m, HumanMessage): + human_content = m.content or "" + if isinstance(m, ToolMessage) and getattr(m, "name", "") == "sqlite_query": + tool_content = m.content or "" + if not tool_content or not tool_content.strip(): + logger.warning("agent_formulate: no tool_content (sqlite_query) in state.messages") + return {"messages": [AIMessage(content="Die Datenbankabfrage konnte keine Ergebnisse liefern. Bitte versuchen Sie es mit einer anderen Formulierung.")]} + # When SQL failed, return error directly - don't let model hallucinate success + if "Query failed" in tool_content or tool_content.strip().startswith("Error"): + err_summary = "Die Datenbankabfrage ist fehlgeschlagen." + if "no such column" in tool_content: + err_summary += " Ein Spaltenname scheint nicht zu passen. Bitte die Anfrage anders formulieren." + return {"messages": [AIMessage(content=err_summary)]} + # Convert to markdown table so model copies exact values instead of reformatting/hallucinating + formatted_data = _tool_output_to_markdown_table(tool_content) + logger.debug(f"agent_formulate: tool_content length={len(tool_content)}, formatted={len(formatted_data)}") + ctx_len = _get_context_length() + max_system_chars = max(3000, int(ctx_len * 0.5) * bytes_per_token) - len(FORMULATE_TASK) + # Use intro + response_structure (mandatory format) + resp_struct = _prompt_sections["response_structure"] or _prompt_sections["intro"] + intro_formulate = _prompt_sections["intro"] + combined = f"{intro_formulate}\n\n{resp_struct}" if intro_formulate != resp_struct else resp_struct + # Fit within context; prefer keeping response_structure intact + if len(combined) + len(FORMULATE_TASK) > max_system_chars: + combined = _truncate_system_prompt(combined, max_system_chars - len(FORMULATE_TASK), "") + system_content = combined + FORMULATE_TASK + prompt = ( + f"Benutzerfrage: {human_content}\n\n" + "--- VORGEGEBENE DATEN (diese Tabelle/Zahlen UNVERÄNDERT in die Antwort übernehmen): ---\n" + f"{formatted_data}\n\n" + "Die obige Tabelle bzw. Zahlen sind die EINZIGEN erlaubten Daten. Kopiere sie 1:1. " + "Berechne keine eigenen Summen/Anzahlen - nutze die gelieferten Werte. Formuliere die Antwort:" + ) + window = [SystemMessage(content=system_content), HumanMessage(content=prompt)] + try: + response = await llm_plain.ainvoke(window) + except ValueError as exc: + if "No suitable model found" in str(exc): + response = AIMessage(content="Es gab einen Fehler bei der Formulierung. Bitte versuchen Sie es erneut.") + else: + raise + # Sanitize: strip leaked chat template tokens (<|im_start|> etc.) and trailing junk + if response.content: + response = AIMessage(content=_sanitize_llm_response(response.content)) + return {"messages": [response]} + + async def agent_tavily_node(state: ChatState) -> dict: + """Agent with Tavily only. Uses intro + response_structure (no schema).""" + resp_struct = _prompt_sections["response_structure"] or "" + intro_tavily = _prompt_sections["intro"] + combined = f"{intro_tavily}\n\n{resp_struct}" if resp_struct else intro_tavily + system_content = _truncate_system_prompt(combined, 6000, "") + return await _agent_common(state, system_content, llm_tavily, "agent_tavily") + + async def agent_answer_node(state: ChatState) -> dict: + """Agent with no tools. Uses intro + response_structure.""" + resp_struct = _prompt_sections["response_structure"] or "" + intro_answer = _prompt_sections["intro"] + combined = f"{intro_answer}\n\n{resp_struct}" if resp_struct else intro_answer + system_content = _truncate_system_prompt(combined, 6000, "") + return await _agent_common(state, system_content, llm_plain, "agent_answer") + + def should_continue_tavily(state: ChatState) -> str: + last = state.messages[-1] + return "tools" if getattr(last, "tool_calls", None) else END + + def route_back(state: ChatState) -> str: + """Route from tools back to agent_tavily (SQL path uses parse_execute_sql, no tools loop).""" + # Tools node is only reached from agent_tavily when it returns tool_calls + return "agent_tavily" if tavily_tool else "agent_answer" async def tools_with_retry(state: ChatState) -> dict: """Tools node with parallel execution and retry logic. @@ -341,13 +690,24 @@ class Chatbot: return result - # Compose the workflow + # Compose the workflow: planner -> route -> agent_* -> tools (Tavily only) or END workflow = StateGraph(ChatState) - workflow.add_node("agent", agent_node) + workflow.add_node("planner", planner_node) + workflow.add_node("agent_sql_plan", agent_sql_plan_node) + workflow.add_node("parse_execute_sql", parse_execute_sql_node) + workflow.add_node("agent_formulate", agent_formulate_node) + workflow.add_node("agent_tavily", agent_tavily_node) + workflow.add_node("agent_answer", agent_answer_node) workflow.add_node("tools", tools_with_retry) - workflow.add_edge(START, "agent") - workflow.add_conditional_edges("agent", should_continue) - workflow.add_edge("tools", "agent") + workflow.add_edge(START, "planner") + workflow.add_conditional_edges("planner", route_by_plan) + # SQL path: agent_sql_plan -> parse_execute_sql -> agent_formulate -> END (no tools, /api/analyze compatible) + workflow.add_edge("agent_sql_plan", "parse_execute_sql") + workflow.add_edge("parse_execute_sql", "agent_formulate") + workflow.add_edge("agent_formulate", END) + workflow.add_conditional_edges("agent_tavily", should_continue_tavily) + workflow.add_edge("agent_answer", END) + workflow.add_conditional_edges("tools", route_back) return workflow.compile(checkpointer=memory) async def chat(self, message: str, chat_id: str = "default") -> List[BaseMessage]: @@ -429,6 +789,8 @@ class Chatbot: ) # Normalize for the frontend (only user/assistant with text content) + # Exclude planner-only and SQL-path intermediate messages from chat display + _planner_only = frozenset(("sql", "tavily", "both", "none")) chat_history_payload: List[dict] = [] for m in final_msgs: if isinstance(m, BaseMessage): @@ -437,8 +799,24 @@ class Chatbot: d = ChatStreamingHelper.dict_message_to_dict(obj=m) else: continue - if d.get("role") in ("user", "assistant") and d.get("content"): - chat_history_payload.append(d) + if d.get("role") not in ("user", "assistant") or not d.get("content"): + continue + content = (d.get("content") or "").strip() + if d.get("role") == "assistant" and content.lower() in _planner_only: + continue # Skip planner routing message + # Skip agent_sql_plan output: ```sql block OR raw SQL (SELECT...FROM/JOIN) + if d.get("role") == "assistant": + cu = content.upper() + if content.startswith("```") or ( + cu.startswith("SELECT") and ("FROM" in cu or "JOIN" in cu) + ): + continue + # Strip leaked chat template tokens (<|im_start|> etc.) from assistant messages + content = _sanitize_llm_response(content) + if not content: + continue + d = {**d, "content": content} + chat_history_payload.append(d) yield { "type": "final", diff --git a/modules/features/chatbot/chatbotConstants.py b/modules/features/chatbot/chatbotConstants.py index 34368550..0366e4b4 100644 --- a/modules/features/chatbot/chatbotConstants.py +++ b/modules/features/chatbot/chatbotConstants.py @@ -36,7 +36,7 @@ async def generate_conversation_name( # Check if AI service is available if not hasattr(services, 'ai') or services.ai is None: logger.warning("AI service not available, generating name from prompt") - return _generate_name_from_prompt(prompt) + return generate_name_from_prompt(prompt) # Ensure AI service is initialized before use await services.ai.ensureAiObjectsInitialized() @@ -80,7 +80,7 @@ Antworte NUR mit dem deutschen Titel, ohne Anführungszeichen oder Erklärungen. if not response or not hasattr(response, 'content') or not response.content: logger.warning("AI response invalid, generating name from prompt") - return _generate_name_from_prompt(prompt) + return generate_name_from_prompt(prompt) logger.info(f"AI response received: {response.content[:100]}...") @@ -102,7 +102,7 @@ Antworte NUR mit dem deutschen Titel, ohne Anführungszeichen oder Erklärungen. english_words = ["search", "find", "show", "display", "query", "article", "product", "item", "led articles", "product search"] if any(word in name_lower for word in english_words): logger.warning(f"AI generated English name '{name}', generating from prompt instead") - return _generate_name_from_prompt(prompt) + return generate_name_from_prompt(prompt) # Limit to 50 characters if len(name) > 50: @@ -114,14 +114,14 @@ Antworte NUR mit dem deutschen Titel, ohne Anführungszeichen oder Erklärungen. return name else: logger.warning(f"Generated name is too short: '{name}', generating from prompt") - return _generate_name_from_prompt(prompt) + return generate_name_from_prompt(prompt) except Exception as e: logger.error(f"Error generating conversation name with AI: {e}", exc_info=True) - return _generate_name_from_prompt(prompt) + return generate_name_from_prompt(prompt) -def _generate_name_from_prompt(prompt: str) -> str: +def generate_name_from_prompt(prompt: str) -> str: """ Generate a conversation name directly from the German prompt. Creates a concise title by extracting key words and formatting them. diff --git a/modules/features/chatbot/config.py b/modules/features/chatbot/config.py index b80409a0..d10ce57e 100644 --- a/modules/features/chatbot/config.py +++ b/modules/features/chatbot/config.py @@ -7,7 +7,7 @@ Loads configuration from the database (FeatureInstance.config JSONB field). import logging from dataclasses import dataclass, field -from typing import Optional, Dict, Any, TYPE_CHECKING +from typing import Optional, Dict, Any, List, TYPE_CHECKING if TYPE_CHECKING: from modules.datamodels.datamodelFeatures import FeatureInstance @@ -63,6 +63,7 @@ class ModelConfig: """Model configuration for a chatbot instance.""" operationType: str = "DATA_ANALYSE" processingMode: str = "BASIC" # Changed from DETAILED for faster responses + allowedProviders: List[str] = field(default_factory=list) # Empty = all providers allowed @dataclass @@ -119,9 +120,11 @@ class ChatbotConfig: # Parse model config with defaults model_data = data.get("model", {}) + allowed_providers = model_data.get("allowedProviders") or data.get("allowedProviders", []) model_config = ModelConfig( operationType=model_data.get("operationType", "DATA_ANALYSE"), - processingMode=model_data.get("processingMode", "DETAILED") + processingMode=model_data.get("processingMode", "DETAILED"), + allowedProviders=allowed_providers if isinstance(allowed_providers, list) else [] ) return cls( @@ -184,7 +187,8 @@ class ChatbotConfig: # Model config defaults - use BASIC for faster responses converted["model"] = { "operationType": "DATA_ANALYSE", - "processingMode": "BASIC" + "processingMode": "BASIC", + "allowedProviders": data.get("allowedProviders", []) } # Copy other fields @@ -213,7 +217,8 @@ class ChatbotConfig: }, "model": { "operationType": self.model.operationType, - "processingMode": self.model.processingMode + "processingMode": self.model.processingMode, + "allowedProviders": self.model.allowedProviders } } @@ -249,8 +254,11 @@ def load_chatbot_config_from_instance(instance: 'FeatureInstance') -> ChatbotCon logger.warning(f"Instance {instance_id} has no config, using minimal defaults") config_data = {} + logger.debug(f"Instance {instance_id} raw config keys: {list(config_data.keys()) if config_data else []}, allowedProviders: {config_data.get('allowedProviders')}") + # Create config from dictionary config = ChatbotConfig.from_dict(config_data, config_id=instance_id) + logger.debug(f"Instance {instance_id} parsed config.model.allowedProviders: {config.model.allowedProviders}") # Cache the config _config_cache[cache_key] = config diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 2c801d11..d7c1e8fa 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -2,13 +2,16 @@ # All rights reserved. """ Interface to Chatbot database and AI Connectors. -Uses the PostgreSQL connector for data access with user/mandate filtering. +Uses the PostgreSQL connector for data access with user/feature-instance filtering. +Chatbot-specific models in poweron_chatbot (separate from workflow engine). """ import logging import uuid import math from typing import Dict, Any, List, Optional, Union +from enum import Enum +from pydantic import BaseModel, Field import asyncio @@ -16,21 +19,93 @@ from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.datamodels.datamodelChat import ( - ChatDocument, - ChatStat, - ChatLog, - ChatMessage, - ChatWorkflow, - WorkflowModeEnum, - UserInputRequest -) +from modules.datamodels.datamodelChat import UserInputRequest +from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp + +# ============================================================================= +# Chatbot-specific Pydantic models for poweron_chatbot (per-instance isolation) +# ============================================================================= + + +class ChatbotDocument(BaseModel): + """Documents attached to chatbot messages.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + messageId: str = Field(description="Foreign key to message") + fileId: str = Field(description="Foreign key to file") + fileName: str = Field(description="Name of the file") + fileSize: int = Field(description="Size of the file") + mimeType: str = Field(description="MIME type of the file") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + actionId: Optional[str] = Field(None, description="ID of the action that created this document") + + +class ChatbotMessage(BaseModel): + """Messages in chatbot conversations. Must match bridge format in memory.py.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + parentMessageId: Optional[str] = Field(None, description="Parent message ID for threading") + documents: List[ChatbotDocument] = Field(default_factory=list, description="Associated documents") + documentsLabel: Optional[str] = Field(None, description="Label for the set of documents") + message: Optional[str] = Field(None, description="Message content") + role: str = Field(description="Role of the message sender") + status: str = Field(description="Status of the message (first, step, last)") + sequenceNr: Optional[int] = Field(default=0, description="Sequence number of the message") + publishedAt: Optional[float] = Field(default=None, description="When the message was published (UTC timestamp)") + success: Optional[bool] = Field(None, description="Whether the message processing was successful") + actionId: Optional[str] = Field(None, description="ID of the action that produced this message") + actionMethod: Optional[str] = Field(None, description="Method of the action that produced this message") + actionName: Optional[str] = Field(None, description="Name of the action that produced this message") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + taskProgress: Optional[str] = Field(None, description="Task progress status") + actionProgress: Optional[str] = Field(None, description="Action progress status") + + +class ChatbotLog(BaseModel): + """Log entries for chatbot conversations.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + message: str = Field(description="Log message") + type: str = Field(description="Log type (info, warning, error, etc.)") + timestamp: float = Field(default_factory=getUtcTimestamp, description="When the log entry was created (UTC timestamp)") + status: Optional[str] = Field(None, description="Status of the log entry") + progress: Optional[float] = Field(None, description="Progress indicator (0.0 to 1.0)") + performance: Optional[Dict[str, Any]] = Field(None, description="Performance metrics") + parentId: Optional[str] = Field(None, description="Parent operation ID") + operationId: Optional[str] = Field(None, description="Operation ID to group related log entries") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + + +class ChatbotWorkflowModeEnum(str, Enum): + WORKFLOW_CHATBOT = "Chatbot" + + +class ChatbotConversation(BaseModel): + """Chatbot conversation container. Per feature-instance isolation via featureInstanceId.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + featureInstanceId: str = Field(description="Feature instance ID for per-instance isolation") + name: Optional[str] = Field(None, description="Name of the conversation") + status: str = Field(default="running", description="Current status") + currentRound: int = Field(default=0, description="Current round number") + lastActivity: float = Field(default_factory=getUtcTimestamp, description="Timestamp of last activity") + startedAt: float = Field(default_factory=getUtcTimestamp, description="When the conversation started") + workflowMode: ChatbotWorkflowModeEnum = Field(default=ChatbotWorkflowModeEnum.WORKFLOW_CHATBOT, description="Workflow mode") + maxSteps: int = Field(default=10, description="Maximum number of iterations") + # Hydrated from child tables (not stored in DB) + logs: List[ChatbotLog] = Field(default_factory=list, description="Conversation logs") + messages: List[ChatbotMessage] = Field(default_factory=list, description="Conversation messages") + + import json from modules.datamodels.datamodelUam import User # DYNAMIC PART: Connectors to the Interface from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.interfaces.interfaceRbac import getRecordsetWithRBAC @@ -51,7 +126,7 @@ def storeDebugMessageAndDocuments(message, currentUser, mandateId=None, featureI - document_###_ (actual file bytes) Args: - message: ChatMessage object to store + message: ChatbotMessage or ChatMessage-like object to store currentUser: Current user for component interface access mandateId: Mandate ID for RBAC context (avoids overwriting singleton state) featureInstanceId: Feature instance ID for RBAC context @@ -86,7 +161,7 @@ def storeDebugMessageAndDocuments(message, currentUser, mandateId=None, featureI # Convert message to dict manually to avoid model_dump() issues message_dict = { "id": message.id, - "workflowId": message.workflowId, + "workflowId": getattr(message, "conversationId", None) or getattr(message, "workflowId", ""), "parentMessageId": message.parentMessageId, "message": message.message, "role": message.role, @@ -193,6 +268,7 @@ class ChatObjects: # Use mandateId from parameter (Request-Context), not from user object self.mandateId = mandateId self.featureInstanceId = featureInstanceId + self.featureCode = "chatbot" # For RBAC buildDataObjectKey self.rbac = None # RBAC interface # Initialize services @@ -363,7 +439,7 @@ class ChatObjects: tableName = modelClass.__name__ from modules.interfaces.interfaceRbac import buildDataObjectKey - objectKey = buildDataObjectKey(tableName, featureCode=self.featureCode if hasattr(self, 'featureCode') else None) + objectKey = buildDataObjectKey(tableName, featureCode=getattr(self, 'featureCode', 'chatbot')) permissions = self.rbac.getUserPermissions( self.currentUser, AccessRuleContext.DATA, @@ -596,11 +672,11 @@ class ChatObjects: - # Workflow methods + # Conversation methods (chatbot-specific, poweron_chatbot) - def getWorkflows(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + def getConversations(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: """ - Returns workflows based on user access level. + Returns conversations for current feature instance based on user access. Supports optional pagination, sorting, and filtering. Args: @@ -611,272 +687,244 @@ class ChatObjects: If pagination is provided: PaginatedResult with items and metadata """ # Use RBAC filtering with featureInstanceId for instance-level isolation - filteredWorkflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, + filteredConversations = getRecordsetWithRBAC(self.db, + ChatbotConversation, self.currentUser, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode="chatbot" ) # If no pagination requested, return all items (no sorting - frontend handles it) if pagination is None: - return filteredWorkflows + return filteredConversations # Apply filtering (if filters provided) if pagination.filters: - filteredWorkflows = self._applyFilters(filteredWorkflows, pagination.filters) + filteredConversations = self._applyFilters(filteredConversations, pagination.filters) # Apply sorting (in order of sortFields) - only if provided by frontend if pagination.sort: - filteredWorkflows = self._applySorting(filteredWorkflows, pagination.sort) + filteredConversations = self._applySorting(filteredConversations, pagination.sort) # Count total items after filters - totalItems = len(filteredWorkflows) + totalItems = len(filteredConversations) totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 # Apply pagination (skip/limit) startIdx = (pagination.page - 1) * pagination.pageSize endIdx = startIdx + pagination.pageSize - pagedWorkflows = filteredWorkflows[startIdx:endIdx] + pagedConversations = filteredConversations[startIdx:endIdx] return PaginatedResult( - items=pagedWorkflows, + items=pagedConversations, totalItems=totalItems, totalPages=totalPages ) - def getWorkflow(self, workflowId: str) -> Optional[ChatWorkflow]: - """Returns a workflow by ID if user has access.""" + def getWorkflows(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: + """Backward-compat alias for getConversations.""" + return self.getConversations(pagination) + + def getConversation(self, conversationId: str) -> Optional[ChatbotConversation]: + """Returns a conversation by ID if user has access.""" # Use RBAC filtering with featureInstanceId for instance-level isolation - workflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, + conversations = getRecordsetWithRBAC(self.db, + ChatbotConversation, self.currentUser, - recordFilter={"id": workflowId}, + recordFilter={"id": conversationId}, mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId + featureInstanceId=self.featureInstanceId, + featureCode="chatbot" ) - if not workflows: + if not conversations: return None - workflow = workflows[0] + conv = conversations[0] try: # Load related data from normalized tables - logs = self.getLogs(workflowId) - messages = self.getMessages(workflowId) - stats = self.getStats(workflowId) + logs = self.getLogs(conversationId) + messages = self.getMessages(conversationId) - # Validate workflow data against ChatWorkflow model - return ChatWorkflow( - id=workflow["id"], - status=workflow.get("status", "running"), - name=workflow.get("name"), - currentRound=workflow.get("currentRound", 0) or 0, - currentTask=workflow.get("currentTask", 0) or 0, - currentAction=workflow.get("currentAction", 0) or 0, - totalTasks=workflow.get("totalTasks", 0) or 0, - totalActions=workflow.get("totalActions", 0) or 0, - lastActivity=workflow.get("lastActivity", getUtcTimestamp()), - startedAt=workflow.get("startedAt", getUtcTimestamp()), + # Build ChatbotConversation with hydrated logs/messages + return ChatbotConversation( + id=conv["id"], + featureInstanceId=conv.get("featureInstanceId") or self.featureInstanceId or "", + name=conv.get("name"), + status=conv.get("status", "running"), + currentRound=conv.get("currentRound", 0) or 0, + lastActivity=conv.get("lastActivity", getUtcTimestamp()), + startedAt=conv.get("startedAt", getUtcTimestamp()), + workflowMode=ChatbotWorkflowModeEnum(conv.get("workflowMode", "Chatbot")), + maxSteps=conv.get("maxSteps") if conv.get("maxSteps") is not None else 10, logs=logs, - messages=messages, - stats=stats, - mandateId=workflow.get("mandateId", self.mandateId), - featureInstanceId=workflow.get("featureInstanceId") or self.featureInstanceId or "", - workflowMode=workflow.get("workflowMode", "chatbot"), - maxSteps=workflow.get("maxSteps") if workflow.get("maxSteps") is not None else 1 + messages=messages ) except Exception as e: - logger.error(f"Error validating workflow data: {str(e)}") + logger.error(f"Error validating conversation data: {str(e)}") return None - def createWorkflow(self, workflowData: Dict[str, Any]) -> ChatWorkflow: - """Creates a new workflow if user has permission.""" - if not self.checkRbacPermission(ChatWorkflow, "create"): - raise PermissionError("No permission to create workflows") + def getWorkflow(self, workflowId: str) -> Optional[ChatbotConversation]: + """Backward-compat alias: workflowId maps to conversationId.""" + return self.getConversation(workflowId) + + def createConversation(self, conversationData: Dict[str, Any]) -> ChatbotConversation: + """Creates a new conversation if user has permission.""" + if not self.checkRbacPermission(ChatbotConversation, "create"): + raise PermissionError("No permission to create conversations") # Set timestamp if not present currentTime = getUtcTimestamp() - if "startedAt" not in workflowData: - workflowData["startedAt"] = currentTime + if "startedAt" not in conversationData: + conversationData["startedAt"] = currentTime + if "lastActivity" not in conversationData: + conversationData["lastActivity"] = currentTime - if "lastActivity" not in workflowData: - workflowData["lastActivity"] = currentTime + # Set featureInstanceId from context (no mandateId in DB) + if "featureInstanceId" not in conversationData or not conversationData["featureInstanceId"]: + conversationData["featureInstanceId"] = self.featureInstanceId or "" + if not conversationData.get("featureInstanceId"): + conversationData["featureInstanceId"] = self.featureInstanceId or "" - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in workflowData or not workflowData["mandateId"]: - workflowData["mandateId"] = self.mandateId - if "featureInstanceId" not in workflowData or not workflowData["featureInstanceId"]: - workflowData["featureInstanceId"] = self.featureInstanceId - - # Ensure featureInstanceId is set (required field) - if not workflowData.get("featureInstanceId"): - workflowData["featureInstanceId"] = self.featureInstanceId or "" - - # Use generic field separation based on ChatWorkflow model - simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) + # Use generic field separation - logs/messages go to objectFields, not stored + simpleFields, objectFields = self._separateObjectFields(ChatbotConversation, conversationData) - # Create workflow in database - created = self.db.recordCreate(ChatWorkflow, simpleFields) + # Create conversation in database + created = self.db.recordCreate(ChatbotConversation, simpleFields) - - # Convert to ChatWorkflow model (empty related data for new workflow) - return ChatWorkflow( + return ChatbotConversation( id=created["id"], - status=created.get("status", "running"), + featureInstanceId=created.get("featureInstanceId") or self.featureInstanceId or "", name=created.get("name"), + status=created.get("status", "running"), currentRound=created.get("currentRound", 0) or 0, - currentTask=created.get("currentTask", 0) or 0, - currentAction=created.get("currentAction", 0) or 0, - totalTasks=created.get("totalTasks", 0) or 0, - totalActions=created.get("totalActions", 0) or 0, lastActivity=created.get("lastActivity", currentTime), startedAt=created.get("startedAt", currentTime), + workflowMode=ChatbotWorkflowModeEnum(created.get("workflowMode", "Chatbot")), + maxSteps=created.get("maxSteps", 10), logs=[], - messages=[], - stats=[], - mandateId=created.get("mandateId", self.mandateId), - featureInstanceId=created.get("featureInstanceId") or self.featureInstanceId or "", - workflowMode=created["workflowMode"], - maxSteps=created.get("maxSteps", 1) + messages=[] ) - def updateWorkflow(self, workflowId: str, workflowData: Dict[str, Any]) -> ChatWorkflow: - """Updates a workflow if user has access.""" - # Check if the workflow exists and user has access - workflow = self.getWorkflow(workflowId) - if not workflow: + def createWorkflow(self, workflowData: Dict[str, Any]) -> ChatbotConversation: + """Backward-compat alias: maps workflowData to conversationData.""" + return self.createConversation(workflowData) + + def updateConversation(self, conversationId: str, conversationData: Dict[str, Any]) -> ChatbotConversation: + """Updates a conversation if user has access.""" + conv = self.getConversation(conversationId) + if not conv: return None - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to update workflow {workflowId}") + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + raise PermissionError(f"No permission to update conversation {conversationId}") - # Use generic field separation based on ChatWorkflow model - simpleFields, objectFields = self._separateObjectFields(ChatWorkflow, workflowData) - - # Set update time for main workflow + simpleFields, objectFields = self._separateObjectFields(ChatbotConversation, conversationData) simpleFields["lastActivity"] = getUtcTimestamp() - # Update main workflow in database - updated = self.db.recordModify(ChatWorkflow, workflowId, simpleFields) - - # Removed cascade writes for logs/messages/stats during workflow update. - # CUD for child entities must be executed via dedicated service methods. + updated = self.db.recordModify(ChatbotConversation, conversationId, simpleFields) - # Load fresh data from normalized tables - logs = self.getLogs(workflowId) - messages = self.getMessages(workflowId) - stats = self.getStats(workflowId) + logs = self.getLogs(conversationId) + messages = self.getMessages(conversationId) - # Convert to ChatWorkflow model - return ChatWorkflow( + return ChatbotConversation( id=updated["id"], - status=updated.get("status", workflow.status), - name=updated.get("name", workflow.name), - currentRound=updated.get("currentRound", workflow.currentRound), - currentTask=updated.get("currentTask", workflow.currentTask), - currentAction=updated.get("currentAction", workflow.currentAction), - totalTasks=updated.get("totalTasks", workflow.totalTasks), - totalActions=updated.get("totalActions", workflow.totalActions), - lastActivity=updated.get("lastActivity", workflow.lastActivity), - startedAt=updated.get("startedAt", workflow.startedAt), + featureInstanceId=updated.get("featureInstanceId") or conv.featureInstanceId or self.featureInstanceId or "", + name=updated.get("name", conv.name), + status=updated.get("status", conv.status), + currentRound=updated.get("currentRound", conv.currentRound), + lastActivity=updated.get("lastActivity", conv.lastActivity), + startedAt=updated.get("startedAt", conv.startedAt), + workflowMode=ChatbotWorkflowModeEnum(updated.get("workflowMode", conv.workflowMode.value)), + maxSteps=updated.get("maxSteps") if updated.get("maxSteps") is not None else conv.maxSteps, logs=logs, - messages=messages, - stats=stats, - mandateId=updated.get("mandateId", workflow.mandateId), - featureInstanceId=updated.get("featureInstanceId") or workflow.featureInstanceId or self.featureInstanceId or "", - workflowMode=updated.get("workflowMode") if updated.get("workflowMode") is not None else (workflow.workflowMode if hasattr(workflow, 'workflowMode') and workflow.workflowMode else "chatbot"), - maxSteps=updated.get("maxSteps") if updated.get("maxSteps") is not None else (workflow.maxSteps if hasattr(workflow, 'maxSteps') and workflow.maxSteps is not None else 1) + messages=messages ) - def deleteWorkflow(self, workflowId: str) -> bool: - """Deletes a workflow and all related data if user has access.""" + def updateWorkflow(self, workflowId: str, workflowData: Dict[str, Any]) -> ChatbotConversation: + """Backward-compat alias.""" + return self.updateConversation(workflowId, workflowData) + + def deleteConversation(self, conversationId: str) -> bool: + """Deletes a conversation and all related data if user has access.""" try: - # Check if the workflow exists and user has access - workflow = self.getWorkflow(workflowId) - if not workflow: + conv = self.getConversation(conversationId) + if not conv: return False - if not self.checkRbacPermission(ChatWorkflow, "delete", workflowId): - raise PermissionError(f"No permission to delete workflow {workflowId}") + if not self.checkRbacPermission(ChatbotConversation, "delete", conversationId): + raise PermissionError(f"No permission to delete conversation {conversationId}") # CASCADE DELETE: Delete all related data first - # 1. Delete all workflow messages and their related data - messages = self.getMessages(workflowId) + # 1. Delete all messages and their documents + messages = self.getMessages(conversationId) for message in messages: messageId = message.id if messageId: - # Delete message stats - existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) - for stat in existing_stats: - self.db.recordDelete(ChatStat, stat["id"]) - - # Delete message documents (but NOT the files!) - existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + existing_docs = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") for doc in existing_docs: - self.db.recordDelete(ChatDocument, doc["id"]) - - # Delete the message itself - self.db.recordDelete(ChatMessage, messageId) + self.db.recordDelete(ChatbotDocument, doc["id"]) + self.db.recordDelete(ChatbotMessage, messageId) - # 2. Delete workflow stats - existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) - for stat in existing_stats: - self.db.recordDelete(ChatStat, stat["id"]) - - # 3. Delete workflow logs - existing_logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + # 2. Delete conversation logs + existing_logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, featureCode="chatbot") for log in existing_logs: - self.db.recordDelete(ChatLog, log["id"]) - - # 4. Finally delete the workflow itself - success = self.db.recordDelete(ChatWorkflow, workflowId) + self.db.recordDelete(ChatbotLog, log["id"]) + # 3. Delete the conversation + success = self.db.recordDelete(ChatbotConversation, conversationId) return success except Exception as e: - logger.error(f"Error deleting workflow {workflowId}: {str(e)}") + logger.error(f"Error deleting conversation {conversationId}: {str(e)}") return False + def deleteWorkflow(self, workflowId: str) -> bool: + """Backward-compat alias.""" + return self.deleteConversation(workflowId) + # Message methods - def getMessages(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatMessage], PaginatedResult]: + def getMessages(self, conversationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatbotMessage], PaginatedResult]: """ - Returns messages for a workflow if user has access to the workflow. + Returns messages for a conversation if user has access. Supports optional pagination, sorting, and filtering. Args: - workflowId: The workflow ID to get messages for + conversationId: The conversation ID (workflowId for backward compat) pagination: Optional pagination parameters. If None, returns all items. Returns: - If pagination is None: List[ChatMessage] + If pagination is None: List[ChatbotMessage] If pagination is provided: PaginatedResult with items and metadata """ - # Check workflow access first (without calling getWorkflow to avoid circular reference) - # Use RBAC filtering - workflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, + # Check conversation access first + conversations = getRecordsetWithRBAC(self.db, + ChatbotConversation, self.currentUser, - recordFilter={"id": workflowId} + recordFilter={"id": conversationId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode="chatbot" ) - if not workflows: + if not conversations: if pagination is None: return [] return PaginatedResult(items=[], totalItems=0, totalPages=0) - # Get messages for this workflow from normalized table - messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + # Get messages for this conversation + messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") # Convert raw messages to dict format for sorting/filtering messageDicts = [] for msg in messages: messageDicts.append({ "id": msg.get("id"), - "workflowId": msg.get("workflowId"), + "conversationId": msg.get("conversationId"), "parentMessageId": msg.get("parentMessageId"), "documentsLabel": msg.get("documentsLabel"), "message": msg.get("message"), @@ -892,9 +940,7 @@ class ChatObjects: "taskNumber": msg.get("taskNumber"), "actionNumber": msg.get("actionNumber"), "taskProgress": msg.get("taskProgress"), - "actionProgress": msg.get("actionProgress"), - "mandateId": msg.get("mandateId") or self.mandateId or "", - "featureInstanceId": msg.get("featureInstanceId") or self.featureInstanceId or "" + "actionProgress": msg.get("actionProgress") }) # Apply default sorting by publishedAt if no sort specified @@ -911,16 +957,12 @@ class ChatObjects: # If no pagination requested, return all items if pagination is None: - # Convert messages to ChatMessage objects and load documents chat_messages = [] for msg in messageDicts: - # Load documents from normalized documents table documents = self.getDocuments(msg["id"]) - - # Create ChatMessage object with loaded documents - chat_message = ChatMessage( + chat_message = ChatbotMessage( id=msg["id"], - workflowId=msg["workflowId"], + conversationId=msg["conversationId"], parentMessageId=msg.get("parentMessageId"), documents=documents, documentsLabel=msg.get("documentsLabel"), @@ -937,9 +979,7 @@ class ChatObjects: taskNumber=msg.get("taskNumber"), actionNumber=msg.get("actionNumber"), taskProgress=msg.get("taskProgress"), - actionProgress=msg.get("actionProgress"), - mandateId=msg.get("mandateId") or self.mandateId or "", - featureInstanceId=msg.get("featureInstanceId") or self.featureInstanceId or "" + actionProgress=msg.get("actionProgress") ) chat_messages.append(chat_message) @@ -955,16 +995,12 @@ class ChatObjects: endIdx = startIdx + pagination.pageSize pagedMessageDicts = messageDicts[startIdx:endIdx] - # Convert messages to ChatMessage objects and load documents chat_messages = [] for msg in pagedMessageDicts: - # Load documents from normalized documents table documents = self.getDocuments(msg["id"]) - - # Create ChatMessage object with loaded documents - chat_message = ChatMessage( + chat_message = ChatbotMessage( id=msg["id"], - workflowId=msg["workflowId"], + conversationId=msg["conversationId"], parentMessageId=msg.get("parentMessageId"), documents=documents, documentsLabel=msg.get("documentsLabel"), @@ -983,7 +1019,6 @@ class ChatObjects: taskProgress=msg.get("taskProgress"), actionProgress=msg.get("actionProgress") ) - chat_messages.append(chat_message) return PaginatedResult( @@ -992,27 +1027,27 @@ class ChatObjects: totalPages=totalPages ) - def createMessage(self, messageData: Dict[str, Any]) -> ChatMessage: - """Creates a message for a workflow if user has access.""" + def createMessage(self, messageData: Dict[str, Any]) -> ChatbotMessage: + """Creates a message for a conversation if user has access. Accepts workflowId (from bridge) or conversationId.""" try: - # Ensure ID is present if "id" not in messageData or not messageData["id"]: messageData["id"] = f"msg_{uuid.uuid4()}" - # Check required fields - requiredFields = ["id", "workflowId"] + # Map workflowId to conversationId (bridge compatibility) + if "workflowId" in messageData and "conversationId" not in messageData: + messageData["conversationId"] = messageData["workflowId"] + requiredFields = ["id", "conversationId"] for field in requiredFields: if field not in messageData: logger.error(f"Required field '{field}' missing in messageData") raise ValueError(f"Required field '{field}' missing in message data") - # Check workflow access - workflowId = messageData["workflowId"] - workflow = self.getWorkflow(workflowId) - if not workflow: - raise PermissionError(f"No access to workflow {workflowId}") + conversationId = messageData["conversationId"] + conv = self.getConversation(conversationId) + if not conv: + raise PermissionError(f"No access to conversation {conversationId}") - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to modify workflow {workflowId}") + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + raise PermissionError(f"No permission to modify conversation {conversationId}") # Validate that ID is not None if messageData["id"] is None: @@ -1030,47 +1065,33 @@ class ChatObjects: if "agentName" not in messageData: messageData["agentName"] = "" - # CRITICAL FIX: Automatically set roundNumber, taskNumber, and actionNumber if not provided - # This ensures messages have the correct progress context when workflows are continued + # Set roundNumber, taskNumber, actionNumber if not provided if "roundNumber" not in messageData: - messageData["roundNumber"] = workflow.currentRound - + messageData["roundNumber"] = conv.currentRound if "taskNumber" not in messageData: - messageData["taskNumber"] = workflow.currentTask - + messageData["taskNumber"] = 0 if "actionNumber" not in messageData: - messageData["actionNumber"] = workflow.currentAction + messageData["actionNumber"] = 0 - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in messageData or not messageData["mandateId"]: - messageData["mandateId"] = self.mandateId - if "featureInstanceId" not in messageData or not messageData["featureInstanceId"]: - messageData["featureInstanceId"] = self.featureInstanceId - - # Use generic field separation based on ChatMessage model - simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) + # Use generic field separation - no mandateId/featureInstanceId in message + simpleFields, objectFields = self._separateObjectFields(ChatbotMessage, messageData) # Handle documents separately - they will be stored in normalized documents table documents_to_create = objectFields.get("documents", []) - # Create message in normalized table using only simple fields - createdMessage = self.db.recordCreate(ChatMessage, simpleFields) + createdMessage = self.db.recordCreate(ChatbotMessage, simpleFields) - - # Create documents in normalized documents table created_documents = [] logger.debug(f"Creating {len(documents_to_create)} document(s) for message {createdMessage['id']}") for idx, doc_data in enumerate(documents_to_create): try: - # Normalize to plain dict before assignment - if isinstance(doc_data, ChatDocument): + if isinstance(doc_data, ChatbotDocument): doc_dict = doc_data.model_dump() elif isinstance(doc_data, dict): doc_dict = dict(doc_data) else: - # Attempt to coerce to ChatDocument then dump try: - doc_dict = ChatDocument(**doc_data).model_dump() + doc_dict = ChatbotDocument(**doc_data).model_dump() except Exception as e: logger.error(f"Invalid document data type for message creation (document {idx + 1}/{len(documents_to_create)}): {e}") continue @@ -1090,44 +1111,37 @@ class ChatObjects: logger.info(f"Created {len(created_documents)}/{len(documents_to_create)} document(s) for message {createdMessage['id']}") - # Convert to ChatMessage model - chat_message = ChatMessage( + chat_message = ChatbotMessage( id=createdMessage["id"], - workflowId=createdMessage["workflowId"], + conversationId=createdMessage["conversationId"], parentMessageId=createdMessage.get("parentMessageId"), - agentName=createdMessage.get("agentName"), documents=created_documents, documentsLabel=createdMessage.get("documentsLabel"), message=createdMessage.get("message"), role=createdMessage.get("role", "assistant"), status=createdMessage.get("status", "step"), - sequenceNr=len(workflow.messages) + 1, # Use messages list length for sequence number + sequenceNr=len(conv.messages) + 1, publishedAt=createdMessage.get("publishedAt", getUtcTimestamp()), - stats=objectFields.get("stats"), # Use stats from objectFields roundNumber=createdMessage.get("roundNumber"), taskNumber=createdMessage.get("taskNumber"), actionNumber=createdMessage.get("actionNumber"), success=createdMessage.get("success"), actionId=createdMessage.get("actionId"), actionMethod=createdMessage.get("actionMethod"), - actionName=createdMessage.get("actionName"), - mandateId=createdMessage.get("mandateId") or self.mandateId or "", - featureInstanceId=createdMessage.get("featureInstanceId") or self.featureInstanceId or "" + actionName=createdMessage.get("actionName") ) - # Emit message event for streaming (if event manager is available) try: - from modules.features.chatbot.eventManager import get_event_manager # type: ignore + from modules.features.chatbot.streaming.events import get_event_manager event_manager = get_event_manager() message_timestamp = parseTimestamp(chat_message.publishedAt, default=getUtcTimestamp()) - # Emit message event in exact chatData format: {type, createdAt, item} asyncio.create_task(event_manager.emit_event( - context_id=workflowId, + context_id=conversationId, event_type="chatdata", data={ "type": "message", "createdAt": message_timestamp, - "item": chat_message.dict() + "item": chat_message.model_dump() }, event_category="chat" )) @@ -1144,134 +1158,94 @@ class ChatObjects: logger.error(f"Error creating workflow message: {str(e)}") return None - def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatMessage]: - """Updates a workflow message if user has access to the workflow.""" + def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatbotMessage]: + """Updates a conversation message if user has access.""" try: - - # Ensure messageId is provided if not messageId: logger.error("No messageId provided for updateMessage") raise ValueError("messageId cannot be empty") - # Check if message exists in database - messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"id": messageId}) + if "workflowId" in messageData and "conversationId" not in messageData: + messageData["conversationId"] = messageData["workflowId"] + + messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"id": messageId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") if not messages: logger.warning(f"Message with ID {messageId} does not exist in database") - - # If message doesn't exist but we have workflowId, create it - if "workflowId" in messageData: - workflowId = messageData.get("workflowId") - - # Check workflow access - workflow = self.getWorkflow(workflowId) - if not workflow: - raise PermissionError(f"No access to workflow {workflowId}") - - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to modify workflow {workflowId}") - - logger.info(f"Creating new message with ID {messageId} for workflow {workflowId}") - return self.db.recordCreate(ChatMessage, messageData) - else: - logger.error(f"Workflow ID missing for new message {messageId}") - return None + if "conversationId" in messageData: + conv = self.getConversation(messageData["conversationId"]) + if not conv: + raise PermissionError(f"No access to conversation {messageData['conversationId']}") + if not self.checkRbacPermission(ChatbotConversation, "update", messageData["conversationId"]): + raise PermissionError(f"No permission to modify conversation") + logger.info(f"Creating new message with ID {messageId} for conversation {messageData['conversationId']}") + created = self.db.recordCreate(ChatbotMessage, messageData) + return ChatbotMessage(**created) if created else None + logger.error("Conversation ID missing for new message") + return None - # Update existing message existingMessage = messages[0] + conversationId = existingMessage.get("conversationId") + conv = self.getConversation(conversationId) + if not conv: + raise PermissionError(f"No access to conversation {conversationId}") + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + raise PermissionError(f"No permission to modify conversation {conversationId}") - # Check workflow access - workflowId = existingMessage.get("workflowId") - workflow = self.getWorkflow(workflowId) - if not workflow: - raise PermissionError(f"No access to workflow {workflowId}") - - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to modify workflow {workflowId}") + simpleFields, objectFields = self._separateObjectFields(ChatbotMessage, messageData) - # Use generic field separation based on ChatMessage model - simpleFields, objectFields = self._separateObjectFields(ChatMessage, messageData) - - # Ensure required fields present - for key in ["role", "agentName"]: - if key not in simpleFields and key not in existingMessage: - simpleFields[key] = "assistant" if key == "role" else "" - - # Ensure ID is in the dataset + if "role" not in simpleFields and "role" not in existingMessage: + simpleFields["role"] = "assistant" if 'id' not in simpleFields: simpleFields['id'] = messageId - # Convert createdAt to startedAt if needed if "createdAt" in simpleFields and "startedAt" not in simpleFields: simpleFields["startedAt"] = simpleFields["createdAt"] del simpleFields["createdAt"] - # Update the message with simple fields only - updatedMessage = self.db.recordModify(ChatMessage, messageId, simpleFields) + updatedMessage = self.db.recordModify(ChatbotMessage, messageId, simpleFields) - # Handle object field updates (documents, stats) inline if 'documents' in objectFields: - documents_data = objectFields['documents'] - try: - for doc_data in documents_data: - # Normalize to dict before mutation - if isinstance(doc_data, ChatDocument): + for doc_data in objectFields['documents']: + try: + if isinstance(doc_data, ChatbotDocument): doc_dict = doc_data.model_dump() elif isinstance(doc_data, dict): doc_dict = dict(doc_data) else: - try: - doc_dict = ChatDocument(**doc_data).model_dump() - except Exception: - logger.error("Invalid document data type for message update") - continue + doc_dict = ChatbotDocument(**doc_data).model_dump() doc_dict["messageId"] = messageId self.createDocument(doc_dict) - except Exception as e: - logger.error(f"Error updating message documents: {str(e)}") + except Exception as e: + logger.error(f"Error updating message documents: {e}") if not updatedMessage: logger.warning(f"Failed to update message {messageId}") return None - # Convert to ChatMessage model - return ChatMessage(**updatedMessage) + return ChatbotMessage(**updatedMessage) except Exception as e: logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) raise ValueError(f"Error updating message {messageId}: {str(e)}") - def deleteMessage(self, workflowId: str, messageId: str) -> bool: - """Deletes a workflow message and all related data if user has access to the workflow.""" + def deleteMessage(self, conversationId: str, messageId: str) -> bool: + """Deletes a conversation message and related data if user has access.""" try: - # Check workflow access - workflow = self.getWorkflow(workflowId) - if not workflow: - logger.warning(f"No access to workflow {workflowId}") + conv = self.getConversation(conversationId) + if not conv: + logger.warning(f"No access to conversation {conversationId}") return False - - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to modify workflow {workflowId}") - - # Check if the message exists - messages = self.getMessages(workflowId) - message = next((m for m in messages if m.get("id") == messageId), None) + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + raise PermissionError(f"No permission to modify conversation {conversationId}") + messages = self.getMessages(conversationId) + message = next((m for m in messages if m.id == messageId), None) if not message: - logger.warning(f"Message {messageId} for workflow {workflowId} not found") + logger.warning(f"Message {messageId} for conversation {conversationId} not found") return False - # CASCADE DELETE: Delete all related data first - - # 1. Delete message stats - existing_stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"messageId": messageId}) - for stat in existing_stats: - self.db.recordDelete(ChatStat, stat["id"]) - - # 2. Delete message documents (but NOT the files!) - existing_docs = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + existing_docs = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") for doc in existing_docs: - self.db.recordDelete(ChatDocument, doc["id"]) - - # 3. Finally delete the message itself - success = self.db.recordDelete(ChatMessage, messageId) + self.db.recordDelete(ChatbotDocument, doc["id"]) + success = self.db.recordDelete(ChatbotMessage, messageId) return success @@ -1279,21 +1253,17 @@ class ChatObjects: logger.error(f"Error deleting message {messageId}: {str(e)}") return False - def deleteFileFromMessage(self, workflowId: str, messageId: str, fileId: str) -> bool: + def deleteFileFromMessage(self, conversationId: str, messageId: str, fileId: str) -> bool: """Removes a file reference from a message if user has access.""" try: - # Check workflow access - workflow = self.getWorkflow(workflowId) - if not workflow: - logger.warning(f"No access to workflow {workflowId}") + conv = self.getConversation(conversationId) + if not conv: + logger.warning(f"No access to conversation {conversationId}") return False - - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - raise PermissionError(f"No permission to modify workflow {workflowId}") + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + raise PermissionError(f"No permission to modify conversation {conversationId}") - - # Get documents for this message from normalized table - documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) + documents = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") if not documents: logger.warning(f"No documents found for message {messageId}") @@ -1315,7 +1285,7 @@ class ChatObjects: if shouldRemove: # Delete the document from normalized table - success = self.db.recordDelete(ChatDocument, docId) + success = self.db.recordDelete(ChatbotDocument, docId) if success: removed = True else: @@ -1331,43 +1301,25 @@ class ChatObjects: # Document methods - def getDocuments(self, messageId: str) -> List[ChatDocument]: + def getDocuments(self, messageId: str) -> List[ChatbotDocument]: """Returns documents for a message from normalized table.""" try: - documents = getRecordsetWithRBAC(self.db, ChatDocument, self.currentUser, recordFilter={"messageId": messageId}) - # Ensure mandateId and featureInstanceId are set for each document - return [ChatDocument(**{**doc, "mandateId": doc.get("mandateId") or self.mandateId or "", "featureInstanceId": doc.get("featureInstanceId") or self.featureInstanceId or ""}) for doc in documents] + documents = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") + return [ChatbotDocument(**doc) for doc in documents] except Exception as e: logger.error(f"Error getting message documents: {str(e)}") return [] - def createDocument(self, documentData: Dict[str, Any]) -> ChatDocument: + def createDocument(self, documentData: Dict[str, Any]) -> ChatbotDocument: """Creates a document for a message in normalized table.""" try: - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in documentData or not documentData["mandateId"]: - documentData["mandateId"] = self.mandateId - if "featureInstanceId" not in documentData or not documentData["featureInstanceId"]: - documentData["featureInstanceId"] = self.featureInstanceId - - # Validate and normalize document data to dict - document = ChatDocument(**documentData) - logger.debug(f"Creating document in database: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") - created = self.db.recordCreate(ChatDocument, document.model_dump()) - + document = ChatbotDocument(**documentData) + logger.debug(f"Creating document: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") + created = self.db.recordCreate(ChatbotDocument, document.model_dump()) if created: - # Ensure mandateId and featureInstanceId are set - doc_dict = dict(created) - if "mandateId" not in doc_dict or not doc_dict["mandateId"]: - doc_dict["mandateId"] = self.mandateId or "" - if "featureInstanceId" not in doc_dict or not doc_dict["featureInstanceId"]: - doc_dict["featureInstanceId"] = self.featureInstanceId or "" - created_doc = ChatDocument(**doc_dict) - logger.debug(f"Successfully created document in database: {created_doc.fileName} (id: {created_doc.id})") - return created_doc - else: - logger.error(f"Failed to create document in database: recordCreate returned None for fileName={document.fileName}") - return None + return ChatbotDocument(**created) + logger.error(f"Failed to create document for fileName={document.fileName}") + return None except Exception as e: logger.error(f"Error creating message document: {str(e)}", exc_info=True) return None @@ -1375,50 +1327,36 @@ class ChatObjects: # Log methods - def getLogs(self, workflowId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatLog], PaginatedResult]: + def getLogs(self, conversationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatbotLog], PaginatedResult]: """ - Returns logs for a workflow if user has access to the workflow. + Returns logs for a conversation if user has access. Supports optional pagination, sorting, and filtering. - - Args: - workflowId: The workflow ID to get logs for - pagination: Optional pagination parameters. If None, returns all items. - - Returns: - If pagination is None: List[ChatLog] - If pagination is provided: PaginatedResult with items and metadata """ - # Check workflow access first (without calling getWorkflow to avoid circular reference) - # Use RBAC filtering - workflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, + conversations = getRecordsetWithRBAC(self.db, + ChatbotConversation, self.currentUser, - recordFilter={"id": workflowId} + recordFilter={"id": conversationId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode="chatbot" ) - - if not workflows: + if not conversations: if pagination is None: return [] return PaginatedResult(items=[], totalItems=0, totalPages=0) - # Get logs for this workflow from normalized table - logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - # Convert raw logs to dict format for sorting/filtering logDicts = [] for log in logs: logDicts.append({ "id": log.get("id"), - "workflowId": log.get("workflowId"), + "conversationId": log.get("conversationId"), "message": log.get("message"), "type": log.get("type"), "timestamp": log.get("timestamp", getUtcTimestamp()), - "agentName": log.get("agentName"), "status": log.get("status"), - "progress": log.get("progress"), - "mandateId": log.get("mandateId") or self.mandateId or "", - "featureInstanceId": log.get("featureInstanceId") or self.featureInstanceId or "", - "userId": log.get("userId") + "progress": log.get("progress") }) # Apply default sorting by timestamp if no sort specified @@ -1433,23 +1371,15 @@ class ChatObjects: if pagination and pagination.sort: logDicts = self._applySorting(logDicts, pagination.sort) - # If no pagination requested, return all items if pagination is None: - # Ensure mandateId and featureInstanceId are set for each log - return [ChatLog(**{**log, "mandateId": log.get("mandateId") or self.mandateId or "", "featureInstanceId": log.get("featureInstanceId") or self.featureInstanceId or ""}) for log in logDicts] + return [ChatbotLog(**log) for log in logDicts] - # Count total items after filters totalItems = len(logDicts) totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) startIdx = (pagination.page - 1) * pagination.pageSize endIdx = startIdx + pagination.pageSize pagedLogDicts = logDicts[startIdx:endIdx] - - # Convert to model objects - # Ensure mandateId and featureInstanceId are set for each log - items = [ChatLog(**{**log, "mandateId": log.get("mandateId") or self.mandateId or "", "featureInstanceId": log.get("featureInstanceId") or self.featureInstanceId or ""}) for log in pagedLogDicts] + items = [ChatbotLog(**log) for log in pagedLogDicts] return PaginatedResult( items=items, @@ -1457,185 +1387,81 @@ class ChatObjects: totalPages=totalPages ) - def createLog(self, logData: Dict[str, Any]) -> ChatLog: - """Creates a log entry for a workflow if user has access.""" - # Check workflow access - workflowId = logData.get("workflowId") - if not workflowId: - logger.error("No workflowId provided for createLog") + def createLog(self, logData: Dict[str, Any]) -> Optional[ChatbotLog]: + """Creates a log entry for a conversation if user has access. Accepts workflowId for backward compat.""" + conversationId = logData.get("conversationId") or logData.get("workflowId") + if not conversationId: + logger.error("No conversationId/workflowId provided for createLog") return None - workflow = self.getWorkflow(workflowId) - if not workflow: - logger.warning(f"No access to workflow {workflowId}") + conv = self.getConversation(conversationId) + if not conv: + logger.warning(f"No access to conversation {conversationId}") return None - - if not self.checkRbacPermission(ChatWorkflow, "update", workflowId): - logger.warning(f"No permission to modify workflow {workflowId}") + if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): + logger.warning(f"No permission to modify conversation {conversationId}") return None - # Make sure required fields are present if "timestamp" not in logData: logData["timestamp"] = getUtcTimestamp() - - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in logData or not logData["mandateId"]: - logData["mandateId"] = self.mandateId - if "featureInstanceId" not in logData or not logData["featureInstanceId"]: - logData["featureInstanceId"] = self.featureInstanceId - - # Add status information if not present + logData["conversationId"] = conversationId if "status" not in logData and "type" in logData: - if logData["type"] == "error": - logData["status"] = "error" - else: - logData["status"] = "running" - - # Add progress information if not present + logData["status"] = "error" if logData["type"] == "error" else "running" if "progress" not in logData: - # Default progress values based on log type (0.0 to 1.0 format) - if logData.get("type") == "info": - logData["progress"] = 0.5 # Default middle progress - elif logData.get("type") == "error": - logData["progress"] = 1.0 # Error state - completed (failed) - elif logData.get("type") == "warning": - logData["progress"] = 0.5 # Default middle progress + logData["progress"] = 1.0 if logData.get("type") == "error" else 0.5 - # Validate log data against ChatLog model try: - log_model = ChatLog(**logData) + log_model = ChatbotLog(**logData) except Exception as e: - logger.error(f"Invalid log data: {str(e)}") + logger.error(f"Invalid log data: {e}") return None - # Create log in normalized table - createdLog = self.db.recordCreate(ChatLog, log_model) + createdLog = self.db.recordCreate(ChatbotLog, log_model.model_dump()) + if not createdLog: + return None - # Emit log event for streaming (only for chatbot workflows) - # Only emit events for chatbot workflows, not for automation or dynamic workflows - if workflow.workflowMode == WorkflowModeEnum.WORKFLOW_CHATBOT: - try: - from modules.features.chatbot.eventManager import get_event_manager # type: ignore - event_manager = get_event_manager() - log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) - # Emit log event in exact chatData format: {type, createdAt, item} - asyncio.create_task(event_manager.emit_event( - workflowId, - "chatdata", - "New log", - "log", - { - "type": "log", - "createdAt": log_timestamp, - "item": ChatLog(**{**createdLog, "mandateId": createdLog.get("mandateId") or self.mandateId or "", "featureInstanceId": createdLog.get("featureInstanceId") or self.featureInstanceId or ""}).model_dump() - } - )) - except Exception as e: - # Event manager not available or error - continue without emitting - logger.debug(f"Could not emit log event: {e}") - - # Return validated ChatLog instance - # Ensure mandateId and featureInstanceId are set - log_dict = dict(createdLog) - if "mandateId" not in log_dict or not log_dict["mandateId"]: - log_dict["mandateId"] = self.mandateId or "" - if "featureInstanceId" not in log_dict or not log_dict["featureInstanceId"]: - log_dict["featureInstanceId"] = self.featureInstanceId or "" - return ChatLog(**log_dict) - - # Stats methods - - def getStats(self, workflowId: str) -> List[ChatStat]: - """Returns list of statistics for a workflow if user has access.""" - # Check workflow access first (without calling getWorkflow to avoid circular reference) - # Use RBAC filtering - workflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, - self.currentUser, - recordFilter={"id": workflowId} - ) - - if not workflows: - return [] - - # Get stats for this workflow from normalized table - stats = getRecordsetWithRBAC(self.db, ChatStat, self.currentUser, recordFilter={"workflowId": workflowId}) - - if not stats: - return [] - - # Return all stats records sorted by _createdAt (system field from DB) - stats.sort(key=lambda x: x.get("_createdAt", 0)) - # Ensure mandateId and featureInstanceId are set for each stat - return [ChatStat(**{**stat, "mandateId": stat.get("mandateId") or self.mandateId or "", "featureInstanceId": stat.get("featureInstanceId") or self.featureInstanceId or ""}) for stat in stats] - - - def createStat(self, statData: Dict[str, Any]) -> ChatStat: - """Creates a new stats record and returns it.""" try: - # Ensure workflowId is present in statData - if "workflowId" not in statData: - raise ValueError("workflowId is required in statData") - - # Set mandateId and featureInstanceId from context for proper data isolation - if "mandateId" not in statData or not statData["mandateId"]: - statData["mandateId"] = self.mandateId - if "featureInstanceId" not in statData or not statData["featureInstanceId"]: - statData["featureInstanceId"] = self.featureInstanceId - - # Validate the stat data against ChatStat model - stat = ChatStat(**statData) - - # Create the stat record in the database - created = self.db.recordCreate(ChatStat, stat) - - # Return the created ChatStat - # Ensure mandateId and featureInstanceId are set - stat_dict = dict(created) - if "mandateId" not in stat_dict or not stat_dict["mandateId"]: - stat_dict["mandateId"] = self.mandateId or "" - if "featureInstanceId" not in stat_dict or not stat_dict["featureInstanceId"]: - stat_dict["featureInstanceId"] = self.featureInstanceId or "" - return ChatStat(**stat_dict) + from modules.features.chatbot.streaming.events import get_event_manager + event_manager = get_event_manager() + log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) + asyncio.create_task(event_manager.emit_event( + context_id=conversationId, + event_type="chatdata", + data={"type": "log", "createdAt": log_timestamp, "item": ChatbotLog(**createdLog).model_dump()}, + event_category="log", + message="New log" + )) except Exception as e: - logger.error(f"Error creating workflow stat: {str(e)}") - raise - - - def getUnifiedChatData(self, workflowId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: + logger.debug(f"Could not emit log event: {e}") + + return ChatbotLog(**createdLog) + + def getUnifiedChatData(self, conversationId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: """ - Returns unified chat data (messages, logs, stats) for a workflow in chronological order. + Returns unified chat data (messages, logs) for a conversation in chronological order. Uses timestamp-based selective data transfer for efficient polling. """ - # Check workflow access first - # Use RBAC filtering - workflows = getRecordsetWithRBAC(self.db, - ChatWorkflow, + conversations = getRecordsetWithRBAC(self.db, + ChatbotConversation, self.currentUser, - recordFilter={"id": workflowId} + recordFilter={"id": conversationId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode="chatbot" ) - - if not workflows: + if not conversations: return {"items": []} - # Get all data types and filter in Python (PostgreSQL connector doesn't support $gt operators) items = [] - - # Get messages - messages = getRecordsetWithRBAC(self.db, ChatMessage, self.currentUser, recordFilter={"workflowId": workflowId}) + messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") for msg in messages: - # Apply timestamp filtering in Python msgTimestamp = parseTimestamp(msg.get("publishedAt"), default=getUtcTimestamp()) if afterTimestamp is not None and msgTimestamp <= afterTimestamp: continue - - # Load documents for each message documents = self.getDocuments(msg["id"]) - - # Create ChatMessage object with loaded documents - chatMessage = ChatMessage( + chatMessage = ChatbotMessage( id=msg["id"], - workflowId=msg["workflowId"], + conversationId=msg["conversationId"], parentMessageId=msg.get("parentMessageId"), documents=documents, documentsLabel=msg.get("documentsLabel"), @@ -1652,53 +1478,19 @@ class ChatObjects: taskNumber=msg.get("taskNumber"), actionNumber=msg.get("actionNumber"), taskProgress=msg.get("taskProgress"), - actionProgress=msg.get("actionProgress"), - mandateId=msg.get("mandateId") or self.mandateId or "", - featureInstanceId=msg.get("featureInstanceId") or self.featureInstanceId or "" + actionProgress=msg.get("actionProgress") ) - - # Use publishedAt as the timestamp for chronological ordering - items.append({ - "type": "message", - "createdAt": msgTimestamp, - "item": chatMessage - }) + items.append({"type": "message", "createdAt": msgTimestamp, "item": chatMessage}) - # Get logs - return all logs with roundNumber if available - logs = getRecordsetWithRBAC(self.db, ChatLog, self.currentUser, recordFilter={"workflowId": workflowId}) + logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") for log in logs: - # Apply timestamp filtering in Python logTimestamp = parseTimestamp(log.get("timestamp"), default=getUtcTimestamp()) if afterTimestamp is not None and logTimestamp <= afterTimestamp: continue - - # Ensure mandateId and featureInstanceId are set - log_dict = {**log, "mandateId": log.get("mandateId") or self.mandateId or "", "featureInstanceId": log.get("featureInstanceId") or self.featureInstanceId or ""} - chatLog = ChatLog(**log_dict) - items.append({ - "type": "log", - "createdAt": logTimestamp, - "item": chatLog - }) + chatLog = ChatbotLog(**log) + items.append({"type": "log", "createdAt": logTimestamp, "item": chatLog}) - # Get stats - ChatStat model now supports _createdAt via extra="allow" - stats = self.getStats(workflowId) - for stat in stats: - # Apply timestamp filtering in Python - # Use _createdAt (system field from DB, preserved via model_config extra="allow") - stat_timestamp = getattr(stat, '_createdAt', None) or getUtcTimestamp() - if afterTimestamp is not None and stat_timestamp <= afterTimestamp: - continue - - items.append({ - "type": "stat", - "createdAt": stat_timestamp, - "item": stat - }) - - # Sort all items by createdAt timestamp for chronological order items.sort(key=lambda x: parseTimestamp(x.get("createdAt"), default=0)) - return {"items": items} diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 7c5e880e..031056e9 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -18,15 +18,10 @@ FEATURE_ICON = "mdi-robot" # UI Objects for RBAC catalog UI_OBJECTS = [ { - "objectKey": "ui.feature.chatbot.chat", - "label": {"en": "Chat", "de": "Chat", "fr": "Chat"}, - "meta": {"area": "chat"} - }, - { - "objectKey": "ui.feature.chatbot.threads", - "label": {"en": "Threads", "de": "Threads", "fr": "Threads"}, - "meta": {"area": "threads"} - }, + "objectKey": "ui.feature.chatbot.conversations", + "label": {"en": "Conversations", "de": "Konversationen", "fr": "Conversations"}, + "meta": {"area": "conversations"} + } ] # Resource Objects for RBAC catalog @@ -81,7 +76,7 @@ TEMPLATE_ROLES = [ }, "accessRules": [ # UI: full access to all views - {"context": "UI", "item": "ui.feature.chatbot.chat", "view": True}, + {"context": "UI", "item": "ui.feature.chatbot.conversations", "view": True}, {"context": "UI", "item": "ui.feature.chatbot.threads", "view": True}, # Resource access: can start/stop chats, view threads, delete own {"context": "RESOURCE", "item": "resource.feature.chatbot.startStream", "view": True}, diff --git a/modules/features/chatbot/routeFeatureChatbot.py b/modules/features/chatbot/routeFeatureChatbot.py index ca0c8054..a88c5242 100644 --- a/modules/features/chatbot/routeFeatureChatbot.py +++ b/modules/features/chatbot/routeFeatureChatbot.py @@ -25,8 +25,9 @@ from modules.interfaces.interfaceDbApp import getRootInterface from modules.interfaces.interfaceFeatures import getFeatureInterface # Import models -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum +from modules.datamodels.datamodelChat import UserInputRequest from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata +from modules.features.chatbot.interfaceFeatureChatbot import ChatbotConversation # Import chatbot feature from modules.features.chatbot import chatProcess @@ -101,6 +102,119 @@ def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: return str(instance.mandateId) + +# ============================================================================= +# List threads - MUST be first to avoid /{instanceId}/{workflowId} matching +# GET /api/chatbot/{instanceId}/threads before DELETE /api/chatbot/{instanceId}/{workflowId} +# ============================================================================= +@router.get("/{instanceId}/threads") +@limiter.limit("120/minute") +def get_chatbot_threads( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + workflowId: Optional[str] = Query(None, description="Optional workflow ID to get details and chat data for a specific thread"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object (only used when workflowId is not provided)"), + context: RequestContext = Depends(getRequestContext) +) -> Union[PaginatedResponse[ChatbotConversation], Dict[str, Any]]: + """ + List all chatbot workflows (threads) for the current user, or get details and chat data for a specific thread. + + - If workflowId is provided: Returns the workflow details and all chat data (messages, logs, stats) + - If workflowId is not provided: Returns a paginated list of all workflows + """ + mandateId = _validateInstanceAccess(instanceId, context) + + try: + interfaceDbChat = _getServiceChat(context, instanceId) + + if workflowId: + workflow = interfaceDbChat.getWorkflow(workflowId) + if not workflow: + raise HTTPException( + status_code=404, + detail=f"Workflow with ID {workflowId} not found" + ) + + if hasattr(workflow, 'model_dump'): + workflow_dict = workflow.model_dump() + elif hasattr(workflow, 'dict'): + workflow_dict = workflow.dict() + elif isinstance(workflow, dict): + workflow_dict = dict(workflow) + else: + workflow_dict = workflow + + if workflow_dict.get("maxSteps") is None: + workflow_dict["maxSteps"] = 10 + + chatData = interfaceDbChat.getUnifiedChatData(workflowId, None) + + return { + "workflow": workflow_dict, + "chatData": chatData + } + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + paginationParams = PaginationParams(**paginationDict) if paginationDict else None + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException( + status_code=400, + detail=f"Invalid pagination parameter: {str(e)}" + ) + + all_workflows = interfaceDbChat.getWorkflows(pagination=None) + chatbot_workflows_data = [ + wf for wf in all_workflows + if (wf.get("workflowMode") or getattr(wf, "workflowMode", None)) == "Chatbot" + ] + + if paginationParams: + if paginationParams.sort: + chatbot_workflows_data = interfaceDbChat._applySorting(chatbot_workflows_data, paginationParams.sort) + totalItems = len(chatbot_workflows_data) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + workflows = chatbot_workflows_data[startIdx:endIdx] + else: + workflows = chatbot_workflows_data + totalItems = len(chatbot_workflows_data) + totalPages = 1 + + normalized_workflows = [] + for wf in workflows: + normalized_wf = dict(wf) + if normalized_wf.get("maxSteps") is None: + normalized_wf["maxSteps"] = 10 + normalized_workflows.append(normalized_wf) + + metadata = PaginationMetadata( + currentPage=paginationParams.page if paginationParams else 1, + pageSize=paginationParams.pageSize if paginationParams else len(workflows), + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort if paginationParams else [], + filters=paginationParams.filters if paginationParams else None + ) + + return PaginatedResponse( + items=normalized_workflows, + pagination=metadata + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting chatbot threads: {str(e)}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Error getting chatbot threads: {str(e)}" + ) + + # Chatbot streaming endpoint (SSE) @router.post("/{instanceId}/start/stream") @limiter.limit("120/minute") @@ -189,7 +303,7 @@ async def stream_chatbot_start( serializable_item = { "type": item.get("type"), "createdAt": item.get("createdAt"), - "item": item.get("item").dict() if hasattr(item.get("item"), "dict") else item.get("item") + "item": item.get("item").model_dump() if hasattr(item.get("item"), "model_dump") else (item.get("item").dict() if hasattr(item.get("item"), "dict") else item.get("item")) } # Emit item directly in exact chatData format: {type, createdAt, item} yield f"data: {json.dumps(serializable_item)}\n\n" @@ -257,7 +371,10 @@ async def stream_chatbot_start( # Ensure item field is serializable (convert Pydantic models to dicts) if isinstance(chatdata_item, dict) and "item" in chatdata_item: item_obj = chatdata_item.get("item") - if hasattr(item_obj, "dict"): + if hasattr(item_obj, "model_dump"): + chatdata_item = chatdata_item.copy() + chatdata_item["item"] = item_obj.model_dump() + elif hasattr(item_obj, "dict"): chatdata_item = chatdata_item.copy() chatdata_item["item"] = item_obj.dict() yield f"data: {json.dumps(chatdata_item)}\n\n" @@ -310,14 +427,14 @@ async def stream_chatbot_start( # Workflow stop endpoint -@router.post("/{instanceId}/stop/{workflowId}", response_model=ChatWorkflow) +@router.post("/{instanceId}/stop/{workflowId}", response_model=ChatbotConversation) @limiter.limit("120/minute") async def stop_chatbot( request: Request, instanceId: str = Path(..., description="Feature Instance ID"), workflowId: str = Path(..., description="ID of the workflow to stop"), context: RequestContext = Depends(getRequestContext) -) -> ChatWorkflow: +) -> ChatbotConversation: """Stops a running chatbot workflow.""" # Validate instance access _validateInstanceAccess(instanceId, context) @@ -384,138 +501,6 @@ async def stop_chatbot( detail=str(e) ) -# List chatbot threads/workflows or get specific thread details -# NOTE: This route MUST be defined BEFORE /{instanceId}/{workflowId} routes -# to prevent "threads" from being matched as a workflowId -@router.get("/{instanceId}/threads") -@limiter.limit("120/minute") -def get_chatbot_threads( - request: Request, - instanceId: str = Path(..., description="Feature Instance ID"), - workflowId: Optional[str] = Query(None, description="Optional workflow ID to get details and chat data for a specific thread"), - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object (only used when workflowId is not provided)"), - context: RequestContext = Depends(getRequestContext) -) -> Union[PaginatedResponse[ChatWorkflow], Dict[str, Any]]: - """ - List all chatbot workflows (threads) for the current user, or get details and chat data for a specific thread. - - - If workflowId is provided: Returns the workflow details and all chat data (messages, logs, stats) - - If workflowId is not provided: Returns a paginated list of all workflows - """ - # Validate instance access - mandateId = _validateInstanceAccess(instanceId, context) - - try: - interfaceDbChat = _getServiceChat(context, instanceId) - - # If workflowId is provided, return single workflow with chat data - if workflowId: - workflow = interfaceDbChat.getWorkflow(workflowId) - if not workflow: - raise HTTPException( - status_code=404, - detail=f"Workflow with ID {workflowId} not found" - ) - - # Normalize workflow data to match ChatWorkflow model requirements - # Convert workflow object to dict if needed, and normalize None values - if hasattr(workflow, 'model_dump'): - workflow_dict = workflow.model_dump() - elif hasattr(workflow, 'dict'): - workflow_dict = workflow.dict() - elif isinstance(workflow, dict): - workflow_dict = dict(workflow) - else: - workflow_dict = workflow - - # Set maxSteps to default value of 10 if None (as per ChatWorkflow model) - if workflow_dict.get("maxSteps") is None: - workflow_dict["maxSteps"] = 10 - - # Get unified chat data for this workflow - chatData = interfaceDbChat.getUnifiedChatData(workflowId, None) - - return { - "workflow": workflow_dict, - "chatData": chatData - } - - # Otherwise, return paginated list of workflows - # Parse pagination parameter - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - paginationParams = PaginationParams(**paginationDict) if paginationDict else None - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException( - status_code=400, - detail=f"Invalid pagination parameter: {str(e)}" - ) - - # Get all workflows filtered by mandateId (RBAC handles this automatically) - # We get all workflows first to filter by workflowMode before pagination - all_workflows = interfaceDbChat.getWorkflows(pagination=None) - - # Filter to only include chatbot workflows - chatbot_workflows_data = [ - wf for wf in all_workflows - if wf.get("workflowMode") == WorkflowModeEnum.WORKFLOW_CHATBOT.value - ] - - # Apply pagination if requested - if paginationParams: - # Apply sorting if provided - if paginationParams.sort: - chatbot_workflows_data = interfaceDbChat._applySorting(chatbot_workflows_data, paginationParams.sort) - - # Count total items after filtering - totalItems = len(chatbot_workflows_data) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - workflows = chatbot_workflows_data[startIdx:endIdx] - else: - workflows = chatbot_workflows_data - totalItems = len(chatbot_workflows_data) - totalPages = 1 - - # Normalize workflow data to match ChatWorkflow model requirements - # Convert None values to defaults before response validation - normalized_workflows = [] - for wf in workflows: - normalized_wf = dict(wf) # Create a copy - # Set maxSteps to default value of 10 if None (as per ChatWorkflow model) - if normalized_wf.get("maxSteps") is None: - normalized_wf["maxSteps"] = 10 - normalized_workflows.append(normalized_wf) - - # Create paginated response - metadata = PaginationMetadata( - currentPage=paginationParams.page if paginationParams else 1, - pageSize=paginationParams.pageSize if paginationParams else len(workflows), - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort if paginationParams else [], - filters=paginationParams.filters if paginationParams else None - ) - - return PaginatedResponse( - items=normalized_workflows, - pagination=metadata - ) - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting chatbot threads: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=f"Error getting chatbot threads: {str(e)}" - ) - # Delete chatbot workflow endpoint # NOTE: This catch-all route MUST be defined AFTER more specific routes like /threads @router.delete("/{instanceId}/{workflowId}", response_model=Dict[str, Any]) @@ -543,7 +528,7 @@ def delete_chatbot( ) # Check if workflow is a chatbot workflow - if workflow.workflowMode != WorkflowModeEnum.WORKFLOW_CHATBOT.value: + if (workflow.workflowMode or getattr(workflow, "workflowMode", None)) != "Chatbot": raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"Workflow {workflowId} is not a chatbot workflow" diff --git a/modules/features/chatbot/service.py b/modules/features/chatbot/service.py index 525aac7a..f3ab4e0b 100644 --- a/modules/features/chatbot/service.py +++ b/modules/features/chatbot/service.py @@ -13,7 +13,8 @@ import asyncio import re from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, ChatLog, ChatDocument +from modules.datamodels.datamodelChat import UserInputRequest +from modules.features.chatbot.interfaceFeatureChatbot import ChatbotConversation, ChatbotDocument from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference @@ -21,7 +22,7 @@ from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp from modules.services import getInterface as getServices from modules.features.chatbot.streaming.events import get_event_manager from modules.features.chatbot.chatbot import Chatbot -from modules.features.chatbot.bridges.ai import AICenterChatModel +from modules.features.chatbot.bridges.ai import AICenterChatModel, clear_workflow_allowed_providers from modules.features.chatbot.bridges.memory import DatabaseCheckpointer from modules.features.chatbot.config import ( load_chatbot_config_from_instance, @@ -30,7 +31,7 @@ from modules.features.chatbot.config import ( from modules.datamodels.datamodelAi import OperationTypeEnum, ProcessingModeEnum from modules.workflows.methods.methodAi.methodAi import MethodAi from modules.connectors.connectorPreprocessor import PreprocessorConnector -from modules.features.chatbot.chatbotConstants import generate_conversation_name +from modules.features.chatbot.chatbotConstants import generate_conversation_name, generate_name_from_prompt import base64 logger = logging.getLogger(__name__) @@ -69,7 +70,7 @@ async def chatProcess( userInput: UserInputRequest, workflowId: Optional[str] = None, featureInstanceId: Optional[str] = None -) -> ChatWorkflow: +) -> ChatbotConversation: """ Simple chatbot processing - analyze user input and generate queries. @@ -87,14 +88,21 @@ async def chatProcess( featureInstanceId: Feature instance ID for loading instance-specific config Returns: - ChatWorkflow instance + ChatbotConversation instance """ try: # Get services with mandate and feature instance context services = getServices(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) services.featureCode = 'chatbot' - interfaceDbChat = services.interfaceDbChat + # Load instance config and apply allowedProviders for AI calls (conversation name + main chat) + chatbot_config = await _load_chatbot_config(featureInstanceId) + if chatbot_config.model.allowedProviders: + services.allowedProviders = chatbot_config.model.allowedProviders + logger.info(f"Chatbot instance {featureInstanceId}: restricting to providers {chatbot_config.model.allowedProviders}") + + from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface + interfaceDbChat = getChatbotInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) # Get event manager and create queue if needed event_manager = get_event_manager() @@ -119,39 +127,53 @@ async def chatProcess( if not event_manager.has_queue(workflowId): event_manager.create_queue(workflowId) else: - # Generate conversation name based on user's prompt - conversation_name = await generate_conversation_name( - services, - userInput.prompt, - userInput.userLanguage - ) + # Use placeholder name immediately - don't block on AI call + prompt_stripped = (userInput.prompt or "").strip() + trivial_prompts = {"test", "hi", "hallo", "hello", "hey"} + is_short = len(prompt_stripped) < 30 or prompt_stripped.lower() in trivial_prompts - # Create new workflow - workflowData = { + if is_short: + conversation_name = generate_name_from_prompt(userInput.prompt) + run_name_task = False + else: + conversation_name = "Neue Unterhaltung" + run_name_task = True + + # Create new conversation (per feature instance) + conversationData = { "id": str(uuid.uuid4()), - "mandateId": mandateId, - "featureInstanceId": featureInstanceId, # Store feature instance for RBAC + "featureInstanceId": featureInstanceId, "status": "running", "name": conversation_name, "currentRound": 1, - "currentTask": 0, - "currentAction": 0, - "totalTasks": 0, - "totalActions": 0, - "workflowMode": WorkflowModeEnum.WORKFLOW_CHATBOT.value, + "workflowMode": "Chatbot", "startedAt": getUtcTimestamp(), "lastActivity": getUtcTimestamp() } - workflow = interfaceDbChat.createWorkflow(workflowData) + workflow = interfaceDbChat.createConversation(conversationData) logger.info(f"Created new chatbot workflow: {workflow.id} with name: {conversation_name}") + # Run AI name generation in background (for longer prompts only) + if run_name_task: + asyncio.create_task(_update_conversation_name_async( + services=services, + currentUser=currentUser, + mandateId=mandateId, + featureInstanceId=featureInstanceId, + workflowId=workflow.id, + prompt=userInput.prompt, + userLanguage=userInput.userLanguage, + interfaceDbChat=interfaceDbChat, + event_manager=event_manager + )) + # Create event queue for new workflow (for streaming) event_manager.create_queue(workflow.id) # Reload workflow to get current message count workflow = interfaceDbChat.getWorkflow(workflow.id) - # Process uploaded files and create ChatDocuments + # Process uploaded files and create ChatbotDocuments user_documents = [] if userInput.listFileId and len(userInput.listFileId) > 0: logger.info(f"Processing {len(userInput.listFileId)} uploaded file(s) for user message") @@ -167,8 +189,8 @@ async def chatProcess( originalMimeType = fileInfo.get("mimeType", "application/octet-stream") fileSizeToUse = fileInfo.get("size", 0) - # Create ChatDocument for the file - document = ChatDocument( + # Create ChatbotDocument for the file + document = ChatbotDocument( id=str(uuid.uuid4()), messageId="", # Will be set when message is created fileId=fileId, @@ -180,14 +202,14 @@ async def chatProcess( actionNumber=0 ) user_documents.append(document) - logger.info(f"Created ChatDocument for file {fileId} -> {originalFileName}") + logger.info(f"Created ChatbotDocument for file {fileId} -> {originalFileName}") except Exception as e: logger.error(f"Error processing file ID {fileId}: {e}", exc_info=True) # Store user message - userMessageData = { + userMessageData: Dict[str, Any] = { "id": f"msg_{uuid.uuid4()}", - "workflowId": workflow.id, + "conversationId": workflow.id, "message": userInput.prompt, "role": "user", "status": "first" if workflowId is None else "step", @@ -197,7 +219,8 @@ async def chatProcess( "taskNumber": 0, "actionNumber": 0 } - + if user_documents: + userMessageData["documents"] = [d.model_dump() for d in user_documents] userMessage = interfaceDbChat.createMessage(userMessageData) logger.info(f"Stored user message: {userMessage.id} with {len(user_documents)} document(s)") @@ -209,7 +232,7 @@ async def chatProcess( data={ "type": "message", "createdAt": message_timestamp, - "item": userMessage.dict() + "item": userMessage.model_dump() }, event_category="chat" ) @@ -219,7 +242,11 @@ async def chatProcess( "status": "running", "lastActivity": getUtcTimestamp() }) - + + # Pre-flight billing check before starting LangGraph (if mandateId present) + if mandateId: + _preflight_billing_check(services, mandateId, featureInstanceId) + # Process in background using LangGraph (async) asyncio.create_task(_processChatbotMessageLangGraph( services, @@ -227,7 +254,8 @@ async def chatProcess( workflow.id, userInput, userMessage.id, - featureInstanceId=featureInstanceId + featureInstanceId=featureInstanceId, + config=chatbot_config )) # Reload workflow to include new message @@ -239,6 +267,46 @@ async def chatProcess( raise +async def _update_conversation_name_async( + services, + currentUser: User, + mandateId: Optional[str], + featureInstanceId: Optional[str], + workflowId: str, + prompt: str, + userLanguage: str, + interfaceDbChat, + event_manager, +) -> None: + """ + Background task: generate conversation name via AI and update workflow. + Runs in parallel to LangGraph so it doesn't block the first response. + """ + try: + new_name = await generate_conversation_name(services, prompt, userLanguage) + if new_name: + interfaceDbChat.updateWorkflow(workflowId, {"name": new_name, "lastActivity": getUtcTimestamp()}) + logger.info(f"Updated workflow {workflowId} name to: {new_name}") + # Emit stat event so frontend can refresh thread list/title + workflow = interfaceDbChat.getWorkflow(workflowId) + if workflow: + wf_dict = workflow.model_dump() if hasattr(workflow, "model_dump") else workflow.dict() + await event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={ + "type": "stat", + "createdAt": getUtcTimestamp(), + "item": wf_dict + }, + event_category="chat", + message="Workflow name updated", + step="workflow_update" + ) + except Exception as e: + logger.warning(f"Background conversation name update failed for workflow {workflowId}: {e}") + + async def _execute_queries_parallel(queries: List[Dict[str, Any]]) -> Dict[str, Any]: """ Execute multiple SQL queries in parallel with shared connector. @@ -345,8 +413,7 @@ async def _emit_log_and_event( # Emit event directly for streaming (using correct signature) if created_log and event_manager: try: - from modules.datamodels.datamodelChat import ChatLog - # Convert to dict if it's a Pydantic model + # Convert to dict if it's a Pydantic model (ChatbotLog from createLog) if hasattr(created_log, "model_dump"): log_dict = created_log.model_dump() elif hasattr(created_log, "dict"): @@ -670,7 +737,7 @@ async def _convert_file_ids_to_document_references( """ references = [] - # Get workflow to search for ChatDocuments + # Get workflow to search for ChatbotDocuments workflow = services.workflow if not workflow: logger.error("Cannot convert file IDs to document references: workflow not set in services") @@ -684,7 +751,7 @@ async def _convert_file_ids_to_document_references( logger.warning(f"File {file_id} not found, skipping") continue - # Find ChatDocument that has this fileId + # Find ChatbotDocument that has this fileId document_id = None if workflow.messages: for message in workflow.messages: @@ -696,14 +763,16 @@ async def _convert_file_ids_to_document_references( if document_id: break - # Search database if not found in messages + # Search chatbot database if not found in messages if not document_id: try: from modules.interfaces.interfaceRbac import getRecordsetWithRBAC + from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface + chatbotInterface = getChatbotInterface(services.user, mandateId=services.mandateId, featureInstanceId=services.featureInstanceId) documents = getRecordsetWithRBAC( - services.interfaceDbChat.db, - ChatDocument, - services.currentUser, + chatbotInterface.db, + ChatbotDocument, + services.user, recordFilter={"fileId": file_id} ) if documents: @@ -715,7 +784,7 @@ async def _convert_file_ids_to_document_references( except Exception: pass # Fallback to fileId - # Use ChatDocument ID if found, otherwise use fileId as fallback + # Use ChatbotDocument ID if found, otherwise use fileId as fallback ref = DocumentItemReference(documentId=document_id if document_id else file_id) references.append(ref) except Exception as e: @@ -779,9 +848,9 @@ async def _create_chat_document_from_action_document( message_id: str, workflow_id: str, round_number: int -) -> ChatDocument: +) -> ChatbotDocument: """ - Create a ChatDocument from an ActionDocument by storing the file data. + Create a ChatbotDocument from an ActionDocument by storing the file data. Args: services: Services instance @@ -791,7 +860,7 @@ async def _create_chat_document_from_action_document( round_number: Round number Returns: - ChatDocument instance + ChatbotDocument instance """ try: # Get file data (could be bytes or string) @@ -838,8 +907,8 @@ async def _create_chat_document_from_action_document( if not success: logger.warning(f"Failed to store file data for {file_item.id}, but continuing...") - # Create ChatDocument - chat_document = ChatDocument( + # Create ChatbotDocument + chat_document = ChatbotDocument( id=str(uuid.uuid4()), messageId=message_id, fileId=file_item.id, @@ -851,11 +920,11 @@ async def _create_chat_document_from_action_document( actionNumber=0 ) - logger.info(f"Created ChatDocument {chat_document.id} from ActionDocument {file_name} (size: {len(file_bytes)} bytes)") + logger.info(f"Created ChatbotDocument {chat_document.id} from ActionDocument {file_name} (size: {len(file_bytes)} bytes)") return chat_document except Exception as e: - logger.error(f"Error creating ChatDocument from ActionDocument: {e}", exc_info=True) + logger.error(f"Error creating ChatbotDocument from ActionDocument: {e}", exc_info=True) raise @@ -977,6 +1046,7 @@ async def _bridge_chatbot_events( message="Chatbot-Verarbeitung abgeschlossen", step="complete" ) + clear_workflow_allowed_providers(workflow_id) # Update workflow status try: @@ -1001,6 +1071,7 @@ async def _bridge_chatbot_events( message=f"Fehler beim Verarbeiten: {error_msg}", step="error" ) + clear_workflow_allowed_providers(workflow_id) # Update workflow status try: @@ -1059,6 +1130,7 @@ async def _bridge_chatbot_events( message="Chatbot-Verarbeitung abgeschlossen", step="complete" ) + clear_workflow_allowed_providers(workflow_id) # Update workflow status try: @@ -1081,6 +1153,7 @@ async def _bridge_chatbot_events( message=f"Fehler beim Verarbeiten: {str(e)}", step="error" ) + clear_workflow_allowed_providers(workflow_id) async def _load_chatbot_config(featureInstanceId: Optional[str]) -> ChatbotConfig: @@ -1121,13 +1194,85 @@ async def _load_chatbot_config(featureInstanceId: Optional[str]) -> ChatbotConfi raise +def _preflight_billing_check(services, mandateId: str, featureInstanceId: Optional[str]) -> None: + """ + Pre-flight billing check before starting chatbot AI processing. + Raises if mandate has insufficient balance or no providers allowed. + """ + from modules.services.serviceBilling.mainServiceBilling import ( + getService as getBillingService, + InsufficientBalanceException, + ProviderNotAllowedException, + BillingContextError, + ) + user = services.user + featureCode = "chatbot" + try: + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + balanceCheck = billingService.checkBalance(0.01) + if not balanceCheck.allowed: + raise InsufficientBalanceException( + currentBalance=balanceCheck.currentBalance or 0.0, + requiredAmount=0.01, + message=f"Ungenuegendes Guthaben. Aktuell: CHF {balanceCheck.currentBalance:.2f}" + ) + rbacAllowedProviders = billingService.getallowedProviders() + if not rbacAllowedProviders: + raise ProviderNotAllowedException( + provider="any", + message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator." + ) + except (InsufficientBalanceException, ProviderNotAllowedException): + raise + except Exception as e: + logger.error(f"Billing pre-flight failed: {e}") + raise BillingContextError(f"Billing check failed: {e}") + + +def _create_chatbot_billing_callback(services, workflow_id: str): + """ + Create billing callback for AICenterChatModel. Records each AI call to poweron_billing. + """ + from modules.services.serviceBilling.mainServiceBilling import getService as getBillingService + from modules.datamodels.datamodelAi import AiCallResponse + + user = services.user + mandateId = services.mandateId + featureInstanceId = getattr(services, "featureInstanceId", None) + featureCode = "chatbot" + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + + def _billing_callback(response: AiCallResponse) -> None: + if not response or getattr(response, "errorCount", 0) > 0: + return + priceCHF = getattr(response, "priceCHF", 0.0) + if not priceCHF or priceCHF <= 0: + return + provider = getattr(response, "provider", None) or "unknown" + modelName = getattr(response, "modelName", None) or "unknown" + try: + billingService.recordUsage( + priceCHF=priceCHF, + workflowId=workflow_id, + aicoreProvider=provider, + aicoreModel=modelName, + description=f"AI: {modelName}" + ) + logger.debug(f"Chatbot billed: {priceCHF:.4f} CHF, provider={provider}, model={modelName}") + except Exception as e: + logger.error(f"Chatbot billing failed: {e}") + + return _billing_callback + + async def _processChatbotMessageLangGraph( services, currentUser: User, workflowId: str, userInput: UserInputRequest, userMessageId: str, - featureInstanceId: Optional[str] = None + featureInstanceId: Optional[str] = None, + config: Optional[ChatbotConfig] = None ): """ Process chatbot message using LangGraph. @@ -1144,7 +1289,8 @@ async def _processChatbotMessageLangGraph( event_manager = get_event_manager() try: - interfaceDbChat = services.interfaceDbChat + from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface + interfaceDbChat = getChatbotInterface(currentUser, mandateId=services.mandateId, featureInstanceId=featureInstanceId) # Reload workflow to get current messages workflow = interfaceDbChat.getWorkflow(workflowId) @@ -1165,8 +1311,19 @@ async def _processChatbotMessageLangGraph( logger.info(f"Workflow {workflowId} was stopped, aborting processing") return - # Load configuration from FeatureInstance (database) or fall back to file - config = await _load_chatbot_config(featureInstanceId) + # Emit synthetic status for real-time UI feedback + await event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={"type": "status", "label": "Lade Konfiguration..."}, + event_category="chat", + message="Status update", + step="status" + ) + + # Load configuration if not passed (e.g. when resuming) + if config is None: + config = await _load_chatbot_config(featureInstanceId) # Replace {{DATE}} placeholder in system prompt from datetime import datetime @@ -1179,14 +1336,39 @@ async def _processChatbotMessageLangGraph( operation_type = OperationTypeEnum[config.model.operationType] processing_mode = ProcessingModeEnum[config.model.processingMode] + billing_callback = None + if services.mandateId: + billing_callback = _create_chatbot_billing_callback(services, workflowId) + + allowed_providers = config.model.allowedProviders or None + if allowed_providers: + logger.info(f"Chatbot AICenterChatModel: restricting to providers {allowed_providers}") model = AICenterChatModel( user=currentUser, operation_type=operation_type, - processing_mode=processing_mode + processing_mode=processing_mode, + billing_callback=billing_callback, + workflow_id=workflowId, + allowed_providers=allowed_providers ) - # Create memory/checkpointer - memory = DatabaseCheckpointer(user=currentUser, workflow_id=workflowId) + # Emit synthetic status for real-time UI feedback + await event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={"type": "status", "label": "Bereite Chat vor..."}, + event_category="chat", + message="Status update", + step="status" + ) + + # Create memory/checkpointer (uses chatbot's own DB via interfaceFeatureChatbot) + memory = DatabaseCheckpointer( + user=currentUser, + workflow_id=workflowId, + mandateId=services.mandateId, + featureInstanceId=featureInstanceId + ) # Create chatbot instance with config for dynamic tool configuration chatbot = await Chatbot.create( @@ -1197,6 +1379,16 @@ async def _processChatbotMessageLangGraph( config=config ) + # Emit synthetic status for real-time UI feedback + await event_manager.emit_event( + context_id=workflowId, + event_type="chatdata", + data={"type": "status", "label": "Analysiere Anfrage..."}, + event_category="chat", + message="Status update", + step="status" + ) + # Stream events using chatbot event_stream = chatbot.stream_events( message=userInput.prompt, diff --git a/modules/features/chatbotV2/__init__.py b/modules/features/chatbotV2/__init__.py new file mode 100644 index 00000000..70c9f29f --- /dev/null +++ b/modules/features/chatbotV2/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chatbot V2 feature - context-aware chat with file upload and extraction.""" diff --git a/modules/features/chatbotV2/bridges/__init__.py b/modules/features/chatbotV2/bridges/__init__.py new file mode 100644 index 00000000..5d6343cb --- /dev/null +++ b/modules/features/chatbotV2/bridges/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chatbot V2 bridges - AI and memory.""" + +from modules.features.chatbot.bridges.ai import AICenterChatModel, clear_workflow_allowed_providers +from .memory import ChatbotV2Checkpointer + +__all__ = ["AICenterChatModel", "clear_workflow_allowed_providers", "ChatbotV2Checkpointer"] diff --git a/modules/features/chatbotV2/bridges/memory.py b/modules/features/chatbotV2/bridges/memory.py new file mode 100644 index 00000000..ce6c80ed --- /dev/null +++ b/modules/features/chatbotV2/bridges/memory.py @@ -0,0 +1,187 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chatbot V2 checkpointer - maps LangGraph state to ChatbotV2 message storage. +""" + +import logging +import uuid +from typing import Any, Dict, List, Optional, Tuple, NamedTuple + +from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage +from langgraph.checkpoint.base import BaseCheckpointSaver, Checkpoint, CheckpointMetadata + + +class CheckpointTuple(NamedTuple): + """Tuple containing config, checkpoint, metadata, parent_config, and pending_writes.""" + config: Dict[str, Any] + checkpoint: Checkpoint + metadata: CheckpointMetadata + parent_config: Optional[Dict[str, Any]] = None + pending_writes: Optional[List[Tuple[str, Any]]] = None + +from modules.features.chatbotV2.interfaceFeatureChatbotV2 import getInterface as getChatbotV2Interface +from modules.features.chatbotV2.datamodelFeatureChatbotV2 import ChatbotV2Message +from modules.datamodels.datamodelUam import User +from modules.shared.timeUtils import getUtcTimestamp + +logger = logging.getLogger(__name__) + + +def _sanitize_llm_response(text: str) -> str: + """Strip chat template tokens and trailing junk.""" + if not text or not isinstance(text, str): + return text or "" + for sentinel in ("<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|user|>", "<|assistant|>"): + if sentinel in text: + text = text.split(sentinel)[0] + return text.strip() + + +class ChatbotV2Checkpointer(BaseCheckpointSaver): + """Checkpointer that stores messages via ChatbotV2 interface.""" + + def __init__( + self, + user: User, + workflow_id: str, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ): + self.user = user + self.workflow_id = workflow_id + self.interface = getChatbotV2Interface( + user, + mandateId=mandateId, + featureInstanceId=featureInstanceId + ) + + def _to_db_message( + self, + msg: BaseMessage, + sequence_nr: int, + round_number: int + ) -> Dict[str, Any]: + role = "user" + content = "" + if isinstance(msg, HumanMessage): + role = "user" + content = msg.content if isinstance(msg.content, str) else str(msg.content or "") + elif isinstance(msg, AIMessage): + role = "assistant" + content = msg.content if isinstance(msg.content, str) else str(msg.content or "") + content = _sanitize_llm_response(content) + elif isinstance(msg, SystemMessage): + role = "system" + content = msg.content if isinstance(msg.content, str) else str(msg.content or "") + return { + "id": str(uuid.uuid4()), + "conversationId": self.workflow_id, + "message": content, + "role": role, + "status": "step" if sequence_nr > 1 else "first", + "sequenceNr": sequence_nr, + "publishedAt": getUtcTimestamp(), + "roundNumber": round_number + } + + def _to_langchain(self, messages: List[ChatbotV2Message]) -> List[BaseMessage]: + result = [] + for m in messages: + if m.role == "user": + result.append(HumanMessage(content=m.message or "")) + elif m.role == "assistant": + result.append(AIMessage(content=m.message or "")) + elif m.role == "system": + result.append(SystemMessage(content=m.message or "")) + return result + + def put( + self, + config: Dict[str, Any], + checkpoint: Checkpoint, + metadata: CheckpointMetadata, + new_versions: Dict[str, int], + ) -> None: + thread_id = config.get("configurable", {}).get("thread_id", self.workflow_id) + conv = self.interface.getConversation(thread_id) + if not conv: + logger.warning(f"Conversation {thread_id} not found") + return + round_number = conv.currentRound or 1 + state = checkpoint.get("channel_values", {}) + messages = state.get("messages", []) + if not messages: + return + existing = self.interface.getMessages(thread_id) + existing_set = {(m.role, m.message) for m in existing} + existing_count = len(existing) + for i, msg in enumerate(messages): + if not isinstance(msg, (HumanMessage, AIMessage)): + continue + role = "user" if isinstance(msg, HumanMessage) else "assistant" + content = msg.content if isinstance(msg.content, str) else str(msg.content or "") + if isinstance(msg, AIMessage): + content = _sanitize_llm_response(content) + if not content or not content.strip(): + continue + if (role, content) in existing_set: + continue + existing_set.add((role, content)) + existing_count += 1 + db_msg = self._to_db_message(msg, existing_count, round_number) + self.interface.createMessage(db_msg) + self.interface.updateConversation(thread_id, {"lastActivity": getUtcTimestamp()}) + + def get(self, config: Dict[str, Any]) -> Optional[Checkpoint]: + thread_id = config.get("configurable", {}).get("thread_id", self.workflow_id) + conv = self.interface.getConversation(thread_id) + if not conv: + return None + messages = self.interface.getMessages(thread_id) + lc_messages = self._to_langchain(messages) + return { + "id": str(uuid.uuid4()), + "v": 1, + "ts": getUtcTimestamp(), + "channel_values": {"messages": lc_messages}, + "channel_versions": {}, + "versions_seen": {} + } + + # Async methods required for LangGraph ainvoke/astream + async def aget_tuple( + self, + config: Dict[str, Any], + ) -> Optional[CheckpointTuple]: + """Async version of get that returns tuple of (config, checkpoint, metadata).""" + checkpoint = self.get(config) + if checkpoint: + metadata: CheckpointMetadata = {"step": 0} + return CheckpointTuple( + config=config, + checkpoint=checkpoint, + metadata=metadata, + parent_config=None, + pending_writes=None, + ) + return None + + async def aput( + self, + config: Dict[str, Any], + checkpoint: Checkpoint, + metadata: CheckpointMetadata, + new_versions: Dict[str, int], + ) -> None: + """Async version of put.""" + self.put(config, checkpoint, metadata, new_versions) + + async def aput_writes( + self, + config: Dict[str, Any], + writes: List[Tuple[str, Any]], + task_id: str, + ) -> None: + """Async version of put_writes. No-op - writes are handled through aput().""" + pass diff --git a/modules/features/chatbotV2/chatbotV2.py b/modules/features/chatbotV2/chatbotV2.py new file mode 100644 index 00000000..75ce6aa9 --- /dev/null +++ b/modules/features/chatbotV2/chatbotV2.py @@ -0,0 +1,210 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chatbot V2 domain logic - simple chat LangGraph with context injection. +Uses chunk-based retrieval with retry: when AI says "nicht enthalten", +tries the next chunk batch until content is found or all chunks searched. +""" + +import asyncio +import logging +import re +from typing import Annotated, Optional, TYPE_CHECKING, List, Dict, Any + +from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage +from langgraph.graph.message import add_messages +from langgraph.graph import StateGraph, START, END +from langgraph.graph.state import CompiledStateGraph +from pydantic import BaseModel + +from modules.features.chatbotV2.bridges import AICenterChatModel, ChatbotV2Checkpointer +from modules.features.chatbotV2.contextChunkRetrieval import ( + chunk_sections, + chunk_text_blocks, + get_ordered_chunks_for_question, + get_chunk_batch, + response_indicates_not_found, + DEFAULT_CHUNK_SIZE, + DEFAULT_CHUNK_OVERLAP, +) + +if TYPE_CHECKING: + from modules.features.chatbotV2.config import ChatbotV2Config + +logger = logging.getLogger(__name__) + + +class ChatState(BaseModel): + """State for Chatbot V2 chat session.""" + messages: Annotated[list[BaseMessage], add_messages] + # Optional context for chunk retrieval (passed at invoke, not persisted) + chatbotv2_context: Optional[Dict[str, Any]] = None + + +# Default max context chars (~20k tokens) - fits GPT 25k limit with room for prompt + response +DEFAULT_MAX_CONTEXT_CHARS = 60_000 + + +def _build_system_prompt_from_chunks( + base_prompt: str, + chunks: List[Dict[str, Any]] +) -> str: + """Build system prompt from a list of chunks.""" + if not chunks: + return base_prompt + header = "\n\n--- DOCUMENT CONTEXT (use this to answer user questions) ---\n" + parts = [base_prompt, header] + current_file = None + for chunk in chunks: + fn = chunk.get("fileName", "document") + if fn != current_file: + parts.append(f"\n### {fn}\n") + current_file = fn + parts.append(chunk.get("text", "")) + return "\n".join(parts) + + +def build_context_system_prompt( + base_prompt: str, + extracted_context: dict, + user_question: str, + max_context_chars: Optional[int] = None, + chunk_size: Optional[int] = None, + chunk_overlap: Optional[int] = None, +) -> str: + """ + Build system prompt with first chunk batch (for single-call use). + For retry loop, use get_ordered_chunks_for_question + get_chunk_batch + _build_system_prompt_from_chunks. + """ + chunks = _get_all_chunks(extracted_context, chunk_size, chunk_overlap) + if not chunks: + return base_prompt + ordered = get_ordered_chunks_for_question(chunks, user_question or "") + selected = get_chunk_batch(ordered, 0, max_context_chars or DEFAULT_MAX_CONTEXT_CHARS) + return _build_system_prompt_from_chunks(base_prompt, selected) + + +def _get_all_chunks( + extracted_context: dict, + chunk_size: Optional[int], + chunk_overlap: Optional[int], +) -> List[Dict[str, Any]]: + """Get all chunks from extracted context.""" + sections = extracted_context.get("sections", []) + text_blocks = extracted_context.get("textBlocks", []) + if not sections and not text_blocks: + return [] + cs = chunk_size if chunk_size and chunk_size > 0 else DEFAULT_CHUNK_SIZE + co = chunk_overlap if chunk_overlap is not None and chunk_overlap >= 0 else DEFAULT_CHUNK_OVERLAP + if sections: + return chunk_sections(sections, chunk_size=cs, chunk_overlap=co) + return chunk_text_blocks(text_blocks, chunk_size=cs, chunk_overlap=co) + + +def create_chat_graph( + model: AICenterChatModel, + memory: ChatbotV2Checkpointer, +) -> CompiledStateGraph: + """ + Create chat graph with retry loop: when AI says content not found, + tries next chunk batch until found or exhausted. + Context params passed via state.chatbotv2_context at invoke time. + """ + + async def chat_node(state: ChatState) -> dict: + # State can be dict (LangGraph) or Pydantic model + state_dict = state if isinstance(state, dict) else (state.model_dump() if hasattr(state, "model_dump") else {}) + msgs = state_dict.get("messages", []) + if not msgs: + return {} + + ctx = state_dict.get("chatbotv2_context") or {} + ctx_dict = ctx.get("ctx_dict", {}) + user_question = ctx.get("user_question", "") + base_prompt = ctx.get("base_prompt", "Answer based on the provided context.") + max_chars = ctx.get("max_context_chars") or DEFAULT_MAX_CONTEXT_CHARS + chunk_size = ctx.get("chunk_size") or DEFAULT_CHUNK_SIZE + chunk_overlap = ctx.get("chunk_overlap") + + if max_chars <= 0: + max_chars = DEFAULT_MAX_CONTEXT_CHARS + if chunk_overlap is None or chunk_overlap < 0: + chunk_overlap = DEFAULT_CHUNK_OVERLAP + + user_msgs = [m for m in msgs if not isinstance(m, SystemMessage)] + if not user_msgs: + return {} + + # Get chunks - use DOCUMENT ORDER for retry (batch 0 = start, batch 1 = next part, etc.) + chunks = _get_all_chunks(ctx_dict, chunk_size, chunk_overlap) + if not chunks: + logger.warning("No chunks from ctx_dict - sections=%s, textBlocks=%s", + len(ctx_dict.get("sections", [])), len(ctx_dict.get("textBlocks", []))) + # Always use document order (chunkIndex) for systematic search through entire document + ordered = sorted(chunks, key=lambda c: c.get("chunkIndex", 0)) + batch_index = 0 + last_response = None + + logger.info("Chunk retrieval: %d chunks total, max_chars=%d, will try batches until found or exhausted", + len(ordered), max_chars) + + while True: + batch = get_chunk_batch(ordered, batch_index, max_chars) if ordered else [] + if not batch: + # No more chunks - return last response or final message + if last_response: + return {"messages": [last_response]} + return {"messages": [AIMessage( + content="Ich habe das gesamte Dokument durchsucht, konnte aber keine " + "passende Information zu Ihrer Frage finden. Bitte formulieren Sie die Frage " + "ggf. anders oder prüfen Sie, ob das Dokument die gewünschten Angaben enthält." + )]} + + system_prompt = _build_system_prompt_from_chunks(base_prompt, batch) + window = [SystemMessage(content=system_prompt)] + user_msgs + + response = None + for attempt in range(3): # Max 3 attempts (initial + 2 retries on rate limit) + try: + response = await model.ainvoke(window) + break + except Exception as exc: + err_str = str(exc).lower() + if ("429" in err_str or "rate limit" in err_str) and attempt < 2: + wait_secs = 6 + match = re.search(r"try again in ([\d.]+)s", err_str, re.IGNORECASE) + if match: + wait_secs = max(6, int(float(match.group(1))) + 1) + logger.warning("Rate limit hit on chunk batch %d, waiting %ds before retry (attempt %d/3)", + batch_index, wait_secs, attempt + 1) + await asyncio.sleep(wait_secs) + else: + if "No suitable model found" in str(exc): + return {"messages": [AIMessage( + content="Es tut mir leid, derzeit steht kein passendes KI-Modell zur Verfügung. " + "Bitte versuchen Sie es später erneut." + )]} + raise + + if response is None: + return {"messages": [AIMessage( + content="Ein Fehler ist aufgetreten. Bitte versuchen Sie es später erneut." + )]} + + content = response.content if hasattr(response, "content") else str(response) + if response_indicates_not_found(content): + logger.info("Chunk batch %d: AI said not found (%.0f chars), trying next batch", + batch_index, len(content)) + batch_index += 1 + last_response = response + await asyncio.sleep(5) # Pause before next batch to avoid rate limits + continue + + logger.info("Chunk batch %d: Found answer (%.0f chars)", batch_index, len(content)) + return {"messages": [response]} + + workflow = StateGraph(ChatState) + workflow.add_node("chat", chat_node) + workflow.add_edge(START, "chat") + workflow.add_edge("chat", END) + return workflow.compile(checkpointer=memory) diff --git a/modules/features/chatbotV2/config.py b/modules/features/chatbotV2/config.py new file mode 100644 index 00000000..e63ba0ae --- /dev/null +++ b/modules/features/chatbotV2/config.py @@ -0,0 +1,98 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Configuration system for Chatbot V2 instances. +Loads configuration from FeatureInstance.config JSONB field. +""" + +import logging +from dataclasses import dataclass +from typing import Optional, Dict, Any, List, TYPE_CHECKING + +if TYPE_CHECKING: + from modules.datamodels.datamodelFeatures import FeatureInstance + +logger = logging.getLogger(__name__) + +_config_cache: Dict[str, 'ChatbotV2Config'] = {} + +DEFAULT_SYSTEM_PROMPT = ( + "You are a helpful assistant. Answer questions based on the provided context documents. " + "When the user asks about the documents, use the extracted content to provide accurate answers. " + "If the context does not contain relevant information, say so." +) + + +@dataclass +class ModelConfig: + """Model configuration for Chatbot V2.""" + operationType: str = "DATA_ANALYSE" + processingMode: str = "BASIC" + allowedProviders: List[str] = None + + def __post_init__(self): + if self.allowedProviders is None: + self.allowedProviders = [] + + +def _parse_int(val: Optional[Any], default: Optional[int] = None) -> Optional[int]: + """Parse int from config value.""" + if val is None: + return default + if isinstance(val, int): + return val + try: + return int(val) + except (TypeError, ValueError): + return default + + +@dataclass +class ChatbotV2Config: + """Configuration for a Chatbot V2 instance.""" + + id: str + name: str + systemPrompt: str + model: ModelConfig + maxContextChars: Optional[int] = None # Max document chars in system prompt (~60k ≈ 20k tokens). None = default 60k. + chunkSize: Optional[int] = None # Chunk size in chars (~15k). None = default. + chunkOverlap: Optional[int] = None # Overlap between chunks in chars (~500). None = default. + + @classmethod + def from_dict(cls, data: Dict[str, Any], config_id: str = "default") -> 'ChatbotV2Config': + """Create ChatbotV2Config from dictionary.""" + system_prompt = data.get("systemPrompt") or DEFAULT_SYSTEM_PROMPT + + model_data = data.get("model", {}) + allowed_providers = model_data.get("allowedProviders") or data.get("allowedProviders", []) + model_config = ModelConfig( + operationType=model_data.get("operationType", "DATA_ANALYSE"), + processingMode=model_data.get("processingMode", "BASIC"), + allowedProviders=allowed_providers if isinstance(allowed_providers, list) else [] + ) + + return cls( + id=data.get("id", config_id), + name=data.get("name", "Chatbot V2"), + systemPrompt=system_prompt, + model=model_config, + maxContextChars=_parse_int(data.get("maxContextChars")), + chunkSize=_parse_int(data.get("chunkSize")), + chunkOverlap=_parse_int(data.get("chunkOverlap")) + ) + + +def load_chatbotv2_config_from_instance(instance: 'FeatureInstance') -> ChatbotV2Config: + """Load Chatbot V2 configuration from a FeatureInstance's config field.""" + instance_id = instance.id + + cache_key = f"instance_{instance_id}" + if cache_key in _config_cache: + return _config_cache[cache_key] + + config_data = instance.config or {} + config = ChatbotV2Config.from_dict(config_data, config_id=instance_id) + _config_cache[cache_key] = config + logger.info(f"Loaded chatbotv2 config from instance {instance_id}") + return config diff --git a/modules/features/chatbotV2/contextChunkRetrieval.py b/modules/features/chatbotV2/contextChunkRetrieval.py new file mode 100644 index 00000000..7e77c0f7 --- /dev/null +++ b/modules/features/chatbotV2/contextChunkRetrieval.py @@ -0,0 +1,272 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chunk-based context retrieval for Chatbot V2. +Splits documents into chunks and selects relevant chunks per user question. +If no relevant chunk is found, falls back to next chunks in document order - no context is lost. +""" + +import re +import logging +from typing import List, Dict, Any, Optional + +logger = logging.getLogger(__name__) + +# Default chunk size (~5k tokens each), overlap for context continuity +DEFAULT_CHUNK_SIZE = 15_000 +DEFAULT_CHUNK_OVERLAP = 500 + +# Stopwords for relevance scoring (DE/EN/FR - minimal set) +STOPWORDS = { + "der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines", + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "le", "la", "les", "un", "une", "des", "du", "de", "et", "ou", + "was", "wer", "wie", "wo", "wann", "warum", "welche", "welcher", +} + + +def _extract_keywords(text: str) -> set: + """Extract significant words for relevance scoring.""" + text_lower = text.lower() + words = re.findall(r"\b[a-zàâäéèêëïîôùûüÿæœçß]{2,}\b", text_lower) + return {w for w in words if w not in STOPWORDS and len(w) > 1} + + +def chunk_sections( + sections: List[Dict[str, Any]], + chunk_size: int = DEFAULT_CHUNK_SIZE, + chunk_overlap: int = DEFAULT_CHUNK_OVERLAP +) -> List[Dict[str, Any]]: + """ + Split sections into overlapping chunks. + Each chunk has: chunkIndex, text, fileName, fileId, sectionIndex. + """ + chunks = [] + chunk_index = 0 + + for sec_idx, section in enumerate(sections): + text = section.get("text", "") + file_name = section.get("fileName", "document") + file_id = section.get("fileId", "") + + if not text: + continue + + start = 0 + while start < len(text): + end = min(start + chunk_size, len(text)) + chunk_text = text[start:end] + chunks.append({ + "chunkIndex": chunk_index, + "text": chunk_text, + "fileName": file_name, + "fileId": file_id, + "sectionIndex": sec_idx, + "startChar": start, + "endChar": end, + }) + chunk_index += 1 + start = end - chunk_overlap if end < len(text) else len(text) + + return chunks + + +def chunk_text_blocks( + text_blocks: List[Dict[str, Any]], + chunk_size: int = DEFAULT_CHUNK_SIZE, + chunk_overlap: int = DEFAULT_CHUNK_OVERLAP +) -> List[Dict[str, Any]]: + """Split textBlocks (blocks per file) into chunks.""" + chunks = [] + chunk_index = 0 + + for doc in text_blocks: + blocks = doc.get("blocks", []) + file_name = doc.get("fileName", "document") + file_id = doc.get("fileId", "") + + text_parts = [] + for b in blocks: + text_parts.append(b.get("text", "")) + + full_text = "\n".join(text_parts) + if not full_text: + continue + + start = 0 + while start < len(full_text): + end = min(start + chunk_size, len(full_text)) + chunk_text = full_text[start:end] + chunks.append({ + "chunkIndex": chunk_index, + "text": chunk_text, + "fileName": file_name, + "fileId": file_id, + "startChar": start, + "endChar": end, + }) + chunk_index += 1 + start = end - chunk_overlap if end < len(full_text) else len(full_text) + + return chunks + + +def score_chunk_relevance(chunk_text: str, question: str) -> float: + """ + Score how relevant a chunk is to the user question. + Uses keyword overlap with simple IDF-like weighting. + Returns 0 if no overlap. + """ + if not question or not chunk_text: + return 0.0 + + q_words = _extract_keywords(question) + if not q_words: + return 0.0 + + chunk_lower = chunk_text.lower() + score = 0.0 + for w in q_words: + count = chunk_lower.count(w) + if count > 0: + score += 1.0 + 0.5 * min(count - 1, 3) + + return score + + +def get_ordered_chunks_for_question( + chunks: List[Dict[str, Any]], + question: str +) -> List[Dict[str, Any]]: + """ + Return chunks ordered by relevance (or doc order if no match). + Does NOT limit by max_context_chars - used for iterative batch retrieval. + """ + if not chunks: + return [] + + scored = [] + for c in chunks: + score = score_chunk_relevance(c.get("text", ""), question) + scored.append((score, c)) + + max_score = max(s for s, _ in scored) + if max_score > 0: + scored.sort(key=lambda x: (-x[0], x[1].get("chunkIndex", 0))) + else: + scored.sort(key=lambda x: x[1].get("chunkIndex", 0)) + logger.debug("No chunk matched question keywords - using chunks in document order") + + return [c for _, c in scored] + + +def get_chunk_batch( + ordered_chunks: List[Dict[str, Any]], + batch_index: int, + max_context_chars: int +) -> List[Dict[str, Any]]: + """ + Get the Nth batch of chunks that fits in max_context_chars. + Batch 0 = first chunk(s) that fit, batch 1 = next chunk(s), etc. + Returns [] when batch_index is beyond available chunks. + """ + if not ordered_chunks or batch_index < 0: + return [] + + used_chars = 0 + chunks_in_batch = 0 + start_idx = 0 + + # Find start index for this batch (skip chunks from previous batches) + for _ in range(batch_index): + batch_chars = 0 + i = start_idx + while i < len(ordered_chunks): + text = ordered_chunks[i].get("text", "") + if batch_chars + len(text) <= max_context_chars: + batch_chars += len(text) + i += 1 + else: + if batch_chars < max_context_chars and len(text) > 0: + batch_chars += min(len(text), max_context_chars - batch_chars) + i += 1 + break + start_idx = i + if start_idx >= len(ordered_chunks): + return [] + + # Collect chunks for this batch + selected = [] + for i in range(start_idx, len(ordered_chunks)): + chunk = ordered_chunks[i] + text = chunk.get("text", "") + if not text: + continue + if used_chars + len(text) <= max_context_chars: + selected.append(chunk) + used_chars += len(text) + else: + if used_chars < max_context_chars: + remaining = max_context_chars - used_chars + selected.append({**chunk, "text": text[:remaining]}) + break + + return selected + + +def select_chunks_for_question( + chunks: List[Dict[str, Any]], + question: str, + max_context_chars: int +) -> List[Dict[str, Any]]: + """ + Select first batch of chunks (for backward compatibility). + For iterative retry, use get_ordered_chunks_for_question + get_chunk_batch. + """ + ordered = get_ordered_chunks_for_question(chunks, question) + return get_chunk_batch(ordered, 0, max_context_chars) + + +# Phrases that indicate the AI found no relevant content (DE/EN/FR) +NOT_FOUND_PHRASES = [ + "nicht enthalten", + "nicht im kontext", + "nicht im dokument", + "nicht im bereitgestellten", + "nicht auffindbar", + "keine information", + "keine angaben", + "nicht gefunden", + "nicht verfügbar", + "kein hinweis", + "enthalten nicht", + "artikel nicht enthalten", + "nicht im vorliegenden", + "bereitgestellten kontext enthält nicht", + "im kontext nicht", + "not contained", + "not found", + "no information", + "not in the context", + "not in the document", + "pas dans le contexte", + "non trouvé", +] + + +def response_indicates_not_found(response_content: str) -> bool: + """ + Check if the AI response indicates that the requested content was not found. + When True, we should try the next chunk batch. + """ + if not response_content or not isinstance(response_content, str): + return False + text_lower = response_content.lower().strip() + # Only treat short/medium responses as "not found" - long answers may mention it incidentally + if len(text_lower) > 600: + return False + for phrase in NOT_FOUND_PHRASES: + if phrase in text_lower: + return True + return False diff --git a/modules/features/chatbotV2/contextExtractionLangGraph.py b/modules/features/chatbotV2/contextExtractionLangGraph.py new file mode 100644 index 00000000..c34bf487 --- /dev/null +++ b/modules/features/chatbotV2/contextExtractionLangGraph.py @@ -0,0 +1,160 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +LangGraph-based pipeline for extracting context from uploaded documents. +Creates chat context from PDF and text files (no domain-specific goals). +""" + +import logging +from typing import TypedDict, List, Dict, Any, Optional +from langgraph.graph import StateGraph, START, END + +logger = logging.getLogger(__name__) + + +class ContextExtractionState(TypedDict): + """State for context extraction pipeline.""" + # Input: list of {fileId, bytes, mimeType, fileName} + files: List[Dict[str, Any]] + # Extracted text blocks per file: [{fileId, fileName, blocks: [{page, text, block_id}]}] + textBlocks: List[Dict[str, Any]] + # Structured sections for chat context (simplified articles/sections) + sections: List[Dict[str, Any]] + # Optional summaries (empty for now - no LLM in extraction) + summaries: List[Dict[str, Any]] + errors: List[str] + + +def extract_text_node(state: ContextExtractionState) -> ContextExtractionState: + """Extract text from each file. PDF via BZOPdfExtractor, TXT as plain text.""" + text_blocks = [] + errors = list(state.get("errors", [])) + + for idx, file_info in enumerate(state.get("files", [])): + file_id = file_info.get("fileId", f"file_{idx}") + file_bytes = file_info.get("bytes") + mime_type = (file_info.get("mimeType") or "").lower() + file_name = file_info.get("fileName", f"document_{idx}") + + if not file_bytes: + errors.append(f"No content for file {file_name} ({file_id})") + continue + + blocks = [] + try: + if "pdf" in mime_type or file_name.lower().endswith(".pdf"): + from modules.features.realEstate.bzoPdfExtractor import BZOPdfExtractor + extractor = BZOPdfExtractor() + tb_list = extractor.extract_text_blocks(file_bytes, file_id) + for tb in tb_list: + blocks.append({ + "page": tb.page, + "text": tb.text, + "block_id": tb.block_id, + "bbox": tb.bbox + }) + logger.info(f"Extracted {len(blocks)} blocks from PDF {file_name}") + elif "text" in mime_type or file_name.lower().endswith(".txt"): + text = file_bytes.decode("utf-8", errors="replace") + lines = text.split("\n") + for i, line in enumerate(lines): + if line.strip(): + blocks.append({ + "page": 1, + "text": line.strip(), + "block_id": f"{file_id}_line_{i}", + "bbox": None + }) + logger.info(f"Extracted {len(blocks)} lines from text file {file_name}") + else: + errors.append(f"Unsupported format for {file_name}: {mime_type}") + except Exception as e: + logger.error(f"Error extracting {file_name}: {e}", exc_info=True) + errors.append(f"Extraction failed for {file_name}: {str(e)}") + + if blocks: + text_blocks.append({ + "fileId": file_id, + "fileName": file_name, + "blocks": blocks + }) + + return { + **state, + "textBlocks": text_blocks, + "errors": errors + } + + +def structure_content_node(state: ContextExtractionState) -> ContextExtractionState: + """Assemble text blocks into sections for chat context.""" + sections = [] + for doc in state.get("textBlocks", []): + file_name = doc.get("fileName", "document") + blocks = doc.get("blocks", []) + if not blocks: + continue + # Build section: combine blocks with page awareness + text_parts = [] + current_page = 0 + for b in blocks: + page = b.get("page", 1) + if page != current_page and text_parts: + text_parts.append("\n\n") + text_parts.append(b.get("text", "")) + current_page = page + full_text = "".join(text_parts).strip() + if full_text: + sections.append({ + "fileId": doc.get("fileId"), + "fileName": file_name, + "text": full_text, + "blockCount": len(blocks) + }) + return { + **state, + "sections": sections + } + + +def create_context_extraction_graph(): + """Create and compile the context extraction LangGraph.""" + workflow = StateGraph(ContextExtractionState) + workflow.add_node("extract_text", extract_text_node) + workflow.add_node("structure_content", structure_content_node) + workflow.add_edge(START, "extract_text") + workflow.add_edge("extract_text", "structure_content") + workflow.add_edge("structure_content", END) + return workflow.compile() + + +def run_extraction(files: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Run the context extraction pipeline on uploaded files. + + Args: + files: List of {fileId, bytes, mimeType, fileName} + + Returns: + { + "textBlocks": [...], + "sections": [...], + "summaries": [], + "errors": [...] + } + """ + state: ContextExtractionState = { + "files": files, + "textBlocks": [], + "sections": [], + "summaries": [], + "errors": [] + } + graph = create_context_extraction_graph() + final_state = graph.invoke(state) + return { + "textBlocks": final_state.get("textBlocks", []), + "sections": final_state.get("sections", []), + "summaries": final_state.get("summaries", []), + "errors": final_state.get("errors", []) + } diff --git a/modules/features/chatbotV2/datamodelFeatureChatbotV2.py b/modules/features/chatbotV2/datamodelFeatureChatbotV2.py new file mode 100644 index 00000000..2fbd463e --- /dev/null +++ b/modules/features/chatbotV2/datamodelFeatureChatbotV2.py @@ -0,0 +1,85 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Pydantic models for Chatbot V2 feature. +Stores context per chat: uploaded files, extracted content, and conversation history. +""" + +import uuid +from typing import Optional, List, Dict, Any +from pydantic import BaseModel, Field + +from modules.shared.timeUtils import getUtcTimestamp + + +class ChatbotV2ContextFile(BaseModel): + """Uploaded file metadata for context extraction.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + fileId: str = Field(description="Foreign key to file in central Files table") + fileName: str = Field(description="Original file name") + mimeType: str = Field(default="application/octet-stream", description="MIME type") + fileSize: int = Field(default=0, description="File size in bytes") + uploadOrder: int = Field(default=0, description="Order of upload (0-based)") + + +class ChatbotV2ExtractedContext(BaseModel): + """Extracted content per conversation - text blocks and summaries for chat context.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + textBlocks: List[Dict[str, Any]] = Field(default_factory=list, description="Extracted text blocks per file (page, text, block_id)") + summaries: List[Dict[str, Any]] = Field(default_factory=list, description="Optional per-document or per-section summaries") + extractionStatus: str = Field(default="pending", description="pending|running|completed|failed") + errors: List[str] = Field(default_factory=list, description="Extraction errors if any") + createdAt: float = Field(default_factory=getUtcTimestamp, description="When extraction completed") + + +class ChatbotV2Document(BaseModel): + """Documents attached to chatbot V2 messages.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + messageId: str = Field(description="Foreign key to message") + fileId: str = Field(description="Foreign key to file") + fileName: str = Field(description="Name of the file") + fileSize: int = Field(default=0, description="Size of the file") + mimeType: str = Field(default="application/octet-stream", description="MIME type") + + +class ChatbotV2Message(BaseModel): + """Messages in Chatbot V2 conversations.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + parentMessageId: Optional[str] = Field(None, description="Parent message ID for threading") + message: Optional[str] = Field(None, description="Message content") + role: str = Field(description="Role: user or assistant") + status: str = Field(default="step", description="Status: first, step, last") + sequenceNr: int = Field(default=0, description="Sequence number of the message") + publishedAt: Optional[float] = Field(default=None, description="When the message was published (UTC timestamp)") + roundNumber: int = Field(default=1, description="Round number in conversation") + + +class ChatbotV2Log(BaseModel): + """Log entries for Chatbot V2 conversations.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + conversationId: str = Field(description="Foreign key to conversation") + message: str = Field(description="Log message") + type: str = Field(default="info", description="Log type: info, warning, error") + timestamp: float = Field(default_factory=getUtcTimestamp, description="When the log entry was created") + status: Optional[str] = Field(None, description="Status of the log entry") + progress: Optional[float] = Field(None, description="Progress indicator (0.0 to 1.0)") + + +class ChatbotV2Conversation(BaseModel): + """Chatbot V2 conversation - stores context information per chat.""" + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + featureInstanceId: str = Field(description="Feature instance ID for per-instance isolation") + mandateId: Optional[str] = Field(None, description="Mandate ID for RBAC") + name: Optional[str] = Field(None, description="Name of the conversation") + status: str = Field(default="extracting", description="extracting|ready|running|stopped") + currentRound: int = Field(default=0, description="Current round number") + lastActivity: float = Field(default_factory=getUtcTimestamp, description="Timestamp of last activity") + startedAt: float = Field(default_factory=getUtcTimestamp, description="When the conversation started") + extractedContextId: Optional[str] = Field(None, description="FK to ChatbotV2ExtractedContext when ready") + maxSteps: int = Field(default=10, description="Maximum number of chat rounds") + # Hydrated from child tables (not stored in DB as columns) + contextFiles: List[ChatbotV2ContextFile] = Field(default_factory=list, description="Uploaded context files") + messages: List[ChatbotV2Message] = Field(default_factory=list, description="Conversation messages") diff --git a/modules/features/chatbotV2/interfaceFeatureChatbotV2.py b/modules/features/chatbotV2/interfaceFeatureChatbotV2.py new file mode 100644 index 00000000..9dbc12a9 --- /dev/null +++ b/modules/features/chatbotV2/interfaceFeatureChatbotV2.py @@ -0,0 +1,440 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Interface to Chatbot V2 database. +Manages context-aware conversations with file upload and extraction. +""" + +import logging +import math +import uuid +from typing import Dict, Any, List, Optional, Union + +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.security.rbac import RbacClass +from modules.datamodels.datamodelRbac import AccessRuleContext +from modules.datamodels.datamodelUam import User, AccessLevel +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult +from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp + +from .datamodelFeatureChatbotV2 import ( + ChatbotV2Conversation, + ChatbotV2ContextFile, + ChatbotV2ExtractedContext, + ChatbotV2Message, + ChatbotV2Document, + ChatbotV2Log, +) + +logger = logging.getLogger(__name__) + +_chatbotV2Interfaces: Dict[str, "ChatbotV2Objects"] = {} +FEATURE_CODE = "chatbotv2" + + +def getInterface( + currentUser: User, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None +) -> "ChatbotV2Objects": + """Get or create a ChatbotV2Objects instance for the given user context.""" + if not currentUser or not currentUser.id: + raise ValueError("Valid user context required") + + key = f"{currentUser.id}_{mandateId or ''}_{featureInstanceId or ''}" + if key not in _chatbotV2Interfaces: + _chatbotV2Interfaces[key] = ChatbotV2Objects( + currentUser, + mandateId=mandateId, + featureInstanceId=featureInstanceId + ) + else: + _chatbotV2Interfaces[key].setUserContext( + currentUser, + mandateId=mandateId, + featureInstanceId=featureInstanceId + ) + return _chatbotV2Interfaces[key] + + +class ChatbotV2Objects: + """Interface to Chatbot V2 database.""" + + def __init__( + self, + currentUser: Optional[User] = None, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ): + self.currentUser = currentUser + self.userId = currentUser.id if currentUser else None + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + self.featureCode = FEATURE_CODE + self.rbac = None + self._initializeDatabase() + if currentUser: + self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) + + def setUserContext( + self, + currentUser: User, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None + ): + """Set user context for the interface.""" + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + if not self.userId: + raise ValueError("Invalid user context: id is required") + from modules.security.rootAccess import getRootDbAppConnector + dbApp = getRootDbAppConnector() + self.rbac = RbacClass(self.db, dbApp=dbApp) + self.db.updateContext(self.userId) + + def __del__(self): + if hasattr(self, "db") and self.db is not None: + try: + self.db.close() + except Exception as e: + logger.error(f"Error closing database connection: {e}") + + def _initializeDatabase(self): + try: + dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") + dbDatabase = "poweron_chatbotv2" + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId + ) + logger.info("ChatbotV2 database initialized successfully") + except Exception as e: + logger.error(f"Failed to initialize ChatbotV2 database: {e}") + raise + + def checkRbacPermission( + self, + modelClass: type, + operation: str, + recordId: Optional[str] = None + ) -> bool: + """Check RBAC permission for an operation on a table.""" + if not self.rbac or not self.currentUser: + return False + from modules.interfaces.interfaceRbac import buildDataObjectKey + objectKey = buildDataObjectKey(modelClass.__name__, featureCode=self.featureCode) + permissions = self.rbac.getUserPermissions( + self.currentUser, + AccessRuleContext.DATA, + objectKey, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId + ) + if operation == "create": + return permissions.create != AccessLevel.NONE + if operation == "update": + return permissions.update != AccessLevel.NONE + if operation == "delete": + return permissions.delete != AccessLevel.NONE + if operation == "read": + return permissions.read != AccessLevel.NONE + return False + + def _separateObjectFields(self, model_class, data: Dict[str, Any]) -> tuple: + """Separate simple fields from object fields (contextFiles, messages).""" + simple_fields = {} + object_fields = {} + model_fields = model_class.model_fields + for field_name, value in data.items(): + if field_name in model_fields: + if field_name in ("contextFiles", "messages"): + object_fields[field_name] = value + continue + if field_name.startswith("_"): + simple_fields[field_name] = value + elif isinstance(value, (str, int, float, bool, type(None))): + simple_fields[field_name] = value + elif field_name in model_fields: + field_info = model_fields[field_name] + if hasattr(field_info, "annotation"): + from typing import get_origin, get_args + origin = get_origin(field_info.annotation) + if origin in (dict, list): + simple_fields[field_name] = value + else: + object_fields[field_name] = value + else: + object_fields[field_name] = value + return simple_fields, object_fields + + # ===== Conversation CRUD ===== + + def getConversations( + self, + pagination: Optional[PaginationParams] = None + ) -> Union[List[Dict[str, Any]], PaginatedResult]: + """Get conversations for current feature instance.""" + record_filter = {} + if self.featureInstanceId: + record_filter["featureInstanceId"] = self.featureInstanceId + records = getRecordsetWithRBAC( + self.db, + ChatbotV2Conversation, + self.currentUser, + recordFilter=record_filter if record_filter else None, + orderBy="lastActivity", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode=self.featureCode + ) + if pagination is None: + return records + total = len(records) + page = pagination.page or 1 + page_size = pagination.pageSize or 20 + start = (page - 1) * page_size + items = records[start : start + page_size] + total_pages = math.ceil(total / page_size) if total > 0 else 0 + return PaginatedResult(items=items, totalItems=total, totalPages=total_pages) + + def getConversation(self, conversationId: str) -> Optional[ChatbotV2Conversation]: + """Get a conversation by ID with hydrated context files and messages.""" + records = getRecordsetWithRBAC( + self.db, + ChatbotV2Conversation, + self.currentUser, + recordFilter={"id": conversationId}, + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode=self.featureCode + ) + if not records: + return None + r = records[0] + context_files = self.getContextFiles(conversationId) + messages = self.getMessages(conversationId) + max_steps = r.get("maxSteps") + if max_steps is None: + max_steps = 10 + return ChatbotV2Conversation( + id=r["id"], + featureInstanceId=r.get("featureInstanceId", "") or self.featureInstanceId or "", + mandateId=r.get("mandateId"), + name=r.get("name"), + status=r.get("status", "extracting"), + currentRound=r.get("currentRound", 0), + lastActivity=r.get("lastActivity", getUtcTimestamp()), + startedAt=r.get("startedAt", getUtcTimestamp()), + extractedContextId=r.get("extractedContextId"), + maxSteps=max_steps, + contextFiles=context_files, + messages=messages + ) + + def createConversation(self, data: Dict[str, Any]) -> ChatbotV2Conversation: + """Create a new conversation.""" + if not self.checkRbacPermission(ChatbotV2Conversation, "create"): + raise PermissionError("No permission to create conversations") + data["featureInstanceId"] = data.get("featureInstanceId") or self.featureInstanceId or "" + data["mandateId"] = data.get("mandateId") or self.mandateId + simple, obj = self._separateObjectFields(ChatbotV2Conversation, data) + if "maxSteps" not in simple or simple["maxSteps"] is None: + simple["maxSteps"] = 10 + created = self.db.recordCreate(ChatbotV2Conversation, simple) + max_steps = created.get("maxSteps") + if max_steps is None: + max_steps = 10 + return ChatbotV2Conversation( + id=created["id"], + featureInstanceId=created.get("featureInstanceId", ""), + mandateId=created.get("mandateId"), + name=created.get("name"), + status=created.get("status", "extracting"), + currentRound=created.get("currentRound", 0), + lastActivity=created.get("lastActivity", getUtcTimestamp()), + startedAt=created.get("startedAt", getUtcTimestamp()), + extractedContextId=created.get("extractedContextId"), + maxSteps=max_steps, + contextFiles=[], + messages=[] + ) + + def updateConversation(self, conversationId: str, data: Dict[str, Any]) -> Optional[ChatbotV2Conversation]: + """Update a conversation.""" + conv = self.getConversation(conversationId) + if not conv: + return None + if not self.checkRbacPermission(ChatbotV2Conversation, "update"): + raise PermissionError("No permission to update conversation") + simple, _ = self._separateObjectFields(ChatbotV2Conversation, data) + simple["lastActivity"] = getUtcTimestamp() + updated = self.db.recordModify(ChatbotV2Conversation, conversationId, simple) + if not updated: + return None + return self.getConversation(conversationId) + + def deleteConversation(self, conversationId: str) -> bool: + """Delete a conversation and all related data.""" + conv = self.getConversation(conversationId) + if not conv: + return False + if not self.checkRbacPermission(ChatbotV2Conversation, "delete"): + raise PermissionError("No permission to delete conversation") + for cf in self.getContextFiles(conversationId): + self.db.recordDelete(ChatbotV2ContextFile, cf.id) + ctx = self.getExtractedContextByConversation(conversationId) + if ctx: + self.db.recordDelete(ChatbotV2ExtractedContext, ctx.id) + for msg in self.getMessages(conversationId): + for doc in self.getDocuments(msg.id): + self.db.recordDelete(ChatbotV2Document, doc.id) + self.db.recordDelete(ChatbotV2Message, msg.id) + for log in self.getLogs(conversationId): + self.db.recordDelete(ChatbotV2Log, log.id) + return self.db.recordDelete(ChatbotV2Conversation, conversationId) + + # ===== Context File CRUD ===== + + def getContextFiles(self, conversationId: str) -> List[ChatbotV2ContextFile]: + """Get context files for a conversation.""" + records = getRecordsetWithRBAC( + self.db, + ChatbotV2ContextFile, + self.currentUser, + recordFilter={"conversationId": conversationId}, + orderBy="uploadOrder", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode=self.featureCode + ) + return [ChatbotV2ContextFile(**r) for r in records] + + def createContextFile(self, data: Dict[str, Any]) -> ChatbotV2ContextFile: + """Create a context file record.""" + return ChatbotV2ContextFile(**self.db.recordCreate(ChatbotV2ContextFile, data)) + + # ===== Extracted Context CRUD ===== + + def getExtractedContext(self, extractedContextId: str) -> Optional[ChatbotV2ExtractedContext]: + """Get extracted context by ID.""" + records = self.db.getRecordset( + ChatbotV2ExtractedContext, + recordFilter={"id": extractedContextId} + ) + if not records: + return None + return ChatbotV2ExtractedContext(**records[0]) + + def getExtractedContextByConversation(self, conversationId: str) -> Optional[ChatbotV2ExtractedContext]: + """Get extracted context for a conversation.""" + records = self.db.getRecordset( + ChatbotV2ExtractedContext, + recordFilter={"conversationId": conversationId} + ) + if not records: + return None + return ChatbotV2ExtractedContext(**records[0]) + + def createExtractedContext(self, data: Dict[str, Any]) -> ChatbotV2ExtractedContext: + """Create extracted context record.""" + created = self.db.recordCreate(ChatbotV2ExtractedContext, data) + return ChatbotV2ExtractedContext(**created) + + def updateExtractedContext(self, extractedContextId: str, data: Dict[str, Any]) -> Optional[ChatbotV2ExtractedContext]: + """Update extracted context.""" + updated = self.db.recordModify(ChatbotV2ExtractedContext, extractedContextId, data) + return ChatbotV2ExtractedContext(**updated) if updated else None + + # ===== Message CRUD ===== + + def getMessages(self, conversationId: str) -> List[ChatbotV2Message]: + """Get messages for a conversation.""" + records = getRecordsetWithRBAC( + self.db, + ChatbotV2Message, + self.currentUser, + recordFilter={"conversationId": conversationId}, + orderBy="sequenceNr", + mandateId=self.mandateId, + featureInstanceId=self.featureInstanceId, + featureCode=self.featureCode + ) + return [ChatbotV2Message(**r) for r in records] + + def createMessage(self, data: Dict[str, Any]) -> ChatbotV2Message: + """Create a message.""" + if "id" not in data or not data["id"]: + data["id"] = str(uuid.uuid4()) + created = self.db.recordCreate(ChatbotV2Message, data) + return ChatbotV2Message(**created) + + # ===== Document CRUD ===== + + def getDocuments(self, messageId: str) -> List[ChatbotV2Document]: + """Get documents for a message.""" + records = self.db.getRecordset( + ChatbotV2Document, + recordFilter={"messageId": messageId} + ) + return [ChatbotV2Document(**r) for r in records] + + def createDocument(self, data: Dict[str, Any]) -> ChatbotV2Document: + """Create a document record.""" + created = self.db.recordCreate(ChatbotV2Document, data) + return ChatbotV2Document(**created) + + # ===== Log CRUD ===== + + def getLogs(self, conversationId: str) -> List[ChatbotV2Log]: + """Get logs for a conversation.""" + records = self.db.getRecordset( + ChatbotV2Log, + recordFilter={"conversationId": conversationId} + ) + # Sort by timestamp (connector doesn't support orderBy) + logs = [ChatbotV2Log(**r) for r in records] + logs.sort(key=lambda log: parseTimestamp(log.timestamp, default=0)) + return logs + + def createLog(self, data: Dict[str, Any]) -> ChatbotV2Log: + """Create a log entry.""" + if "timestamp" not in data: + data["timestamp"] = getUtcTimestamp() + created = self.db.recordCreate(ChatbotV2Log, data) + return ChatbotV2Log(**created) + + # ===== Unified Chat Data (for streaming/polling) ===== + + def getUnifiedChatData( + self, + conversationId: str, + afterTimestamp: Optional[float] = None + ) -> Dict[str, Any]: + """Get unified chat data (messages, logs) in chronological order.""" + conv = self.getConversation(conversationId) + if not conv: + return {"items": []} + items = [] + for msg in conv.messages: + ts = parseTimestamp(msg.publishedAt, default=getUtcTimestamp()) + if afterTimestamp is not None and ts <= afterTimestamp: + continue + items.append({"type": "message", "createdAt": ts, "item": msg}) + for log in self.getLogs(conversationId): + ts = parseTimestamp(log.timestamp, default=getUtcTimestamp()) + if afterTimestamp is not None and ts <= afterTimestamp: + continue + items.append({"type": "log", "createdAt": ts, "item": log}) + items.sort(key=lambda x: x.get("createdAt", 0)) + return {"items": items} diff --git a/modules/features/chatbotV2/mainChatbotV2.py b/modules/features/chatbotV2/mainChatbotV2.py new file mode 100644 index 00000000..11f6336e --- /dev/null +++ b/modules/features/chatbotV2/mainChatbotV2.py @@ -0,0 +1,250 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chatbot V2 Feature Container - Main Module. +Handles feature initialization and RBAC catalog registration. +Context-aware chat with file upload and extraction. +""" + +import logging +from typing import Dict, List, Any + +logger = logging.getLogger(__name__) + +# Feature metadata +FEATURE_CODE = "chatbotv2" +FEATURE_LABEL = {"en": "Chatbot V2", "de": "Chatbot V2", "fr": "Chatbot V2"} +FEATURE_ICON = "mdi-robot-outline" + +# UI Objects for RBAC catalog - single "conversations" view (upload + chat in one page) +UI_OBJECTS = [ + { + "objectKey": "ui.feature.chatbotv2.conversations", + "label": {"en": "Conversations", "de": "Konversationen", "fr": "Conversations"}, + "meta": {"area": "conversations"} + }, +] + +# Resource Objects for RBAC catalog +RESOURCE_OBJECTS = [ + { + "objectKey": "resource.feature.chatbotv2.upload", + "label": {"en": "Upload Files", "de": "Dateien hochladen", "fr": "Télécharger fichiers"}, + "meta": {"endpoint": "/api/chatbotv2/{instanceId}/upload", "method": "POST"} + }, + { + "objectKey": "resource.feature.chatbotv2.startStream", + "label": {"en": "Start Chat (Stream)", "de": "Chat starten (Stream)", "fr": "Démarrer chat (Stream)"}, + "meta": {"endpoint": "/api/chatbotv2/{instanceId}/start/stream", "method": "POST"} + }, + { + "objectKey": "resource.feature.chatbotv2.stop", + "label": {"en": "Stop Chat", "de": "Chat stoppen", "fr": "Arrêter chat"}, + "meta": {"endpoint": "/api/chatbotv2/{instanceId}/stop/{workflowId}", "method": "POST"} + }, + { + "objectKey": "resource.feature.chatbotv2.threads", + "label": {"en": "Get Threads", "de": "Threads abrufen", "fr": "Récupérer threads"}, + "meta": {"endpoint": "/api/chatbotv2/{instanceId}/threads", "method": "GET"} + }, + { + "objectKey": "resource.feature.chatbotv2.delete", + "label": {"en": "Delete Chat", "de": "Chat löschen", "fr": "Supprimer chat"}, + "meta": {"endpoint": "/api/chatbotv2/{instanceId}/conversations/{workflowId}", "method": "DELETE"} + }, +] + +# Template roles for this feature +TEMPLATE_ROLES = [ + { + "roleLabel": "chatbotv2-viewer", + "description": { + "en": "Chatbot V2 Viewer - View threads (read-only)", + "de": "Chatbot V2 Betrachter - Threads ansehen (nur lesen)", + "fr": "Visualiseur Chatbot V2 - Consulter les threads (lecture seule)" + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.chatbotv2.conversations", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.threads", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, + ] + }, + { + "roleLabel": "chatbotv2-user", + "description": { + "en": "Chatbot V2 User - Upload, extract, and chat with own threads", + "de": "Chatbot V2 Benutzer - Hochladen, extrahieren und chatten mit eigenen Threads", + "fr": "Utilisateur Chatbot V2 - Upload, extraction et chat avec ses threads" + }, + "accessRules": [ + {"context": "UI", "item": "ui.feature.chatbotv2.conversations", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.upload", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.startStream", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.stop", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.threads", "view": True}, + {"context": "RESOURCE", "item": "resource.feature.chatbotv2.delete", "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, + ] + }, + { + "roleLabel": "chatbotv2-admin", + "description": { + "en": "Chatbot V2 Admin - Full access to all features", + "de": "Chatbot V2 Admin - Vollzugriff auf alle Funktionen", + "fr": "Administrateur Chatbot V2 - Accès complet" + }, + "accessRules": [ + {"context": "UI", "item": None, "view": True}, + {"context": "RESOURCE", "item": None, "view": True}, + {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, + ] + }, +] + + +def getFeatureDefinition() -> Dict[str, Any]: + """Return the feature definition for registration.""" + return { + "code": FEATURE_CODE, + "label": FEATURE_LABEL, + "icon": FEATURE_ICON, + } + + +def getUiObjects() -> List[Dict[str, Any]]: + """Return UI objects for RBAC catalog registration.""" + return UI_OBJECTS + + +def getResourceObjects() -> List[Dict[str, Any]]: + """Return resource objects for RBAC catalog registration.""" + return RESOURCE_OBJECTS + + +def getTemplateRoles() -> List[Dict[str, Any]]: + """Return template roles for this feature.""" + return TEMPLATE_ROLES + + +def registerFeature(catalogService) -> bool: + """ + Register this feature's RBAC objects in the catalog. + """ + try: + for uiObj in UI_OBJECTS: + catalogService.registerUiObject( + featureCode=FEATURE_CODE, + objectKey=uiObj["objectKey"], + label=uiObj["label"], + meta=uiObj.get("meta") + ) + + for resObj in RESOURCE_OBJECTS: + catalogService.registerResourceObject( + featureCode=FEATURE_CODE, + objectKey=resObj["objectKey"], + label=resObj["label"], + meta=resObj.get("meta") + ) + + _syncTemplateRolesToDb() + + logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") + return True + + except Exception as e: + logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") + return False + + +def _syncTemplateRolesToDb() -> int: + """Sync template roles and their AccessRules to the database.""" + try: + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext + + rootInterface = getRootInterface() + + existingRoles = rootInterface.getRolesByFeatureCode(FEATURE_CODE) + templateRoles = [r for r in existingRoles if r.mandateId is None] + existingRoleLabels = {r.roleLabel: str(r.id) for r in templateRoles} + + createdCount = 0 + for roleTemplate in TEMPLATE_ROLES: + roleLabel = roleTemplate["roleLabel"] + + if roleLabel in existingRoleLabels: + roleId = existingRoleLabels[roleLabel] + _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) + else: + newRole = Role( + roleLabel=roleLabel, + description=roleTemplate.get("description", {}), + featureCode=FEATURE_CODE, + mandateId=None, + featureInstanceId=None, + isSystemRole=False + ) + createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) + roleId = createdRole.get("id") + + _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) + + logger.info(f"Created template role '{roleLabel}' with ID {roleId}") + createdCount += 1 + + if createdCount > 0: + logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") + + return createdCount + + except Exception as e: + logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") + return 0 + + +def _ensureAccessRulesForRole(rootInterface, roleId: str, ruleTemplates: List[Dict[str, Any]]) -> int: + """Ensure AccessRules exist for a role based on templates.""" + from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext + + existingRules = rootInterface.getAccessRulesByRole(roleId) + existingSignatures = set() + for rule in existingRules: + sig = (rule.context.value if rule.context else None, rule.item) + existingSignatures.add(sig) + + createdCount = 0 + for template in ruleTemplates: + context = template.get("context", "UI") + item = template.get("item") + sig = (context, item) + + if sig in existingSignatures: + continue + + if context == "UI": + contextEnum = AccessRuleContext.UI + elif context == "DATA": + contextEnum = AccessRuleContext.DATA + elif context == "RESOURCE": + contextEnum = AccessRuleContext.RESOURCE + else: + contextEnum = context + + newRule = AccessRule( + roleId=roleId, + context=contextEnum, + item=item, + view=template.get("view", False), + read=template.get("read"), + create=template.get("create"), + update=template.get("update"), + delete=template.get("delete"), + ) + rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) + createdCount += 1 + + if createdCount > 0: + logger.debug(f"Created {createdCount} AccessRules for role {roleId}") + + return createdCount diff --git a/modules/features/chatbotV2/routeFeatureChatbotV2.py b/modules/features/chatbotV2/routeFeatureChatbotV2.py new file mode 100644 index 00000000..dd460029 --- /dev/null +++ b/modules/features/chatbotV2/routeFeatureChatbotV2.py @@ -0,0 +1,350 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chatbot V2 routes - context-aware chat with file upload and extraction. +""" + +import asyncio +import json +import math +import logging +import uuid +from typing import Optional, Any, Dict, Union +from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Request, status, UploadFile, File +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field + +from modules.auth import limiter, getRequestContext, RequestContext +from modules.interfaces.interfaceDbApp import getRootInterface +from modules.interfaces.interfaceFeatures import getFeatureInterface +from modules.datamodels.datamodelChat import UserInputRequest +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata +from modules.shared.timeUtils import getUtcTimestamp + +from . import interfaceFeatureChatbotV2 as interfaceDbChat +from .interfaceFeatureChatbotV2 import getInterface as getChatbotV2Interface +from .datamodelFeatureChatbotV2 import ChatbotV2Conversation +from .serviceChatbotV2 import uploadAndExtract, chatProcessV2 +from modules.features.chatbot.streaming.events import get_event_manager + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/chatbotv2", + tags=["Chatbot V2"], + responses={404: {"description": "Not found"}} +) + + +class UploadRequest(BaseModel): + """Request body for file upload - files must be uploaded to central storage first.""" + listFileId: list[str] = Field(default_factory=list, description="List of file IDs from central storage") + + +def _getServiceChat(context: RequestContext, instanceId: Optional[str] = None): + """Get ChatbotV2 interface with instance context.""" + mandateId = str(context.mandateId) if context.mandateId else None + return getChatbotV2Interface( + context.user, + mandateId=mandateId, + featureInstanceId=instanceId + ) + + +def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: + """Validate that the user has access to the feature instance.""" + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException( + status_code=404, + detail=f"Feature instance '{instanceId}' not found" + ) + if instance.featureCode != "chatbotv2": + raise HTTPException( + status_code=400, + detail=f"Instance '{instanceId}' is not a chatbotv2 instance" + ) + if not context.hasSysAdminRole: + featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) + hasAccess = any( + str(fa.featureInstanceId) == instanceId and fa.enabled + for fa in featureAccesses + ) + if not hasAccess: + raise HTTPException( + status_code=403, + detail=f"Access denied to feature instance '{instanceId}'" + ) + return str(instance.mandateId) + + +# ============================================================================= +# Upload - start extraction +# ============================================================================= +@router.post("/{instanceId}/upload") +@limiter.limit("60/minute") +async def upload_files( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + body: UploadRequest = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """ + Upload files as context and start extraction. + Files must be uploaded to central storage first; pass their file IDs in listFileId. + Returns conversationId. Extraction runs in background; poll threads or use SSE for status. + """ + mandateId = _validateInstanceAccess(instanceId, context) + if not body.listFileId: + raise HTTPException(status_code=400, detail="listFileId is required and must not be empty") + try: + conversation = await uploadAndExtract( + context.user, + mandateId=mandateId, + instanceId=instanceId, + listFileId=body.listFileId + ) + return { + "conversationId": conversation.id, + "status": conversation.status, + "message": "Extraction started. Poll GET /threads?workflowId={} for status.".format(conversation.id) + } + except Exception as e: + logger.error(f"Error in upload_files: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# ============================================================================= +# List threads - MUST be first to avoid /{instanceId}/{workflowId} matching +# ============================================================================= +@router.get("/{instanceId}/threads") +@limiter.limit("120/minute") +def get_threads( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + workflowId: Optional[str] = Query(None, description="Optional workflow/conversation ID for details"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), + context: RequestContext = Depends(getRequestContext) +) -> Union[PaginatedResponse, Dict[str, Any]]: + """List conversations or get details for a specific one.""" + _validateInstanceAccess(instanceId, context) + interfaceDbChat = _getServiceChat(context, instanceId) + + if workflowId: + conv = interfaceDbChat.getConversation(workflowId) + if not conv: + raise HTTPException(status_code=404, detail=f"Conversation {workflowId} not found") + workflow_dict = conv.model_dump() + chatData = interfaceDbChat.getUnifiedChatData(workflowId, None) + return {"workflow": workflow_dict, "chatData": chatData} + + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + paginationParams = PaginationParams(**paginationDict) if paginationDict else None + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + all_convs = interfaceDbChat.getConversations(pagination=None) + # all_convs from getConversations can be list of dicts (from getRecordsetWithRBAC) + items = [c if isinstance(c, dict) else c.model_dump() for c in all_convs] + + if paginationParams: + totalItems = len(items) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + workflows = items[startIdx:endIdx] + else: + workflows = items + totalItems = len(items) + totalPages = 1 + + metadata = PaginationMetadata( + currentPage=paginationParams.page if paginationParams else 1, + pageSize=paginationParams.pageSize if paginationParams else len(workflows), + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort if paginationParams else [], + filters=paginationParams.filters if paginationParams else None + ) + return PaginatedResponse(items=workflows, pagination=metadata) + + +# ============================================================================= +# Start/continue chat (SSE stream) +# ============================================================================= +@router.post("/{instanceId}/start/stream") +@limiter.limit("120/minute") +async def stream_chat_start( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + workflowId: Optional[str] = Query(None, description="Optional conversation ID to continue"), + userInput: UserInputRequest = Body(...), + context: RequestContext = Depends(getRequestContext) +) -> StreamingResponse: + """Start or continue a chat with SSE streaming.""" + mandateId = _validateInstanceAccess(instanceId, context) + event_manager = get_event_manager() + final_workflow_id = workflowId or userInput.workflowId + + try: + workflow = await chatProcessV2( + context.user, + mandateId=mandateId, + userInput=userInput, + conversationId=final_workflow_id, + instanceId=instanceId + ) + if not workflow: + raise HTTPException(status_code=500, detail="Failed to create or load workflow") + + queue = event_manager.get_queue(workflow.id) + if not queue: + queue = event_manager.create_queue(workflow.id) + + async def event_stream(): + try: + interfaceDbChat = _getServiceChat(context, instanceId) + chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) + if chatData.get("items"): + for item in chatData["items"]: + ser = { + "type": item.get("type"), + "createdAt": item.get("createdAt"), + "item": item.get("item").model_dump() if hasattr(item.get("item"), "model_dump") else item.get("item") + } + yield f"data: {json.dumps(ser)}\n\n" + + keepalive_interval = 30.0 + last_keepalive = asyncio.get_event_loop().time() + status_check_interval = 5.0 + last_status_check = asyncio.get_event_loop().time() + timeout = 300.0 + start_time = asyncio.get_event_loop().time() + + while True: + elapsed = asyncio.get_event_loop().time() - start_time + if elapsed > timeout: + break + if await request.is_disconnected(): + break + current_time = asyncio.get_event_loop().time() + + if current_time - last_status_check >= status_check_interval: + try: + cw = interfaceDbChat.getConversation(workflow.id) + if cw and cw.status == "stopped": + break + except Exception: + pass + last_status_check = current_time + + try: + event = await asyncio.wait_for(queue.get(), timeout=1.0) + event_type = event.get("type") + event_data = event.get("data", {}) + + if event_type == "chatdata" and event_data: + if event_data.get("type") == "status": + yield f"data: {json.dumps({'type': 'status', 'label': event_data.get('label', '')})}\n\n" + else: + item = event_data + if isinstance(item, dict) and "item" in item: + obj = item.get("item") + if hasattr(obj, "model_dump"): + item = {**item, "item": obj.model_dump()} + yield f"data: {json.dumps(item)}\n\n" + elif event_type in ("complete", "stopped"): + break + elif event_type == "error" and event.get("step") == "error": + break + last_keepalive = current_time + except asyncio.TimeoutError: + if current_time - last_keepalive >= keepalive_interval: + yield ": keepalive\n\n" + last_keepalive = current_time + except Exception as e: + logger.error(f"Error in event stream: {e}") + break + except Exception as e: + logger.error(f"Error in event stream generator: {e}", exc_info=True) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no" + } + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error in stream_chat_start: {str(e)}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +# ============================================================================= +# Stop chat +# ============================================================================= +@router.post("/{instanceId}/stop/{workflowId}") +@limiter.limit("120/minute") +async def stop_chat( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + workflowId: str = Path(..., description="Conversation ID to stop"), + context: RequestContext = Depends(getRequestContext) +) -> ChatbotV2Conversation: + """Stop a running chat.""" + _validateInstanceAccess(instanceId, context) + interfaceDbChat = _getServiceChat(context, instanceId) + conv = interfaceDbChat.getConversation(workflowId) + if not conv: + raise HTTPException(status_code=404, detail=f"Conversation {workflowId} not found") + interfaceDbChat.updateConversation(workflowId, {"status": "stopped", "lastActivity": getUtcTimestamp()}) + interfaceDbChat.createLog({ + "conversationId": workflowId, + "message": "Workflow stopped by user", + "type": "warning", + "status": "stopped", + "timestamp": getUtcTimestamp() + }) + event_manager = get_event_manager() + await event_manager.emit_event( + context_id=workflowId, + event_type="stopped", + data={"workflowId": workflowId}, + event_category="workflow", + message="Workflow stopped by user", + step="stopped" + ) + return interfaceDbChat.getConversation(workflowId) + + +# ============================================================================= +# Delete conversation - use /conversations/{workflowId} to avoid +# /{instanceId}/{workflowId} matching GET /threads (workflowId="threads") +# ============================================================================= +@router.delete("/{instanceId}/conversations/{workflowId}") +@limiter.limit("120/minute") +def delete_conversation( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + workflowId: str = Path(..., description="Conversation ID to delete"), + context: RequestContext = Depends(getRequestContext) +) -> Dict[str, Any]: + """Delete a conversation and its data.""" + _validateInstanceAccess(instanceId, context) + interfaceDbChat = _getServiceChat(context, instanceId) + conv = interfaceDbChat.getConversation(workflowId) + if not conv: + raise HTTPException(status_code=404, detail=f"Conversation {workflowId} not found") + success = interfaceDbChat.deleteConversation(workflowId) + if not success: + raise HTTPException(status_code=500, detail="Failed to delete conversation") + return {"id": workflowId, "message": "Conversation deleted successfully"} diff --git a/modules/features/chatbotV2/serviceChatbotV2.py b/modules/features/chatbotV2/serviceChatbotV2.py new file mode 100644 index 00000000..e71f92b2 --- /dev/null +++ b/modules/features/chatbotV2/serviceChatbotV2.py @@ -0,0 +1,258 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Chatbot V2 service - orchestration for upload/extraction and chat. +""" + +import logging +import uuid +from typing import Optional, List + +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelChat import UserInputRequest +from modules.datamodels.datamodelAi import OperationTypeEnum, ProcessingModeEnum +from modules.shared.timeUtils import getUtcTimestamp + +from modules.services import getInterface as getServices +from .interfaceFeatureChatbotV2 import getInterface as getChatbotV2Interface +from .datamodelFeatureChatbotV2 import ChatbotV2Conversation +from .contextExtractionLangGraph import run_extraction +from .chatbotV2 import create_chat_graph +from .config import load_chatbotv2_config_from_instance +from .bridges import AICenterChatModel, clear_workflow_allowed_providers, ChatbotV2Checkpointer +from modules.features.chatbot.streaming.events import get_event_manager + +logger = logging.getLogger(__name__) + + +async def _load_config(instance_id: Optional[str]): + """Load ChatbotV2 config from feature instance.""" + if not instance_id: + return None + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.interfaces.interfaceFeatures import getFeatureInterface + root = getRootInterface() + feat = getFeatureInterface(root.db) + instance = feat.getFeatureInstance(instance_id) + if not instance: + return None + return load_chatbotv2_config_from_instance(instance) + + +async def uploadAndExtract( + currentUser: User, + mandateId: Optional[str], + instanceId: str, + listFileId: List[str] +) -> ChatbotV2Conversation: + """ + Create conversation, store context files, run extraction, save result. + """ + interface = getChatbotV2Interface(currentUser, mandateId=mandateId, featureInstanceId=instanceId) + services = getServices(currentUser, mandateId=mandateId, featureInstanceId=instanceId) + + conversation_id = str(uuid.uuid4()) + conv_data = { + "id": conversation_id, + "featureInstanceId": instanceId, + "mandateId": mandateId, + "status": "extracting", + "name": "Context Chat", + "currentRound": 0, + "maxSteps": 10, + "startedAt": getUtcTimestamp(), + "lastActivity": getUtcTimestamp() + } + conv = interface.createConversation(conv_data) + + files_for_extraction = [] + for idx, file_id in enumerate(listFileId): + try: + file_info = services.chat.getFileInfo(file_id) + if not file_info: + logger.warning(f"File {file_id} not found") + continue + file_bytes = services.chat.getFileData(file_id) + if not file_bytes: + logger.warning(f"No data for file {file_id}") + continue + interface.createContextFile({ + "conversationId": conversation_id, + "fileId": file_id, + "fileName": file_info.get("fileName", "document"), + "mimeType": file_info.get("mimeType", "application/octet-stream"), + "fileSize": file_info.get("size", 0), + "uploadOrder": idx + }) + files_for_extraction.append({ + "fileId": file_id, + "bytes": file_bytes, + "mimeType": file_info.get("mimeType", "application/octet-stream"), + "fileName": file_info.get("fileName", "document") + }) + except Exception as e: + logger.error(f"Error loading file {file_id}: {e}", exc_info=True) + + if not files_for_extraction: + interface.updateConversation(conversation_id, {"status": "ready"}) + interface.createExtractedContext({ + "conversationId": conversation_id, + "textBlocks": [], + "summaries": [], + "extractionStatus": "failed", + "errors": ["No files could be loaded"] + }) + return interface.getConversation(conversation_id) + + result = run_extraction(files_for_extraction) + # Store sections in summaries for chat context (build_context_system_prompt uses sections) + summaries = result.get("summaries", []) or result.get("sections", []) + extracted = interface.createExtractedContext({ + "conversationId": conversation_id, + "textBlocks": result.get("textBlocks", []), + "summaries": summaries, + "extractionStatus": "completed", + "errors": result.get("errors", []), + "createdAt": getUtcTimestamp() + }) + interface.updateConversation(conversation_id, { + "status": "ready", + "extractedContextId": extracted.id, + "lastActivity": getUtcTimestamp() + }) + return interface.getConversation(conversation_id) + + +async def chatProcessV2( + currentUser: User, + mandateId: Optional[str], + userInput: UserInputRequest, + conversationId: Optional[str], + instanceId: str +) -> Optional[ChatbotV2Conversation]: + """ + Run chat with extracted context. Creates or resumes conversation. + """ + interface = getChatbotV2Interface(currentUser, mandateId=mandateId, featureInstanceId=instanceId) + event_manager = get_event_manager() + + config = await _load_config(instanceId) + base_prompt = config.systemPrompt if config else "You are a helpful assistant. Answer based on the provided context." + + if conversationId: + conv = interface.getConversation(conversationId) + if not conv: + raise ValueError(f"Conversation {conversationId} not found") + if conv.status == "extracting": + raise ValueError("Conversation not ready for chat (status: extracting). Wait for extraction to complete.") + # Reset stale "running" from previous failed/interrupted request + if conv.status == "running": + logger.info("Resetting stale conversation status from 'running' to 'ready'") + interface.updateConversation(conversationId, {"status": "ready"}) + new_round = conv.currentRound + 1 + interface.updateConversation(conversationId, { + "status": "running", + "currentRound": new_round, + "lastActivity": getUtcTimestamp() + }) + conv = interface.getConversation(conversationId) + if not event_manager.has_queue(conversationId): + event_manager.create_queue(conversationId) + else: + raise ValueError("conversationId is required for Chatbot V2 chat") + + extracted_ctx = interface.getExtractedContextByConversation(conversationId) + ctx_dict = {"textBlocks": [], "sections": []} + if extracted_ctx: + tb = extracted_ctx.textBlocks or [] + ctx_dict["textBlocks"] = tb + # summaries hold sections from extraction (fileName, text, blockCount) + ctx_dict["sections"] = extracted_ctx.summaries if isinstance(extracted_ctx.summaries, list) else [] + if not ctx_dict["sections"] and tb: + # Build sections from textBlocks if summaries empty + for doc in tb: + blocks = doc.get("blocks", []) + text_parts = [b.get("text", "") for b in blocks] + ctx_dict["sections"].append({ + "fileId": doc.get("fileId"), + "fileName": doc.get("fileName", "document"), + "text": "\n".join(text_parts), + "blockCount": len(blocks) + }) + + user_msg = userInput.prompt or "" + if not user_msg.strip(): + raise ValueError("Prompt is required") + + max_context_chars = config.maxContextChars if config else None + chunk_size = config.chunkSize if config else None + chunk_overlap = config.chunkOverlap if config else None + # Resolve to concrete values (chat node reads from configurable) + max_ctx = max_context_chars if max_context_chars and max_context_chars > 0 else 60_000 + cs = chunk_size if chunk_size and chunk_size > 0 else 15_000 + co = chunk_overlap if chunk_overlap is not None and chunk_overlap >= 0 else 500 + + allowed_providers = config.model.allowedProviders if config else [] + services = getServices(currentUser, mandateId=mandateId, featureInstanceId=instanceId) + if allowed_providers: + services.allowedProviders = allowed_providers # type: ignore[attr-defined] + + model = AICenterChatModel( + user=currentUser, + operation_type=OperationTypeEnum.DATA_ANALYSE, + processing_mode=ProcessingModeEnum.BASIC, + workflow_id=conversationId, + allowed_providers=allowed_providers if allowed_providers else None + ) + memory = ChatbotV2Checkpointer( + user=currentUser, + workflow_id=conversationId, + mandateId=mandateId, + featureInstanceId=instanceId + ) + app = create_chat_graph(model, memory) + + chatbotv2_context = { + "ctx_dict": ctx_dict, + "user_question": user_msg, + "base_prompt": base_prompt, + "max_context_chars": max_ctx, + "chunk_size": cs, + "chunk_overlap": co, + } + + interface.createMessage({ + "id": str(uuid.uuid4()), + "conversationId": conversationId, + "message": user_msg, + "role": "user", + "status": "first", + "sequenceNr": len(interface.getMessages(conversationId)) + 1, + "publishedAt": getUtcTimestamp(), + "roundNumber": conv.currentRound + }) + + try: + result = await app.ainvoke( + { + "messages": [{"role": "user", "content": user_msg}], + "chatbotv2_context": chatbotv2_context, + }, + config={"configurable": {"thread_id": conversationId}} + ) + # Assistant message is stored by ChatbotV2Checkpointer.put() - do NOT create again here. + # The stream sends chatData at start (from DB), so no need to emit chatdata event. + await event_manager.emit_event( + context_id=conversationId, + event_type="complete", + data={}, + event_category="workflow" + ) + finally: + clear_workflow_allowed_providers(conversationId) + + interface.updateConversation(conversationId, { + "status": "ready", + "lastActivity": getUtcTimestamp() + }) + return interface.getConversation(conversationId) diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 61e32886..34fd59cc 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -1206,14 +1206,20 @@ class ComponentObjects: return False def getFileData(self, fileId: str) -> Optional[bytes]: - """Returns the binary data of a file if user has access.""" - # Check file access + """Returns the binary data of a file if user has access. + + File access is user-scoped (same as routeDataFiles): getFile() verifies the user + owns the FileItem via _createdBy. Once ownership is confirmed, we read FileData + directly to ensure the uploader can always access their file content. + """ + # Check file access (user-scoped via _createdBy - same logic as routeDataFiles) file = self.getFile(fileId) if not file: logger.warning(f"No access to file ID {fileId}") return None - - fileDataEntries = getRecordsetWithRBAC(self.db, FileData, self.currentUser, recordFilter={"id": fileId}, mandateId=self.mandateId) + + # User owns the file - read FileData directly (bypass RBAC for owned files) + fileDataEntries = self.db.getRecordset(FileData, recordFilter={"id": fileId}) if not fileDataEntries: logger.warning(f"No data found for file ID {fileId}") return None diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index f52f0fde..a068abe8 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -61,6 +61,11 @@ TABLE_NAMESPACE = { "ChatStat": "chat", "ChatDocument": "chat", "Prompt": "chat", + # Chatbot (poweron_chatbot) - per feature-instance isolation + "ChatbotConversation": "chatbot", + "ChatbotMessage": "chatbot", + "ChatbotDocument": "chatbot", + "ChatbotLog": "chatbot", # Files - benutzer-eigen "FileItem": "files", "FileData": "files", @@ -70,7 +75,7 @@ TABLE_NAMESPACE = { } # Namespaces ohne Mandantenkontext - GROUP wird auf MY gemappt -USER_OWNED_NAMESPACES = {"chat", "files", "automation"} +USER_OWNED_NAMESPACES = {"chat", "chatbot", "files", "automation"} def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str: diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 75cf076a..18e26335 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -671,6 +671,11 @@ def updateFeatureInstance( detail="Failed to update feature instance" ) + # Clear chatbot config cache when config was updated for chatbot instances + if "config" in updateData and instance.featureCode == "chatbot": + from modules.features.chatbot.config import clear_config_cache + clear_config_cache(instanceId) + logger.info(f"User {context.user.id} updated feature instance {instanceId}: {updateData}") return updated.model_dump() diff --git a/modules/routes/routeSystem.py b/modules/routes/routeSystem.py index 5be43820..12867604 100644 --- a/modules/routes/routeSystem.py +++ b/modules/routes/routeSystem.py @@ -116,6 +116,11 @@ def _getFeatureUiObjects(featureCode: str) -> List[Dict[str, Any]]: return UI_OBJECTS elif featureCode == "neutralization": from modules.features.neutralization.mainNeutralization import UI_OBJECTS + elif featureCode == "chatbot": + from modules.features.chatbot.mainChatbot import UI_OBJECTS + return UI_OBJECTS + elif featureCode == "chatbotv2": + from modules.features.chatbotV2.mainChatbotV2 import UI_OBJECTS return UI_OBJECTS else: logger.warning(f"Unknown feature code: {featureCode}") diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 420c910d..9ee31a79 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -489,18 +489,20 @@ detectedIntent-Werte: balanceCheck = billingService.checkBalance(estimatedCost) if not balanceCheck.allowed: + balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" logger.warning( f"Billing check failed for user {user.id}: " - f"Balance {balanceCheck.currentBalance:.2f} CHF, " + f"Balance {balance_str} CHF, " f"Reason: {balanceCheck.reason}" ) raise InsufficientBalanceException( currentBalance=balanceCheck.currentBalance or 0.0, requiredAmount=estimatedCost, - message=f"Ungenugendes Guthaben. Aktuell: CHF {balanceCheck.currentBalance:.2f}" + message=f"Ungenugendes Guthaben. Aktuell: CHF {balance_str}" ) - - logger.debug(f"Billing check passed: Balance {balanceCheck.currentBalance:.2f} CHF") + + balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" + logger.debug(f"Billing check passed: Balance {balance_str} CHF") # Check if at least one provider is allowed (RBAC check) rbacAllowedProviders = billingService.getallowedProviders() diff --git a/modules/system/registry.py b/modules/system/registry.py index 1c32badd..3f037a51 100644 --- a/modules/system/registry.py +++ b/modules/system/registry.py @@ -36,6 +36,16 @@ def discoverFeatureContainers() -> List[str]: return sorted(containers) +def _router_load_order_key(filepath: str) -> tuple: + """ + Sort key for router loading. Longer/ Lexicographically later prefixes first + so that /api/chatbotv2 is registered before /api/chatbot (avoids prefix collision). + """ + featureDir = os.path.basename(os.path.dirname(filepath)) + # chatbotV2 before chatbot - use negative length then name so longer names sort first + return (-len(featureDir), featureDir) + + def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: """ Dynamically load and register routers from all discovered feature containers. @@ -48,8 +58,9 @@ def loadFeatureRouters(app: FastAPI) -> Dict[str, Any]: """ results = {} pattern = os.path.join(FEATURES_DIR, "*", "routeFeature*.py") - - for filepath in glob.glob(pattern): + filepaths = sorted(glob.glob(pattern), key=_router_load_order_key) + + for filepath in filepaths: featureDir = os.path.basename(os.path.dirname(filepath)) routerFile = os.path.basename(filepath)[:-3] # Remove .py diff --git a/scripts/script_db_export_migration.py b/scripts/script_db_export_migration.py index dd3cc940..9fc8a910 100644 --- a/scripts/script_db_export_migration.py +++ b/scripts/script_db_export_migration.py @@ -11,6 +11,7 @@ erstellt: _structure.json Datenbanken: - poweron_app (User, Mandate, RBAC, Features, etc.) - poweron_chat (Chat-Konversationen und Nachrichten) + - poweron_chatbot (Chatbot-Feature: Konversationen, Nachrichten, Logs) - poweron_management (Workflows, Prompts, Connections, etc.) - poweron_realestate (Real Estate Daten) - poweron_trustee (Trustee Daten) @@ -102,6 +103,7 @@ except Exception as e: ALL_DATABASES = [ "poweron_app", # Haupt-App: User, Mandate, RBAC, Features "poweron_chat", # Chat-Konversationen + "poweron_chatbot", # Chatbot-Feature: Konversationen, Nachrichten, Logs "poweron_management", # Workflows, Prompts, Connections "poweron_realestate", # Real Estate "poweron_trustee", # Trustee @@ -112,6 +114,7 @@ ALL_DATABASES = [ DATABASE_CONFIG = { "poweron_app": "DB_APP", # DB_APP_HOST, DB_APP_USER, DB_APP_PASSWORD_SECRET, etc. "poweron_chat": "DB_CHAT", # DB_CHAT_HOST, DB_CHAT_USER, etc. + "poweron_chatbot": "DB_CHATBOT", # DB_CHATBOT_* (fallsback to DB_*) "poweron_management": "DB_MANAGEMENT", "poweron_realestate": "DB_REALESTATE", "poweron_trustee": "DB_TRUSTEE", @@ -749,6 +752,7 @@ def main(): Datenbanken: poweron_app - User, Mandate, RBAC, Features poweron_chat - Chat-Konversationen + poweron_chatbot - Chatbot-Feature poweron_management - Workflows, Prompts, Connections poweron_realestate - Real Estate Daten poweron_trustee - Trustee Daten @@ -757,7 +761,7 @@ Beispiele: python script_db_export_migration.py python script_db_export_migration.py --pretty python script_db_export_migration.py -o backup.json --pretty - python script_db_export_migration.py --db poweron_app,poweron_chat + python script_db_export_migration.py --db poweron_app,poweron_chat,poweron_chatbot python script_db_export_migration.py --exclude Token,AuthEvent --include-meta python script_db_export_migration.py --summary """ diff --git a/scripts/script_db_init_chatbot.py b/scripts/script_db_init_chatbot.py new file mode 100644 index 00000000..907c3a4b --- /dev/null +++ b/scripts/script_db_init_chatbot.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Initialize poweron_chatbot database for the Chatbot feature. + +Creates the poweron_chatbot database if it does not exist. +Uses DB_CHATBOT_* config (falls back to DB_*). +Tables (ChatbotConversation, ChatbotMessage, ChatbotDocument, ChatbotLog) are +auto-created by the connector on first use. + +Usage: + python script_db_init_chatbot.py [--dry-run] +""" + +import os +import sys +import argparse +import logging +from pathlib import Path + +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +sys.path.insert(0, str(gatewayPath)) +os.chdir(str(gatewayPath)) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +import psycopg2 +from modules.shared.configuration import APP_CONFIG + +DB_NAME = "poweron_chatbot" +CONFIG_PREFIX = "DB_CHATBOT" + + +def _get_config(): + """Get DB config: DB_CHATBOT_* with fallback to DB_*.""" + host = APP_CONFIG.get(f"{CONFIG_PREFIX}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") + port = int(APP_CONFIG.get(f"{CONFIG_PREFIX}_PORT") or APP_CONFIG.get("DB_PORT", "5432")) + user = APP_CONFIG.get(f"{CONFIG_PREFIX}_USER") or APP_CONFIG.get("DB_USER") + password = ( + APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD_SECRET") + or APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD") + or APP_CONFIG.get("DB_PASSWORD_SECRET") + or APP_CONFIG.get("DB_PASSWORD") + ) + return {"host": host, "port": port, "user": user, "password": password} + + +def init_chatbot_db(dry_run: bool = False) -> bool: + """Create poweron_chatbot database if it does not exist.""" + config = _get_config() + if not config["user"] or not config["password"]: + logger.error("DB_USER and DB_PASSWORD (or DB_CHATBOT_*) required") + return False + + try: + conn = psycopg2.connect( + host=config["host"], + port=config["port"], + database="postgres", + user=config["user"], + password=config["password"], + ) + conn.autocommit = True + + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (DB_NAME,), + ) + exists = cur.fetchone() is not None + + if exists: + logger.info(f"Database {DB_NAME} already exists") + else: + if dry_run: + logger.info(f"[DRY-RUN] Would create database {DB_NAME}") + else: + cur.execute(f'CREATE DATABASE "{DB_NAME}"') + logger.info(f"Created database {DB_NAME}") + + conn.close() + return True + except Exception as e: + logger.error(f"Failed to init {DB_NAME}: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="Initialize poweron_chatbot database") + parser.add_argument("--dry-run", action="store_true", help="Do not create, only report") + args = parser.parse_args() + ok = init_chatbot_db(dry_run=args.dry_run) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/script_db_init_chatbotv2.py b/scripts/script_db_init_chatbotv2.py new file mode 100644 index 00000000..b8cda85f --- /dev/null +++ b/scripts/script_db_init_chatbotv2.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Initialize poweron_chatbotv2 database for the Chatbot V2 feature. + +Creates the poweron_chatbotv2 database if it does not exist. +Uses DB_CHATBOTV2_* config (falls back to DB_*). +Tables (ChatbotV2Conversation, ChatbotV2ContextFile, ChatbotV2ExtractedContext, +ChatbotV2Message, ChatbotV2Document, ChatbotV2Log) are auto-created by the +connector on first use. + +Usage: + python script_db_init_chatbotv2.py [--dry-run] +""" + +import os +import sys +import argparse +import logging +from pathlib import Path + +scriptPath = Path(__file__).resolve() +gatewayPath = scriptPath.parent.parent +sys.path.insert(0, str(gatewayPath)) +os.chdir(str(gatewayPath)) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +import psycopg2 +from modules.shared.configuration import APP_CONFIG + +DB_NAME = "poweron_chatbotv2" +CONFIG_PREFIX = "DB_CHATBOTV2" + + +def _get_config(): + """Get DB config: DB_CHATBOTV2_* with fallback to DB_*.""" + host = APP_CONFIG.get(f"{CONFIG_PREFIX}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") + port = int(APP_CONFIG.get(f"{CONFIG_PREFIX}_PORT") or APP_CONFIG.get("DB_PORT", "5432")) + user = APP_CONFIG.get(f"{CONFIG_PREFIX}_USER") or APP_CONFIG.get("DB_USER") + password = ( + APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD_SECRET") + or APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD") + or APP_CONFIG.get("DB_PASSWORD_SECRET") + or APP_CONFIG.get("DB_PASSWORD") + ) + return {"host": host, "port": port, "user": user, "password": password} + + +def init_chatbotv2_db(dry_run: bool = False) -> bool: + """Create poweron_chatbotv2 database if it does not exist.""" + config = _get_config() + if not config["user"] or not config["password"]: + logger.error("DB_USER and DB_PASSWORD (or DB_CHATBOTV2_*) required") + return False + + try: + conn = psycopg2.connect( + host=config["host"], + port=config["port"], + database="postgres", + user=config["user"], + password=config["password"], + ) + conn.autocommit = True + + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (DB_NAME,), + ) + exists = cur.fetchone() is not None + + if exists: + logger.info(f"Database {DB_NAME} already exists") + else: + if dry_run: + logger.info(f"[DRY-RUN] Would create database {DB_NAME}") + else: + cur.execute(f'CREATE DATABASE "{DB_NAME}"') + logger.info(f"Created database {DB_NAME}") + + conn.close() + return True + except Exception as e: + logger.error(f"Failed to init {DB_NAME}: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser(description="Initialize poweron_chatbotv2 database") + parser.add_argument("--dry-run", action="store_true", help="Do not create, only report") + args = parser.parse_args() + ok = init_chatbotv2_db(dry_run=args.dry_run) + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main()