From 24899b0cf2ac5cd0582ed958894582c19219036b Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 3 Jun 2026 09:34:28 +0200 Subject: [PATCH 1/3] fixes private model and udb scoping sources --- .../realEstate/{bzoExtractionLangGraph.py => bzoExtraction.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modules/features/realEstate/{bzoExtractionLangGraph.py => bzoExtraction.py} (100%) diff --git a/modules/features/realEstate/bzoExtractionLangGraph.py b/modules/features/realEstate/bzoExtraction.py similarity index 100% rename from modules/features/realEstate/bzoExtractionLangGraph.py rename to modules/features/realEstate/bzoExtraction.py From d61e29bcac2bed0c637d6c7a2049bca6a7a171af Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 3 Jun 2026 09:37:03 +0200 Subject: [PATCH 2/3] fixes private model and udb scoping sources --- app.py | 29 +- docs/althaus-bot-v2-aufwandsschaetzung.md | 309 ---- docs/althaus-bot-v2-fragenkatalog.md | 143 -- .../althaus-bot-v2-preprocessor-assessment.md | 223 --- env-dev.env | 2 +- env-int.env | 2 +- env-prod.env | 2 +- modules/aicore/aicoreModelRegistry.py | 4 +- modules/aicore/aicorePluginPrivateLlm.py | 59 +- modules/connectors/connectorDbPostgre.py | 14 +- .../providerGoogle/connectorGoogle.py | 317 +++- .../providerInfomaniak/connectorInfomaniak.py | 91 +- .../connectors/providerMsft/connectorMsft.py | 363 +++- modules/datamodels/datamodelChat.py | 5 - modules/datamodels/datamodelDataSource.py | 17 +- modules/datamodels/datamodelFeatures.py | 4 +- modules/datamodels/datamodelJson.py | 3 + modules/features/chatbot/__init__.py | 15 - modules/features/chatbot/bridges/__init__.py | 3 - modules/features/chatbot/bridges/ai.py | 727 -------- modules/features/chatbot/bridges/memory.py | 576 ------ modules/features/chatbot/bridges/tools.py | 390 ---- modules/features/chatbot/chatbot.py | 1301 ------------- modules/features/chatbot/chatbotConstants.py | 170 -- modules/features/chatbot/config.py | 302 --- .../chatbot/interfaceFeatureChatbot.py | 1618 ----------------- modules/features/chatbot/mainChatbot.py | 497 ----- .../features/chatbot/routeFeatureChatbot.py | 571 ------ modules/features/chatbot/service.py | 1525 ---------------- modules/features/commcoach/CONCEPT.md | 2 +- modules/features/realEstate/bzoExtraction.py | 51 +- modules/features/realEstate/mainRealEstate.py | 8 +- .../realEstate/routeFeatureRealEstate.py | 4 +- .../workspace/routeFeatureWorkspace.py | 5 +- modules/interfaces/interfaceFeatures.py | 2 +- modules/interfaces/interfaceRbac.py | 7 +- modules/routes/routeAdminFeatures.py | 9 +- modules/routes/routeAdminSttBenchmark.py | 217 --- modules/routes/routeDataSources.py | 15 +- modules/routes/routeRealEstate.py | 4 +- modules/routes/routeSystem.py | 3 - modules/routes/routeUdb.py | 17 +- modules/security/rbacHelpers.py | 2 +- .../core/serviceStreaming/__init__.py | 3 +- .../core/serviceStreaming/helpers.py | 242 --- .../serviceStreaming/mainServiceStreaming.py | 5 - .../coreTools/_dataSourceTools.py | 160 +- .../coreTools/_featureSubAgentTools.py | 25 +- .../serviceAgent/coreTools/_mediaTools.py | 150 ++ .../services/serviceAgent/datamodelAgent.py | 1 + .../services/serviceAgent/featureDataAgent.py | 24 +- .../serviceAgent/featureDataProvider.py | 204 ++- .../services/serviceAgent/toolboxRegistry.py | 16 + .../services/serviceAi/mainServiceAi.py | 47 +- .../serviceBilling/mainServiceBilling.py | 4 +- .../mainServiceGeneration.py | 16 +- .../serviceGeneration/paths/documentPath.py | 6 +- .../renderers/documentRendererBaseTemplate.py | 36 + .../renderers/rendererDocx.py | 100 +- .../renderers/rendererPdf.py | 126 +- .../serviceGeneration/styleDefaults.py | 110 +- .../serviceGeneration/subDocumentUtility.py | 75 + .../services/serviceKnowledge/_buildTree.py | 2 +- .../serviceKnowledge/_inheritFlags.py | 15 +- .../subConnectorIngestConsumer.py | 3 +- .../serviceKnowledge/subFeatureBootstrap.py | 23 +- .../services/serviceKnowledge/udbNodes.py | 19 +- modules/shared/gdprDeletion.py | 3 - modules/shared/i18nRegistry.py | 2 - modules/system/mainSystem.py | 10 - .../methodAi/actions/generateDocument.py | 4 +- .../workflows/methods/methodAi/methodAi.py | 15 +- .../methods/methodChatbot/__init__.py | 6 - .../methods/methodChatbot/actions/__init__.py | 3 - .../methodChatbot/actions/queryDatabase.py | 157 -- .../methods/methodChatbot/methodChatbot.py | 55 - requirements.txt | 11 +- scripts/script_db_export_migration.py | 6 +- scripts/script_db_init_chatbot.py | 101 - scripts/script_migrate_user_uid.py | 1 - tests/demo/README.md | 1 - tests/demo/test_demo_bootstrap.py | 13 - tests/demo/test_demo_uc3_chatbot.py | 39 - .../test_large_document_render.py | 154 ++ .../test_layout_primitives.py | 146 ++ .../serviceGeneration/test_style_resolver.py | 49 +- .../test_action_signature_validator.py | 1 - .../serviceAgent/test_field_neutralization.py | 149 ++ tests/unit/services/test_inheritFlags.py | 48 +- tests/unit/services/test_udbNodes.py | 6 +- 90 files changed, 2323 insertions(+), 9697 deletions(-) delete mode 100644 docs/althaus-bot-v2-aufwandsschaetzung.md delete mode 100644 docs/althaus-bot-v2-fragenkatalog.md delete mode 100644 docs/althaus-bot-v2-preprocessor-assessment.md delete mode 100644 modules/features/chatbot/__init__.py delete mode 100644 modules/features/chatbot/bridges/__init__.py delete mode 100644 modules/features/chatbot/bridges/ai.py delete mode 100644 modules/features/chatbot/bridges/memory.py delete mode 100644 modules/features/chatbot/bridges/tools.py delete mode 100644 modules/features/chatbot/chatbot.py delete mode 100644 modules/features/chatbot/chatbotConstants.py delete mode 100644 modules/features/chatbot/config.py delete mode 100644 modules/features/chatbot/interfaceFeatureChatbot.py delete mode 100644 modules/features/chatbot/mainChatbot.py delete mode 100644 modules/features/chatbot/routeFeatureChatbot.py delete mode 100644 modules/features/chatbot/service.py delete mode 100644 modules/routes/routeAdminSttBenchmark.py delete mode 100644 modules/serviceCenter/core/serviceStreaming/helpers.py delete mode 100644 modules/workflows/methods/methodChatbot/__init__.py delete mode 100644 modules/workflows/methods/methodChatbot/actions/__init__.py delete mode 100644 modules/workflows/methods/methodChatbot/actions/queryDatabase.py delete mode 100644 modules/workflows/methods/methodChatbot/methodChatbot.py delete mode 100644 scripts/script_db_init_chatbot.py delete mode 100644 tests/demo/test_demo_uc3_chatbot.py create mode 100644 tests/serviceGeneration/test_large_document_render.py create mode 100644 tests/serviceGeneration/test_layout_primitives.py create mode 100644 tests/unit/serviceAgent/test_field_neutralization.py diff --git a/app.py b/app.py index a69e9a7e..0ffa9824 100644 --- a/app.py +++ b/app.py @@ -282,7 +282,7 @@ initLogging() logger = logging.getLogger(__name__) instanceLabel = APP_CONFIG.get("APP_ENV_LABEL") -# Pre-warm AI connectors on process load (before lifespan). Critical for chatbot latency. +# Pre-warm AI connectors on process load (before lifespan). Critical for AI/agent latency. try: import modules.aicore.aicoreModelRegistry # noqa: F401 logger.info("AI connectors pre-warm (app load) triggered") @@ -428,10 +428,20 @@ async def lifespan(app: FastAPI): # --- Shutdown sequence (protected against CancelledError) --- try: - # 1. Stop scheduler first (removes all pending cron/interval jobs) + # 1. Signal DB layer to abort in-flight borrow waits immediately. + # This MUST happen first so that sync worker threads stuck in + # _acquireConn (30 s poll loop) bail out within one backoff tick + # instead of blocking process exit for the full borrow timeout. + try: + from modules.connectors.connectorDbPostgre import closeAllPools + closeAllPools() + except Exception as e: + logger.warning(f"Closing DB connection pools failed: {e}") + + # 2. Stop scheduler (removes all pending cron/interval jobs) eventManager.stop() - # 2. Stop Feature Containers (Plug&Play) + # 3. Stop Feature Containers (Plug&Play) try: mainModules = loadFeatureMainModules() for featureName, module in mainModules.items(): @@ -444,14 +454,6 @@ async def lifespan(app: FastAPI): except Exception as e: logger.warning(f"Could not shutdown feature containers: {e}") - # 3. Close all PostgreSQL connection pools (LAST -- features may still - # issue DB calls during their onStop hooks) - try: - from modules.connectors.connectorDbPostgre import closeAllPools - closeAllPools() - except Exception as e: - logger.warning(f"Closing DB connection pools failed: {e}") - logger.info("Application has been shut down") except asyncio.CancelledError: @@ -628,8 +630,7 @@ app.include_router(connectionsRouter) from modules.routes.routeRagInventory import router as ragInventoryRouter app.include_router(ragInventoryRouter) -from modules.routes.routeAdminSttBenchmark import router as sttBenchmarkRouter -app.include_router(sttBenchmarkRouter) + from modules.routes.routeTableViews import router as tableViewsRouter app.include_router(tableViewsRouter) @@ -745,4 +746,4 @@ if __name__ == "__main__": ], check=True) except ImportError: import uvicorn - uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=5) \ No newline at end of file + uvicorn.run("app:app", host="0.0.0.0", port=port, workers=1, timeout_graceful_shutdown=2) \ No newline at end of file diff --git a/docs/althaus-bot-v2-aufwandsschaetzung.md b/docs/althaus-bot-v2-aufwandsschaetzung.md deleted file mode 100644 index 0e0900d2..00000000 --- a/docs/althaus-bot-v2-aufwandsschaetzung.md +++ /dev/null @@ -1,309 +0,0 @@ -# Aufwandsschätzung Althaus Bot v2 -- Unabhängige Analyse - -**Projekt:** Althaus Bot v2 -- Weiterentwicklung & neue Use Cases -**Kunde:** W. Althaus AG, Aarwangen -**Erstellt:** 13. April 2026 -**Basis:** Code-Analyse Gateway-Repository + Offerte v2 vom 14.04.2026 -**Methodik:** Bottom-Up-Schätzung auf Basis der bestehenden Implementierung, Dreipunktschätzung (Min / Mitte / Max) - ---- - -## 1. Ist-Zustand der Implementierung - -### 1.1 Architekturübersicht - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ React Frontend (SSE-Streaming, Chat-UI) │ -└──────────────────────────┬──────────────────────────────────────┘ - │ /api/chatbot/* -┌──────────────────────────▼──────────────────────────────────────┐ -│ Gateway (Python/FastAPI) │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ Chatbot Feature (modules/features/chatbot/) │ │ -│ │ ┌─────────┐ ┌──────────┐ ┌──────────┐ ┌────────┐ │ │ -│ │ │ Planner │→ │ SQL Plan │→ │ Parse & │→ │Formul. │ │ │ -│ │ │ Node │ │ Node │ │ Execute │ │ Node │ │ │ -│ │ └────┬────┘ └──────────┘ └────┬─────┘ └────────┘ │ │ -│ │ │ │ │ │ -│ │ ├→ Tavily (Web Search) │ │ │ -│ │ └→ Direct Answer │ │ │ -│ └──────────────────────────────────┼──────────────────────┘ │ -│ │ │ -│ ┌──────────────────────────────────▼──────────────────────┐ │ -│ │ PreprocessorConnector (HTTP POST → Azure SQL API) │ │ -│ └─────────────────────────────────────────────────────────┘ │ -│ │ -│ ┌─────────────────────────────────────────────────────────┐ │ -│ │ KnowledgeService (pgvector/RAG) -- NICHT IM CHATBOT │ │ -│ │ Produktiv im AgentService + CommCoach │ │ -│ └─────────────────────────────────────────────────────────┘ │ -└──────────────────────────────────────────────────────────────────┘ - │ -┌──────────────────────────▼──────────────────────────────────────┐ -│ Azure Preprocessing Server (deployed, ERP-Daten deaktiviert) │ -│ Tabellen: Artikel, Einkaufspreis, Lagerplatz, Lagerplatz_Art. │ -│ Repo: github.com/valueonag/gateway_preprocessing │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### 1.2 Vorhandene Komponenten (Wiederverwendung) - -| Komponente | Datei / Modul | Status | Wiederverwendbar für | -|---|---|---|---| -| LangGraph-Workflow | `chatbot/chatbot.py` | Produktiv (deaktiviert) | Alle Positionen -- Grundgerüst | -| PreprocessorConnector | `connectors/connectorPreprocessor.py` | Produktiv (deaktiviert) | Pos. 1, 2, 3, 4 -- SQL-Abfragen | -| ChatbotConfig | `chatbot/config.py` | Produktiv | Alle -- Konfiguration pro Instanz | -| Streaming-Bridge | `chatbot/service.py` | Produktiv | Alle -- SSE ans Frontend | -| ChatbotDocument | `chatbot/interfaceFeatureChatbot.py` | Implementiert | Pos. 1.4, 2.1, 2.5 -- File-Handling | -| KnowledgeService/RAG | `serviceCenter/services/serviceKnowledge/` | Produktiv (AgentService) | Pos. 5 -- Wiki-Integration | -| Automation-Template | `automation/subAutomationTemplates.py` | Produktiv | Pos. 6 -- Preprocessor-Updates | -| SQL-Sanitize | `chatbot.py` → `_sanitize_sql_typos` | Produktiv | Pos. 1.1 -- Gesperrte Artikel | -| Markdown-Tabellen | `chatbot.py` → `_tool_output_to_markdown_table` | Produktiv | Pos. 1.3, 3.3 -- Darstellung | -| File-Upload Backend | `service.py` → `_convert_file_ids_to_document_references` | Implementiert | Pos. 1.4 -- Upload-Pipeline | -| Excel-Export | `service.py` → `_create_chat_document_from_action_document` | Implementiert | Pos. 2.5 -- Kalktool-Export | - -### 1.3 Fehlende Komponenten (Neuentwicklung) - -| Komponente | Benötigt für | Komplexität | -|---|---|---| -| Matching-Engine (exakt → fuzzy → KI) | Pos. 2.2 | Hoch | -| Neuer Planner-Pfad "WIKI" | Pos. 5.2 | Mittel | -| KnowledgeService → Chatbot Integration | Pos. 5.2 | Mittel | -| Wiki-Connector (API/Crawling) | Pos. 5.1 | Unbekannt (Wiki-abhängig) | -| Delta-Sync-Mechanismus | Pos. 5.3 | Mittel | -| Preprocessor: 8-10 neue Tabellen/Views | Pos. 1.5, 3.1, 4.1 | Mittel (Code-Änderung) | -| Frontend: File-Picker, Drag&Drop | Pos. 1.4 | Mittel | -| Frontend: Thread-Liste, Suchfunktion | Pos. 1.2 | Mittel | -| Kalktool-Excel-Format-Export | Pos. 2.5 | Mittel | -| Schwellenwert-Insights | Pos. 4.5 | Mittel | - ---- - -## 2. Detaillierte Aufwandsschätzung - -### Position 1: Basics (Plattform-Verbesserungen) - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 1.1 | Gesperrte Artikel filtern | 4 | 3 | 4 | 4 | System-Prompt + SQL-Sanitize-Regel. Kleine Änderung. | -| 1.2 | Chat-Verlauf speichern | 12 | 12 | 14 | 16 | Backend existiert. Frontend-Aufwand (Thread-Liste, Suche). | -| 1.3 | Längere Antworten | 6 | 4 | 5 | 6 | Streaming-Config + Frontend-Rendering. | -| 1.4 | Datei-Upload | 16 | 16 | 18 | 20 | Full-Stack: Drag&Drop + LangGraph-Integration + Extraktion. | -| 1.5 | Kundenartikelnummern | 8 | 10 | 12 | 14 | Preprocessor-Code + Prompt + Cross-Ref-Queries. ERP-abhängig. | -| 1.6 | Abklärungen & Testing | 8 | 8 | 8 | 8 | Standard. | -| | **Subtotal** | **54** | **53** | **61** | **68** | | - -**Delta zur Offerte: +7h (Mitte) / +14h (Max)** -**Haupttreiber:** Preprocessor-Erweiterung für Kundenartikelnummern (Pos. 1.5) erfordert Code-Änderung, nicht nur Config. Frontend-Aufwand bei Upload (Pos. 1.4) eher am oberen Ende. - ---- - -### Position 2: Use Case Kalktool - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 2.1 | Stücklisten-Upload & Extraktion | 12 | 10 | 12 | 14 | Nutzt Pos. 1.4. serviceExtraction vorhanden. | -| 2.2 | Artikelidentifikation & Matching | 20 | 24 | 28 | 32 | **KRITISCH**: Neue Matching-Engine, 3 Stufen, ERP-abhängig. | -| 2.3 | Automatische Feldergänzung | 16 | 14 | 16 | 18 | Preprocessor + Enrichment-Logik. | -| 2.4 | Alternativartikel-Vorschläge | 12 | 12 | 14 | 16 | KI-Vorschläge + Bestätigungs-Workflow im Chat. | -| 2.5 | Excel-Export (Kalktool-Format) | 12 | 10 | 12 | 14 | Basis existiert. Kalktool-Vorlage-Anpassung. | -| 2.6 | Erweiterbarkeit neue Felder | 8 | 6 | 8 | 10 | Config-gesteuertes Feld-Mapping. | -| 2.7 | Abklärungen & Testing | 12 | 12 | 12 | 12 | Kalktool-Vorlage, Testdaten, UAT. | -| | **Subtotal** | **92** | **88** | **102** | **116** | | - -**Delta zur Offerte: +10h (Mitte) / +24h (Max)** -**Haupttreiber:** Die Matching-Engine (Pos. 2.2) ist die komplexeste Neuentwicklung im gesamten Projekt. Mehrstufiges Matching (exakt → fuzzy → KI-gestützt) ohne bestehende Basis. Die Qualität hängt stark von der ERP-Datenqualität und der Vielfalt der Kunden-Stücklisten-Formate ab. - ---- - -### Position 3: Use Case Materialmanagement 1 - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 3.1 | ERP-Daten erweitern | 16 | 16 | 19 | 22 | Preprocessor: Bestellungen, Wareneingänge, Aufträge. Code nötig. | -| 3.2 | System-Prompt Materialmanagement | 8 | 6 | 8 | 10 | Prompt-Engineering + SQL-Templates. | -| 3.3 | Transparente Statusübersicht | 8 | 6 | 7 | 8 | Markdown-Rendering existiert, Erweiterung nötig. | -| 3.4 | Auswirkungsanalyse & Empfehlungen | 12 | 14 | 16 | 18 | Cross-Table-Queries + KI-Analyse. Komplex. | -| 3.5 | Abklärungen & Testing | 8 | 8 | 8 | 8 | Standard. | -| | **Subtotal** | **52** | **50** | **58** | **66** | | - -**Delta zur Offerte: +6h (Mitte) / +14h (Max)** -**Haupttreiber:** Auswirkungsanalyse (Pos. 3.4) erfordert Multi-Table-Joins und KI-gestützte Bewertung, was über einfache SQL-Abfragen hinausgeht. - ---- - -### Position 4: Use Case Materialmanagement 2 (KPIs) - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 4.1 | ERP-Daten erweitern | 16 | 16 | 19 | 22 | Lagerjournal, Preishistorie. Aggregierte Views. | -| 4.2 | System-Prompt KPI-Analyse | 8 | 6 | 8 | 10 | Prompt-Engineering. | -| 4.3 | Liefertermintreue-Analyse | 10 | 10 | 12 | 14 | Zeitreihen, Lieferantenvergleich, komplexe SQL. | -| 4.4 | Preisentwicklungs-Analyse | 10 | 10 | 11 | 12 | Preishistorie, Abweichungsberechnung. | -| 4.5 | Automatisierte Insights | 8 | 10 | 12 | 14 | Schwellenwert-Warnungen, proaktive Erkennung. Neues Konzept. | -| 4.6 | Abklärungen & Testing | 8 | 8 | 8 | 8 | Standard. | -| | **Subtotal** | **60** | **60** | **70** | **80** | | - -**Delta zur Offerte: +10h (Mitte) / +20h (Max)** -**Haupttreiber:** Automatisierte Insights (Pos. 4.5) erfordern eine neue Logikschicht, die proaktiv Schwellenwerte überwacht und Empfehlungen generiert. Das ist im aktuellen Chat-Flow nicht vorgesehen. - ---- - -### Position 5: Use Case Wiki-Anbindung - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 5.1 | Wiki-Anbindung & Indexierung | 16 | 16 | 20 | 24 | KnowledgeService existiert. Wiki-Zugang UNBEKANNT. | -| 5.2 | RAG-Integration im Chatbot | 12 | 12 | 14 | 16 | Pattern existiert (AgentService), muss portiert werden. | -| 5.3 | Inkrementelle Aktualisierung | 8 | 8 | 11 | 14 | Delta-Sync stark Wiki-abhängig. | -| 5.4 | Abklärungen & Testing | 8 | 8 | 9 | 10 | Relevanz-Tuning ist iterativ. | -| | **Subtotal** | **44** | **44** | **54** | **64** | | - -**Delta zur Offerte: +10h (Mitte) / +20h (Max)** -**Haupttreiber:** Wiki-System ist unbekannt. Bei Wiki mit guter API (Confluence, SharePoint) sind 44h erreichbar. Bei proprietärem System ohne API steigt der Aufwand erheblich. - -**Synergie:** KnowledgeService mit pgvector, Chunking, Embedding und semanticSearch ist bereits produktiv. Die RAG-Pipeline (Ingestion → Embedding → Retrieval) muss nicht neu gebaut werden. Das spart geschätzt 20-30h gegenüber einer Neuentwicklung. - ---- - -### Position 6: Azure-Migration - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 6.1 | Migration Preprocessor | 6 | 4 | 6 | 8 | Config-Änderungen, Env-Files, Netzwerk. | -| 6.2 | Validierung & Smoke-Tests | 4 | 4 | 4 | 4 | End-to-End-Tests. | -| | **Subtotal** | **10** | **8** | **10** | **12** | | - -**Delta zur Offerte: 0h (Mitte)** -**Bewertung:** Realistisch. Einfachste Position. - ---- - -### Position 7: Projektmanagement - -| # | Anforderung | Offerte | Min | Mitte | Max | Begründung | -|---|---|:-:|:-:|:-:|:-:|---| -| 7.1 | Kick-off & Workshop | 4 | 4 | 4 | 4 | Standard. | -| 7.2 | Projektmanagement | 8 | 10 | 12 | 14 | 10-14 Wochen, 3 Ansprechpartner, 7 Positionen. | -| 7.3 | Deployment & Go-Live | 6 | 6 | 7 | 8 | Staging + Prod + erste Betriebswoche. | -| | **Subtotal** | **18** | **20** | **23** | **26** | | - -**Delta zur Offerte: +5h (Mitte) / +8h (Max)** -**Haupttreiber:** PM-Aufwand bei 3-Monats-Projekt mit mehreren Stakeholdern ist erfahrungsgemäss höher. - ---- - -## 3. Gesamtübersicht - -| Pos. | Beschreibung | Offerte (h) | Min (h) | Mitte (h) | Max (h) | Offerte CHF | Mitte CHF | -|---|---|:-:|:-:|:-:|:-:|:-:|:-:| -| 1 | Basics | 54 | 53 | 61 | 68 | 8'100 | 9'150 | -| 2 | Kalktool | 92 | 88 | 102 | 116 | 13'800 | 15'300 | -| 3 | Materialmanagement 1 | 52 | 50 | 58 | 66 | 7'800 | 8'700 | -| 4 | Materialmanagement 2 | 60 | 60 | 70 | 80 | 9'000 | 10'500 | -| 5 | Wiki-Anbindung | 44 | 44 | 54 | 64 | 6'600 | 8'100 | -| 6 | Azure-Migration | 10 | 8 | 10 | 12 | 1'500 | 1'500 | -| 7 | Projektmanagement | 18 | 20 | 23 | 26 | 2'700 | 3'450 | -| | **Gesamt** | **330** | **323** | **378** | **432** | **49'500** | **56'700** | - -### Zusammenfassung - -| Szenario | Stunden | CHF (à 150/h) | Differenz zur Offerte | -|---|:-:|:-:|:-:| -| Offerte (Kostendach) | 330 | 49'500 | -- | -| Eigene Schätzung (Minimum) | 323 | 48'450 | -2% | -| **Eigene Schätzung (Mitte)** | **378** | **56'700** | **+15%** | -| Eigene Schätzung (Maximum) | 432 | 64'800 | +31% | - ---- - -## 4. Risikobewertung - -### Risikomatrix - -| # | Risiko | Wahrscheinlichkeit | Auswirkung | Betroffene Pos. | Möglicher Mehraufwand | -|---|---|:-:|:-:|---|:-:| -| R1 | Matching-Engine komplexer als erwartet | Hoch | Hoch | 2.2 | +10-15h | -| R2 | Wiki-System ohne API | Mittel | Hoch | 5.1, 5.3 | +10-20h | -| R3 | ERP-Datenqualität mangelhaft | Mittel | Mittel | 1.5, 2.2, 3.1, 4.1 | +8-16h | -| R4 | Preprocessor-Erweiterung aufwändiger | Mittel | Mittel | 1.5, 3.1, 4.1 | +8-12h | -| R5 | Frontend-Aufwand unterschätzt | Mittel | Gering | 1.2, 1.4 | +4-8h | -| R6 | KI-Modell-Qualität für SQL-Generierung | Gering | Mittel | 3, 4 | +4-8h | - -### Synergien (Aufwandsreduktion durch bestehende Komponenten) - -| Synergie | Geschätzte Einsparung | Betroffene Pos. | -|---|:-:|---| -| KnowledgeService/RAG existiert produktiv | 20-30h | Pos. 5 | -| ChatbotDocument-Modell existiert | 4-6h | Pos. 1.4, 2.1 | -| LangGraph modular erweiterbar | 6-10h | Pos. 3, 4, 5 | -| Prompt-Engineering über DB-Config | 2-4h | Pos. 1.1, 3.2, 4.2 | -| Excel-Export-Pattern existiert | 2-4h | Pos. 2.5 | -| **Gesamt Einsparung** | **34-54h** | | - ---- - -## 5. Empfehlungen - -### 5.1 Zur Offerte - -Die Offerte mit 330h als Kostendach ist **ambitioniert, aber bei idealem Verlauf erreichbar**. Die grössten Risiken liegen in: -- Position 2 (Kalktool): Die Matching-Engine ist die komplexeste Neuentwicklung -- Position 5 (Wiki): Komplett abhängig vom Wiki-System, das noch unklärt ist - -**Empfehlung:** Offerte bei 330h als Kostendach belassen, aber intern mit 370-380h planen. Die Differenz (~40-50h) als interne Reserve einkalkulieren. - -### 5.2 Priorisierung - -1. **Must-Have (Prio 1):** Pos. 1 (Basics) + Pos. 6 (Azure-Migration) -- Voraussetzung für alles -2. **High-Value (Prio 2):** Pos. 2 (Kalktool) -- Höchster Kundennutzen, aber auch höchstes Risiko -3. **Quick-Win (Prio 3):** Pos. 3+4 (Materialmanagement) -- Nutzen vorhandene Architektur -4. **Abhängig (Prio 4):** Pos. 5 (Wiki) -- Erst nach Wiki-Klärung starten - -### 5.3 Offene Punkte (vor Projektstart zu klären) - -| # | Offener Punkt | Verantwortlich | Kritisch für | -|---|---|---|---| -| O1 | Wiki-System und Zugangsart klären | Althaus (Samuel) | Pos. 5 | -| O2 | ERP-System identifizieren und Datenstrukturen dokumentieren | Althaus (Stefan) | Pos. 1.5, 3.1, 4.1 | -| O3 | Preprocessor-Code-Review für Erweiterbarkeit | PowerOn (Entwicklung) | Pos. 1.5, 3.1, 4.1 | -| O4 | Kalktool-Vorlage erhalten und analysieren | Althaus (Reto) | Pos. 2.5 | -| O5 | Muster-Stücklisten für Matching-Test | Althaus (Reto) | Pos. 2.2 | -| O6 | Azure-Subscription-Details | Althaus | Pos. 6 | - ---- - -## 6. Zeitplan (2 Entwickler) - -``` -Woche 1-2: Kick-off + Azure-Migration (Pos. 6) + Basics 1.1-1.3 - Entwickler A: Azure-Migration + 1.1 (Gesperrte Artikel) - Entwickler B: 1.2 (Chat-Verlauf Frontend) + 1.3 (Lange Antworten) - -Woche 2-5: Basics 1.4-1.6 (Grundlage für Use Cases) - Entwickler A: 1.4 (File-Upload Full-Stack) - Entwickler B: 1.5 (Kundenartikelnummern + Preprocessor) - -Woche 4-9: Kalktool (Pos. 2) -- längster Block, früh starten - Entwickler A: 2.1-2.2 (Upload + Matching-Engine) - Entwickler B: 2.3-2.5 (Feldergänzung + Export) - -Woche 6-9: Materialmanagement 1+2 (Pos. 3+4) -- parallel zum Kalktool - Entwickler B: 3.1-3.4 + 4.1-4.5 (Preprocessor + Prompts) - (Entwickler A bleibt auf Kalktool) - -Woche 9-12: Wiki-Anbindung (Pos. 5) -- nach Klärung des Wiki-Systems - Entwickler A: 5.1-5.2 (Connector + RAG-Integration) - Entwickler B: 5.3 (Delta-Sync) + Integrationstests - -Woche 12-13: Integrationstests, UAT, Go-Live (Pos. 7.3) - Beide Entwickler: E2E-Tests + Deployment + Monitoring -``` - -**Gesamtdauer:** 12-14 Wochen -**Kritischer Pfad:** Pos. 1 → Pos. 2 (Kalktool braucht Upload + Kundenartikelnummern) - ---- - -*Dokument erstellt auf Basis der Code-Analyse des Gateway-Repository (Stand 13.04.2026)* diff --git a/docs/althaus-bot-v2-fragenkatalog.md b/docs/althaus-bot-v2-fragenkatalog.md deleted file mode 100644 index a22910f6..00000000 --- a/docs/althaus-bot-v2-fragenkatalog.md +++ /dev/null @@ -1,143 +0,0 @@ -# Fragenkatalog Althaus Bot v2 -- Kick-off-Vorbereitung - -**Zweck:** Strukturierte Fragen für den Anforderungsworkshop mit W. Althaus AG -**Erstellt:** 13. April 2026 -**Zielgruppe:** Projektleitung PowerOn + Ansprechpartner Althaus (Reto, Stefan, Samuel) - ---- - -## A. Wiki-System (Ansprechpartner: Samuel) - -> **Kritisch für:** Position 5 (Wiki-Anbindung) -- Aufwandsschätzung schwankt zwischen 44h und 64h je nach Wiki-System. - -### A.1 Wiki-Identifikation - -| # | Frage | Hintergrund | -|---|---|---| -| A1.1 | Welches Wiki-System wird eingesetzt? (z.B. Confluence, SharePoint Wiki, MediaWiki, DokuWiki, Notion, anderes) | Bestimmt die Anbindungsstrategie (API vs. Export vs. Crawling) | -| A1.2 | Wo wird das Wiki gehostet? (Cloud-SaaS, On-Premise, Azure) | Netzwerk-Zugang und Firewall-Konfiguration | -| A1.3 | Wie viele Seiten/Artikel enthält das Wiki ungefähr? | Dimensionierung der Erstindexierung und Embedding-Kosten | -| A1.4 | In welchen Formaten liegen die Inhalte vor? (reiner Text, HTML, Markdown, eingebettete PDFs/Bilder) | Bestimmt die Extraktions-Komplexität | - -### A.2 Technischer Zugang - -| # | Frage | Hintergrund | -|---|---|---| -| A2.1 | Gibt es eine REST-API oder ähnliche Schnittstelle zum Lesen der Wiki-Inhalte? | API-Zugang = deutlich weniger Aufwand als Crawling | -| A2.2 | Gibt es eine Export-Funktion? (z.B. XML-Export, PDF-Export, Datenbank-Dump) | Fallback wenn keine API vorhanden | -| A2.3 | Gibt es Authentifizierung (API-Key, OAuth, LDAP)? Welche Credentials werden benötigt? | Konfiguration des Connectors | -| A2.4 | Gibt es eine Change-API oder Webhooks, die bei Änderungen notifizieren? | Bestimmt den Aufwand für inkrementelle Updates (Pos. 5.3) | -| A2.5 | Gibt es Zugriffsbeschränkungen auf bestimmte Wiki-Bereiche? | RBAC-Überlegungen bei der Indexierung | - -### A.3 Inhaltliche Abgrenzung - -| # | Frage | Hintergrund | -|---|---|---| -| A3.1 | Soll das gesamte Wiki indexiert werden oder nur bestimmte Bereiche? | Scope-Begrenzung für Erstindexierung | -| A3.2 | Gibt es vertrauliche Inhalte, die nicht in den Chatbot einfliessen dürfen? | Datenschutz-/Compliance-Anforderung | -| A3.3 | Wie oft werden Wiki-Inhalte aktualisiert? (täglich, wöchentlich, selten) | Bestimmt die Sync-Frequenz | -| A3.4 | Welche Sprache(n) haben die Wiki-Inhalte? (Deutsch, Englisch, gemischt) | Embedding-Modell-Auswahl | - ---- - -## B. ERP-System & Datenstrukturen (Ansprechpartner: Stefan) - -> **Kritisch für:** Positionen 1.5, 2.2-2.3, 3.1, 4.1 -- Preprocessor-Erweiterungen und Matching-Engine. - -### B.1 ERP-Identifikation - -| # | Frage | Hintergrund | -|---|---|---| -| B1.1 | Welches ERP-System wird eingesetzt? (z.B. Abacus, SAP, Microsoft Dynamics, bexio, Sage) | Bestimmt Datenstruktur und Zugriffsmöglichkeiten | -| B1.2 | Wie werden die Daten aktuell an den Preprocessor geliefert? (direkter DB-Zugriff, API, Export-Datei) | Verständnis der bestehenden Datenpipeline | -| B1.3 | In welchem Rhythmus werden die Daten aktualisiert? (Echtzeit, täglich, wöchentlich) | Aktualität der Chatbot-Antworten | - -### B.2 Kundenartikelnummern (Position 1.5) - -| # | Frage | Hintergrund | -|---|---|---| -| B2.1 | Gibt es im ERP eine dedizierte Tabelle für Kundenartikelnummern? Wenn ja, wie heisst sie? | Preprocessor-Schema-Erweiterung | -| B2.2 | Wie ist die Zuordnung: 1 Kundenartikel → 1 ERP-Artikel, oder n:m? | Bestimmt die Mapping-Komplexität | -| B2.3 | Wie viele Kundenartikelnummern gibt es ungefähr? | Dimensionierung | -| B2.4 | Welche Felder hat die Kundenartikelnummern-Tabelle? (z.B. KundenNr, KundenArtikelNr, InterneArtikelNr, Bezeichnung) | Schema-Definition für Preprocessor | - -### B.3 Bestellwesen & Materialmanagement (Positionen 3 + 4) - -| # | Frage | Hintergrund | -|---|---|---| -| B3.1 | Welche ERP-Tabellen/Views gibt es für Bestellungen? (Bestellkopf, Bestellpositionen, Status) | Preprocessor-Erweiterung Pos. 3.1 | -| B3.2 | Gibt es eine Tabelle für Wareneingänge mit Datum und Menge? | Liefertermin-Treue-Berechnung Pos. 4.3 | -| B3.3 | Gibt es eine Preishistorie-Tabelle? Welche Felder enthält sie? (Datum, Preis, Lieferant, Währung) | Preisentwicklungs-Analyse Pos. 4.4 | -| B3.4 | Gibt es ein Lagerjournal mit Buchungsdaten? | KPI-Analyse Pos. 4.1 | -| B3.5 | Gibt es eine Bestandesbedarfsliste oder Dispositions-View? | Material-Analyse Pos. 3.4 | -| B3.6 | Gibt es Felder für "bestätigter Liefertermin" vs. "gewünschter Liefertermin"? | Termintreue-KPI Pos. 4.3 | -| B3.7 | Wie viele offene Bestellungen gibt es typischerweise gleichzeitig? | Performance-Dimensionierung | - -### B.4 Datenqualität - -| # | Frage | Hintergrund | -|---|---|---| -| B4.1 | Wie konsistent sind Lieferanten-Namen im ERP? (exakt gleich oder Varianten wie "Siemens AG" vs. "Siemens") | Matching-Qualität Pos. 2.2 | -| B4.2 | Gibt es Pflichtfelder die häufig leer sind? | Feldergänzungs-Logik Pos. 2.3 | -| B4.3 | Wie sind Preise gespeichert? (Netto, Brutto, mit/ohne MwSt., Währung) | SQL-Query-Generierung | -| B4.4 | Werden gelöschte/gesperrte Datensätze physisch oder nur logisch gelöscht? | Filter-Logik Pos. 1.1 | - ---- - -## C. Kalktool (Ansprechpartner: Reto) - -> **Kritisch für:** Position 2 (Kalktool) -- Höchstes Risiko in der Offerte. - -### C.1 Kalktool-Vorlage - -| # | Frage | Hintergrund | -|---|---|---| -| C1.1 | Können wir die aktuelle Kalktool-Vorlage (Kalktool_Aktuell_2026_V1.4.xlsx) erhalten? | Zielformat für Excel-Export Pos. 2.5 | -| C1.2 | Welche Spalten/Felder sind Pflicht in der Kalktool-Vorlage? | Feldergänzungs-Priorität Pos. 2.3 | -| C1.3 | Gibt es Formeln in der Vorlage, die erhalten bleiben müssen? | Komplexität des Excel-Exports | -| C1.4 | Welches Format haben die Kunden-Stücklisten typischerweise? (PDF, Excel, CSV) | Extraktions-Strategie Pos. 2.1 | - -### C.2 Matching-Anforderungen - -| # | Frage | Hintergrund | -|---|---|---| -| C2.1 | Können wir 3-5 Muster-Stücklisten von verschiedenen Kunden erhalten? | Testdaten für Matching-Engine Pos. 2.2 | -| C2.2 | Welche Identifikationsmerkmale haben Kunden-Stücklisten? (Kundenartikelnr., Hersteller-Typ, Beschreibung) | Matching-Stufen definieren | -| C2.3 | Wie hoch ist die erwartete Trefferquote beim exakten Match? (10%? 50%? 90%?) | Gewichtung exakt vs. fuzzy vs. KI | -| C2.4 | Welche Felder sollen bei nicht-eindeutigem Match als "Alternative durch KI" markiert werden? | Bestätigungs-Workflow Pos. 2.4 | -| C2.5 | Gibt es Produktgruppen, die besonders schwierig zu matchen sind? | Risikobewertung | - ---- - -## D. Infrastruktur & Azure (Ansprechpartner: Stefan / IT) - -| # | Frage | Hintergrund | -|---|---|---| -| D1 | Details zur neuen Azure-Subscription (Subscription-ID, Region, Resource Group) | Pos. 6 -- Migration | -| D2 | Gibt es Netzwerk-Einschränkungen (VPN, Private Endpoints, Firewall)? | Zugang Preprocessor ↔ ERP | -| D3 | Wer hat Admin-Zugang zur neuen Subscription? | Deployment-Planung | -| D4 | Gibt es Budget-Limits auf der Azure-Subscription? | Betriebskosten-Planung | - ---- - -## E. Priorisierung & Vorgehensweise - -| # | Frage | Hintergrund | -|---|---|---| -| E1 | Sollen alle 7 Positionen umgesetzt werden, oder gibt es eine Priorisierung? | Scope-Bestätigung | -| E2 | Gibt es einen gewünschten Go-Live-Termin? | Zeitplanung | -| E3 | Wie soll die UAT organisiert werden? (dedizierte Testphase, laufend, Key-User) | Testplanung | -| E4 | Wer sind die Pilot-User für den reaktivierten Bot? | UAT-Teilnehmer | -| E5 | Sollen Schulungen für Endanwender durchgeführt werden? (nicht in Offerte enthalten) | Ggf. Nachtragsofferte | - ---- - -## Nächste Schritte - -1. **Vor dem Kick-off:** Fragenkatalog an Althaus senden, damit Antworten vorbereitet werden können -2. **Im Kick-off:** Fragen durchgehen, fehlende Antworten als Action Items festhalten -3. **Nach dem Kick-off:** Aufwandsschätzung anhand der Antworten finalisieren, insbesondere Pos. 2.2 (Matching) und Pos. 5 (Wiki) - ---- - -*PowerOn AG -- Vorbereitung Anforderungsworkshop Althaus Bot v2* diff --git a/docs/althaus-bot-v2-preprocessor-assessment.md b/docs/althaus-bot-v2-preprocessor-assessment.md deleted file mode 100644 index c8fb3848..00000000 --- a/docs/althaus-bot-v2-preprocessor-assessment.md +++ /dev/null @@ -1,223 +0,0 @@ -# Preprocessor Assessment -- Althaus Bot v2 - -**Zweck:** Technische Analyse des Preprocessing-Servers für die Aufwandsschätzung der Erweiterungen -**Erstellt:** 13. April 2026 -**Quellen:** Gateway-Code-Analyse (Repo nicht lokal verfügbar: github.com/valueonag/gateway_preprocessing) - ---- - -## 1. Ist-Zustand (abgeleitet aus Gateway-Code) - -### 1.1 Infrastruktur - -| Eigenschaft | Wert | -|---|---| -| **Host** | Azure App Service (Switzerland North) | -| **URL (Datenverarbeitung)** | `poweron-althaus-preprocess-prod-*.azurewebsites.net/api/v1/dataprocessor/update-db-with-config` | -| **URL (Abfragen)** | `poweron-althaus-preprocess-prod-*.azurewebsites.net/api/v1/dataquery/query` | -| **Authentifizierung** | `X-PP-API-Key` (Abfragen) / `X-DB-API-Key` (Abfragen) | -| **Status** | Deployed, ERP-Datenanbindung deaktiviert | -| **Quellcode** | `github.com/valueonag/gateway_preprocessing` (separates Repo) | - -### 1.2 Aktuelle Tabellen-Konfiguration - -Aus dem Automation-Template (`subAutomationTemplates.py`) extrahiert: - -```json -{ - "tables": [ - { - "name": "Artikel", - "powerbi_table_name": "Artikel", - "steps": [ - { - "keep": { - "columns": [ - "I_ID", "Artikelbeschrieb", "Artikelbezeichnung", - "Artikelgruppe", "Artikelkategorie", "Artikelkürzel", - "Artikelnummer", "Einheit", "Gesperrt", - "Keywords", "Lieferant", "Warengruppe" - ] - } - }, - { - "fillna": { - "column": "Lieferant", - "value": "Unbekannt" - } - } - ] - }, - { - "name": "Einkaufspreis", - "powerbi_table_name": "Einkaufspreis", - "steps": [ - { - "to_numeric": { - "column": "EP_CHF", - "errors": "coerce" - } - }, - { - "dropna": { - "subset": ["EP_CHF"] - } - } - ] - } - ] -} -``` - -### 1.3 Zusätzliche Tabellen (im Chatbot referenziert, aber nicht in der Config) - -Aus den SQL-Beispielen in `bridges/tools.py` und `chatbot.py`: - -| Tabelle | Spalten (referenziert im Code) | Joins | -|---|---|---| -| `Lagerplatz_Artikel` | `R_ARTIKEL`, `R_LAGERPLATZ`, `S_IST_BESTAND`, `S_RESERVIERTER__BESTAND` | ON `Artikel.I_ID = Lagerplatz_Artikel.R_ARTIKEL` | -| `Lagerplatz` | `I_ID`, `Lagerplatz` (Name) | ON `Lagerplatz_Artikel.R_LAGERPLATZ = Lagerplatz.I_ID` | - -Diese Tabellen sind vermutlich in einer älteren Config-Version oder direkt im Preprocessor konfiguriert. - -### 1.4 API-Schnittstellen - -**Abfrage-API** (genutzt vom `PreprocessorConnector`): -- Methode: `POST` -- Payload: `{"query": "SELECT ..."}` -- Header: `X-DB-API-Key: ` -- Response: `{"success": true/false, "data": [...], "row_count": N, "message": "..."}` -- Einschränkung: Nur SELECT-Queries (validiert im Gateway) - -**Update-API** (genutzt vom Automation-Template): -- Methode: `POST` -- Payload: `configJson` (Tabellendefinitionen + Transformationsschritte) -- Header: `X-PP-API-Key: ` -- Zweck: Datenbank mit neuer Konfiguration aktualisieren - -### 1.5 Transformation-Steps (bekannte Operationen) - -Aus der Config-JSON abgeleitet: - -| Operation | Parameter | Beschreibung | -|---|---|---| -| `keep` | `columns: [...]` | Nur angegebene Spalten behalten | -| `fillna` | `column`, `value` | NULL-Werte ersetzen | -| `to_numeric` | `column`, `errors` | Spalte in numerischen Typ konvertieren | -| `dropna` | `subset: [...]` | Zeilen mit NULL in angegebenen Spalten entfernen | - ---- - -## 2. Benötigte Erweiterungen (nach Position) - -### 2.1 Position 1.5: Kundenartikelnummern - -**Neue Tabelle: `Kundenartikelnummer`** - -| Spalte (geschätzt) | Typ | Beschreibung | -|---|---|---| -| `I_ID` | INT | Primary Key | -| `R_ARTIKEL` | INT | FK auf Artikel.I_ID | -| `Kundenummer` | VARCHAR | Kundennummer | -| `Kundenartikelnummer` | VARCHAR | Kunden-eigene Artikelnummer | -| `Bezeichnung` | VARCHAR | Kundenbezeichnung (optional) | - -**Config-Erweiterung:** -```json -{ - "name": "Kundenartikelnummer", - "powerbi_table_name": "Kundenartikelnummer", - "steps": [ - {"keep": {"columns": ["I_ID", "R_ARTIKEL", "Kundenummer", "Kundenartikelnummer", "Bezeichnung"]}} - ] -} -``` - -**Aufwand-Bewertung:** Falls der Preprocessor neue Tabellen per Config akzeptiert: ~2-3h Config + Test. Falls neuer Code nötig: ~6-8h. - -### 2.2 Position 3.1: Bestellwesen (Materialmanagement 1) - -**Neue Tabellen (geschätzt 3-4 Tabellen):** - -| Tabelle | Wichtige Spalten | Zweck | -|---|---|---| -| `Bestellkopf` | ID, Bestellnummer, Lieferant, Bestelldatum, Status, Wunschtermin | Bestellübersicht | -| `Bestellposition` | ID, R_Bestellung, R_Artikel, Menge, Preis, Status, Bestätigter_Termin | Positionsdetails | -| `Wareneingang` | ID, R_Bestellung, R_Position, Eingangsdatum, Menge, Qualität | Lieferverfolgung | -| `Auftrag` | ID, Auftragsnummer, Kunde, R_Artikel, Menge, Termin | Betroffene Aufträge | - -**Aufwand-Bewertung:** 4 Tabellen × ~4h pro Tabelle (Config + Code + Transformationen + Test) = ~16h. Bei komplexen Transformationen (Joins, Aggregationen): +4-6h. - -### 2.3 Position 4.1: KPI-Daten (Materialmanagement 2) - -**Neue Tabellen/Views (geschätzt 3-4):** - -| Tabelle/View | Wichtige Spalten | Zweck | -|---|---|---| -| `Lagerjournal` | ID, R_Artikel, Buchungsdatum, Menge, Typ | Lagerbewegungen | -| `Preishistorie` | ID, R_Artikel, R_Lieferant, Datum, Preis, Währung | Preisentwicklung | -| `Bestandesbedarfsliste` | R_Artikel, Bedarf, Bestand, Fehlmenge, Datum | Dispositionsplanung | -| `View_Termintreue` | R_Lieferant, Wunschtermin, Bestätigt, Geliefert, Abweichung_Tage | Aggregierte KPIs | - -**Aufwand-Bewertung:** 4 Tabellen/Views × ~4h = ~16h. Aggregierte Views (Termintreue): +4-6h für Berechnungslogik im Preprocessor. - ---- - -## 3. Gesamtbewertung Preprocessor-Erweiterungen - -### 3.1 Zusammenfassung - -| Position | Neue Tabellen | Config-Aufwand | Code-Aufwand | Test | Gesamt | -|---|:-:|:-:|:-:|:-:|:-:| -| 1.5 (Kundenartikelnummern) | 1 | 1h | 3-5h | 2h | **6-8h** | -| 3.1 (Bestellwesen) | 3-4 | 2h | 8-12h | 4h | **14-18h** | -| 4.1 (KPIs) | 3-4 | 2h | 8-12h | 4h | **14-18h** | -| **Gesamt** | **7-9** | **5h** | **19-29h** | **10h** | **34-44h** | - -### 3.2 Offene Fragen (Code-Review des Preprocessor-Repos erforderlich) - -| # | Frage | Auswirkung | -|---|---|---| -| P1 | Unterstützt der Preprocessor neue Tabellen per Config-Erweiterung, oder muss für jede Tabelle Code geschrieben werden? | Bestimmt ob Config-only (~2h/Tabelle) oder Code (~4h/Tabelle) | -| P2 | Können aggregierte Views/Berechnungen im Preprocessor definiert werden? | Termintreue-KPI, Bestandsreichweite | -| P3 | Wie werden Joins zwischen Tabellen gehandhabt? (SQLite-seitig oder Preprocessor-seitig) | Komplexität der Cross-Table-Queries | -| P4 | Gibt es Rate-Limits oder Grössen-Limits bei der Query-API? | Performance bei komplexen KPI-Abfragen | -| P5 | Wie gross ist die aktuelle SQLite-Datenbank? Wie viele Artikel? | Dimensionierung für 8-10 neue Tabellen | - -### 3.3 Empfehlung - -**Vor Projektstart sollte ein Code-Review des Preprocessor-Repos durchgeführt werden** (geschätzter Aufwand: 2-4h). Dabei klären: - -1. Erweiterbarkeit: Kann der Preprocessor neue Tabellen per Config akzeptieren? -2. Transformationen: Welche Operationen sind neben `keep`, `fillna`, `to_numeric`, `dropna` verfügbar? -3. Performance: Wie skaliert die SQLite-DB mit 8-10 zusätzlichen Tabellen? -4. Deployment: Wie wird der Preprocessor deployed? (CI/CD, manuell, Azure DevOps) - -Das Ergebnis dieses Reviews kann die Aufwandsschätzung für Pos. 1.5, 3.1 und 4.1 um jeweils 4-6h nach oben oder unten korrigieren. - ---- - -## 4. Aktueller Datenfluss (zur Referenz) - -``` -ERP (Althaus) - │ - ▼ (Power BI Export / API / DB-Zugriff -- Mechanismus unklar) -Preprocessor Server (Azure) - │ - ├── /api/v1/dataprocessor/update-db-with-config ← Automation-Template - │ (Tabellen laden, transformieren, in SQLite schreiben) - │ - └── /api/v1/dataquery/query ← PreprocessorConnector (Gateway) - (SQL SELECT auf SQLite ausführen) - │ - ▼ -Gateway (Chatbot LangGraph) - │ - ▼ -React Frontend (Chat-UI) -``` - ---- - -*Assessment erstellt auf Basis der Gateway-Code-Analyse. Für eine genauere Schätzung ist ein Code-Review des Preprocessor-Repos erforderlich.* diff --git a/env-dev.env b/env-dev.env index 467f70b4..748d8c8d 100644 --- a/env-dev.env +++ b/env-dev.env @@ -71,7 +71,7 @@ Connector_AiMistral_API_SECRET = DEV_ENC:Z0FBQUFBQnFCdlFlelh2T2hqNGcxV0hMV1FKbmF Service_MSFT_TENANT_ID = common # Google Cloud Speech Services configuration -Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpETk5FWWM3Q0JKMzhIYTlyMkhuNjA4NlF4dk82U2NScHhTVGY3UG83NkhfX3RrcWVtWWcyLXRjU1dTT21zWEl6YWRMMUFndXpsUnJOeHh3QThsNDZKRXROTzdXRUdsT0JZajZJNVlfb0gtMXkwWm9DOERPVnpjU0pyUEZfOGJsUnprT3ltMVVhalUyUm9hMUFtZEtHUnJqOGZ4dEZjZm5SWVVTckVCWnY1UkdVSHVmUlgwbnAyc0xDQW84R3ViSko5OHVCVWZRUVNiaG1pVFB6X3EwS0FPd2dUYjhiSmRjcXh2WEZiXzI4SFZqT21tbDduUWRyVWdFZXpmcVM5ZDR0VWtzZnF5UER6cGwwS2JlLV9CSTZ0Z0IyQ1h0YW9TcmhRTXZEckp4bWhmTkt6UTNYMk4zVkpnbUJmaDIxZnoyR2dWTEYwTUFEV0w2eUdUUGpoZk9XRkt4RVF1Z1NPdUpBeTcyWV9PY1Ffd2s0ZEdVekxGekhoeEl4TmNqaXYtbUJuSVdycFducERWdWtZajZnX011Q2w4eE9VMTBqQ1ZxRmdScWhXY1E3WWhzX1JZcHhxam9FbDVPN3Q1MWtrMUZuTUg3LVFQVHp1T1hpQWNDMzEzekVJWk9ybl91YUVjSkFob1VaMi1ONEtuMnRSOEg1S3QybUMwbVZDejItajBLTjM2Zy1hNzZQMW5LLVVDVGdFWm5BZUxNeEFnUkZzU3dxV0lCUlc0LWo4b05GczVpOGZSV2ZxbFBwUml6OU5tYjdnTks3Y3hrVEZVTHlmc1NPdFh4WE5pWldEZklOQUxBbjBpMTlkX3FFQVJ6c2NSZGdzTThycE92VW82enZKamhiRGFnU25aZGlHZHhZd2lUUmhuTVptNjhoWVlJQkxIOEkzbzJNMjZCZFJyM25tdXBnQ2ZWaHV3b2p6UWJpdk9xUEhBc1dyTlNmeF9wbm5yYUhHV01UZnVXWDFlNzBkdXlWUWhvcmJpSmljbmE3LUpUZEg4VzRwZ2JVSjdYUm1sODViQXVxUzdGTmZFbVpiN2V1YW5XV3U4b2VRWmxldGVGVHZsSldoekhVLU9wZ2V0cGZIYkNqM2pXVGctQVAyUm4xTHhpd1VVLXFhcnVEV21Rby1hbTlqTl84TjVveHdYTExUVkhHQ0ltaTB2WXJnY1NQVE5PbWg3ejgySElYc1JSTlQ3NDlFUWR6STZVUjVqaXFRN200NF9LY1ljQ0R2UldlWUtKY1NQVnJ4QXRyYTBGSWVuenhyM0Z0cWtndTd1eG8xRzY5a2dNZ1hkQm5MV3BHVzA2N1QwUkd6WlRGYTZQOUhnVWQ2S0Y5U0s1dXFNVXh5Q2pLWVUxSUQ2MlR1ak52NmRIZ2hlYTk1SGZGWS1RV3hWVU9rR3d1Rk9MLS11REZXbzhqMHpsSm1HYW1jMUNLT29YOHZsRWNaLTVvOFpmT3l3MHVwaERTT0dNLWFjcGRYZ25qT2szTkVFUnRFR3JWYS1aNXFIRnMyalozTlQzNFF2NXJLVHVPVF9zdTF6ZjlkbzJ4RFc2ZENmNFFxZDZzTzhfMUl0bW96V0lPZkh1dXFYZlEteFBlSG84Si1FNS1TTi1OMkFnX2pOYW8xY3MxMVJnVC02MDUyaXZfMEVHWDQtVlRpcENmV0h3V0dCWEFRS2prQXdNRlQ5dnRFVHU0Q1dNTmh0SlBCaU55bFMydWM1TTFFLW96ODBnV3dNZHFZTWZhRURYSHlrdzF3RlRuWDBoQUhSOUJWemtRM3pxcDJFbGJoaTJ3ZktRTlJxbXltaHBoZXVJVDlxS3cxNWo2c0ZBV0NzaUstRWdsMW1xLXFkanZGYUFiU0tSLXFQa0tkcDFoMV9kak41ZjQ0R214UmtOR1ZBanRuemY3Mmw1SkZ5aDZodGIzT3N2aV85MW9kcld6c0g0ZDgtTWo3b3Y3VjJCRnR2U2tMVm9rUXNVRnVHbzZXVTZ6RmI2RkNmajBfMWVnODVFbnpkT0oyci15czJHU0p1cUowTGZJMzVnd3hIRjQyTVhKOGRkcFRKdVpyQ3Yzd01Jb1lSajFmV0paeEV0cjk1SmpmdWpDVFJMUmMtUFctOGhaTmlKQXNRVlVUNlhJemxudHZCR056SVlBb3NOTEYxRTRLaFlVd2d3TWtxVlB6ZEtQLTkxOGMyY3N0a2pYRFUweDBNaGhja2xSSklPOUZla1dKTWRNbG8tUGdSNEV5cW90OWlOZFlIUExBd3U2b2hyS1owbXVMM3p0Qm41cUtzWUxYNzB1N3JpUTNBSGdsT0NuamNTb1lIbXR4MG1sakNPVkxBUXRLVE1xX0YxWDhOcERIY1lTQVFqS01CaXZKNllFaXlIR0JsM1pKMmV1OUo3TGI1WkRaVnYxUTl1LTM0SU1qN1V1b0RCT0x0VHNLTmNLZnk1S0MxYnBBcm03WnVua0xqaEhGUzhOU253ZkppRzdudXBSVlMxeFVOSWxtZ1o2RVBSQUhEUEFuQ1hxSVZMME4yWUtaU3VyRGo3RkUyRUNjT0pNcE1BdE1ZRzdXVl8ydUtXZjdMdHdEVW4teHUtTi1HSGliLUxud21TX0NtcGVkRFBHNkZ1WTlNczR4OUJfUVluc1BoV09oWS1scUdsNnB5d1U5M1huX3k4QzAyNldtb2hybktYN2xKZ1NTNWFsaWwzV3pCRVhkaGR5eTNlV1d6ZzFfaFZTT0E4UjRpQ3pKdEZxUlJ6UFZXM3laUndyWEk2NlBXLUpoajVhZzVwQXpWVzUtVjVNZFBwdWdQa3AxZC1KdGdqNnhibjN4dmFYb2cxcEVwc1g5R09zRUdINUZtOE5QRjVUU0dpZy1QVl9odnFtVDNuWFZLSURtMXlSMlhRNTBWSVFJbEdOOWpfVWV0SmdRWDdlUXZZWE8xRUxDN1I0aEN6MHYwNzM1cmpJS0ZpMnBYWkxfb3FsbEV1VnlqWGxqdVJ6SHlwSjAzRlMycTBaQ295NXNnZERpUnJQcjhrUUd3bkI4bDVzRmxQblhkaFJPTTdISnVUQmhET3BOMTM4bjVvUEc2VmZhb2lrR1FyTUl2RWNEeGg0U0dsNnV6eU5zOUxiNDY5SXBxR0hBS00wOTgyWTFnWkQyaEtLVUloT3ZxZGh0RWVGRmJzenFsaUtfZENQM0JzdkVVeTdXR3hUSmJST1NBMUI1NkVFWncwNW5JZVVLX1p1RXdqVnFfQWpvQ08yQjZhN1NkTkpTSnUxOVRXZXE0WFEtZWxhZW1NNXYtQ2sya0VGLURmS01lMkctNVY3c2ZhN0ZGRFgwWHlabTFkeS1hcUZ1dDZ3cnpPQ3hha2IzVE11M0pqbklmU0diczBqTFBNZC1QZGp6VzNTSnJVSjJoWkJUQjVORG4tYUJmMEJtSUNUdVpEaGt6OTM3TjFOdVhXUHItZjRtZ25nU3NhZC1sVTVXNTRDTmxZbnlfeHNsdkpuMXhUYnE1MnpVQ0ZOclRWM1M4eHdXTzRXbFRZZVQtTS1iRVdXVWZMSGotcWg3MUxUYTFnSEEtanBCRHlZRUNIdGdpUFhsYjdYUndCZnRITzhMZVJ1dHFoVlVNb0duVjlxd0U4OGRuQVV3MG90R0hiYW5MWkxWVklzbWFRNzBfSUNrdzc5bVdtTXg0dExEYnRCaDI3c1I4TWFwLXZKR0wxSjRZYjZIV3ZqZjNqTWhFT0RGSDVMc1A1UzY2bDBiMGFSUy1fNVRQRzRJWDVydUpqb1ZfSHNVbldVeUN2YlAxSW5WVDdxVzJ1WHpLeUdmb0xWMDNHN05oQzY3YnhvUUdhS2xaOHNidkVvbTZtSHFlblhOYmwyR3NQdVJDRUdxREhWdF9ZcXhwUWxHc2hyLW5vUGhIUVhJNUNhY0hFU0ptVnI0TFVhZDE1TFBBUEstSkRoZWJ5MHJhUmZrR1ZrRlFtRGpxS1pOMmFMQjBsdjluY3FiYUU4eGJVVXlZVEpuNWdHVVhJMGtwaTdZR2NDbXd2eHpOQ09SeTV6N1BaVUpsR1pQVDBZcElJUUt6VnVpQmxSYnE4Y1BCWV9IRWdVV0p3enBGVHItdnBGN3NyNWFBWmkySnByWThsbDliSlExQmp3LVlBaDIyZXp6UnR6cU9rTzJmTDBlSVpON0tiWllMdm1oME1zTFl2S2ZYYllhQlY2VHNZRGtHUDY4U1lIVExLZTU4VzZxSTZrZHl1ZTBDc0g4SjI4WGYyZHV1bm9wQ3R2Z09ld1ZmUkN5alJGeHZKSHl1bWhQVXpNMzdjblpLcUhfSm02Qlh5S1FVN3lIcHl0NnlRPT0= +Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQnFIc3YtU0x4LTlHbTY1NUVGY2V2bUdmck85dDh1ZWVKa2ktR0N6NjdlTGFrUHMybVQ2bVRLN01XNFRZR2lyN0ZNSHhzWVVGNnVtZjRjV2hhR0ViTDYwT25lSmxJY0pSTkl3OUEyT0JxMFVYRndfUFJudExMajdTYUNXS01JU2lhQzZmNWFYdXA4aVZ5Zkh4Zko1Z00tcEE5ZFEwQkFVa1oyR296YXozRFI2WUdXN0ZSREFFclFNaTd6OUVlSmFxS1BTSlNJbnlWNHNfbkk4QzVOUGlkMzdfQUZxUlJOVEZzUlN1aWRWY01JZmlRM0JNZE1EZ3BmbW10c3BDdERpa2FMakstQUlqVEVlRC1hUmZoeFVoQ3pYNXRlRFVSTlI3ekJrU0QwSHBSaWxiSGU0akFGMXUtY2Q0RnUzS0tPOEQtcTdVdWhQeHFDM1hRRVVMcUxCeklvWHNWRUN2bjVHZUUwLTVtaGpUbWdPUnJabWlIcHZ5UjNtN0NMTUNRN29ZRGVXU28xQmhJTVg2eEZnaUdrcW9UVklHMHJycm1nT0JkdGJReVVHeV8tYm12UDlOU0lpNHFidXBQbUFSSVVmWUl1M1BVMFFncm0xSldkVzBrb2poRFMyaVUwcUZvMHl0QlZIZ1h1MjZwR3AtZWhqdzN4UVhtT2hUa1lQU3VudzNXdW1FcVY3VnQ3RmpkQnFQemlrQlF3WGhBNWxOZXJ6Zm9KVFlEZExUXzlqODhYaFNNMzVWTzFNMmVTcWdodDZoRmZTUzlhLVlOSU5fYW1vNXctaFpFMC1pUllRZW11d1JQN25sbldHVjI1anc2UC1ycndjTGtxWk55WmpJeU1wOVR0RnlTdFpad1dkRmlUNDE0d240TDlKc3JFUXdOYzd5UTFYSXUzLTQ2Y1ZGcWE3R2RyQ0I1WDMtMHBScEFzZDV4UEkyanh4ckJZUjdTYnJGZjAxQkU3MEJ6OXdybGRaWHNod1hZZEhVOXRpMWRLbVJsRGd0UDRDN3JsRzF4T0RpcnczRU5TM0RKVjVkWTRqNTl6bmhQdmdvaEg1U2kya0QtQ0l4ZHVUcGxkNi1vNVVVOEcyWXhxZWc5N1lKMk4tT0o3ZFVzYjJtT3NVZFJiSTFNUnpaSmFOeDZaLWVpZlc0VUhZRHdXOUMyQ3cwaXBQUDRJN1g1YkwzaTFiRVRxRFY5UTdZU1dSaGR6NUw3aEtac2RENXF3WEpVN0dXVTlQR0F6MFlpWl83MU44NVR1ZUtPVUNlZ205YUIwOFoxUDBvTlI0SU52emVvQ3VZXy1jTlFXRWZXQ0d5RHJ0eV9JeE5wMHl0b3FVSjNoVzg2d21hYVNYY3Q0dkFaVEZwa09tRnFBbEtoOUlGY2xkeVJoZGYzQUxYNFZfb0ZiaU5VRjJPbGhieXYtWTFKckZwenVCUGFva1IwVVFORVQ4SDMxWHVuRWhBRGd0cVlsc3kyQ0RyY2ZIVDlwcGh5ampySV9uOVpsVmlWbGoxMEg3SXh6NzRJbmZXRlhMMWc0RXhzeWtnQlJ0VnZSdENkbEpOdENwUzItUjZhZWFYRFhzbDM1WDBxaGFPX19CSG1KZjRTTU5JemcxZzJRSFY5bkx4TTlIZFNHOW1USWxBYWhEZ1FSNVdSSDJETUZwMi1Hd0RESkF2cVA1TVJGTEtPUl9oN3gzVEIwSzZOVzlOWXhNa2I1Vzc1SV9tdENfRy1rQTNzRlZGSTYwQmJIaGswZUNWSnRDVXFfdWFCckZZcnJOT2Rfb3FrcWI4S1lVRTMyRnZJQTRZV1VsU0xobGRjekhtbG9LamR2d1hfVklsM3JBeW9SRzJnWVdiWDRzN1ltcXdSVGoxRVBvczViVXNjMUxBazZUdS1WbkRQX0h1MzdNd3ltVDUzd2FGdi1XeUMybV9ia1YxQVBPdnUxY1dfT2M5eEpZR2JHMkdZbWdDZTRERXRYOWxodndkTXltVW40c0t0bVA5YWxuRzM3LWlCdmJiYmF5dkNBY3ozbUw1Zm5zRmpBdk5ORmFZRWJKM3Q2UDdKNl9zaUV5eVVGbkF0QmZSZzk5dGo3UjNIQWxwcjRlVTdUT2s1VGFjdndvX2c3d1VmaHRMZU10M1ZKVk9Ma3dZb1kwYVV5Z2NlTjUxdUYtZXRnRTRzQlp1aFp0OUF5TVBwN1gzU21kRmJ6OUlOeUFOOEhEOU5WSENNZndvLXdoVUFJYVFDTWEyakJEcTVSVDhJOWJscU8taThqNUZkdThCOUlXcldndFBTZk9QVnlMaUphUU5sUktpb1plZDZOQnFzNFNMUzRWbWFVQWhUWmJfem96X0cxWXVTcUxCeDhOc3E2OEpFa2lzWHFIV0p3eGdBZmN1aXBhYjExZTZqaUY4S0ZudTNhcUx2WlpuTU9lNUk2ZmNyN0JCODdYMGNEU2JsZkZXYlRFaTJQUTI5RU5SMmtkV1NHQTVTTjEyZGZLYnhTNTg2Nl9aaWJqX2Q1U1NwQ3pRTGRBSUw0N3FNQ0ItMks1QVZmbURYVWdHMWFZTWhGNURVOUg0bGVuMUozanlxTnRwbVlGX2RnN2FBVTZlZjhDaXVzZEtVR1Z5azhzWHRrS1dYSG9rYkowTjQ1N0hyRWdNVWMya1ZmWmZvSnVTdHNiMHFDODNLckpjQ081SFlieGxuM0picGhKMnNQRURwY2hpQzF3dHRnNEFWcUlPYjVxZEhod0JDbWZhU01Ob21UWmRwd0NQRlpjOE5CUFBOT004U2JKNkFSUlFzRklYZGJobUoxQzZzT2wzZ3J1Z05aYThRVVNzcFktMGJDcXFfSkxVS2hhajI3dTdrR2poa21ZM3Z4UzFRblFsOFlOZVVUM0YxaFRuNjFWQ2E4ZlhvZjZpMWFtOGRuaGx0MTZxZE9TY1dsTTMyMHhsNXJ2MkduaGRkZXpYUWJ3cEt1U3YwMC1IRzM5eWRCb0lvaUhTQ2R4XzhEZl9zRk5GeHhCSWx2X3BkUkJ4NFZLVzdVRFZkbnpNNkpjUTFHY1pDV0ZOMFBaNTVpLUlmSnFrX1N5X05MTjRUeTVERUs5MG9kMFJ3di03U3BpMUM4YXNwaG1fangwYURIVjBpSVdCUkt4UW5HbWtGOUh3TUdPZjMxYXpVZDcwTmlDcTR6WldZb3VzbHRpRUgyN2lFTjlpUV85T0M4blJxMWx0cC1iU0FDOHhueDBLYjdLZGhNbjFPbE1RdmhhNlEzX3ZpT2ZsYllwNkU5TE9fZWFabDE4RWRoRWxiMk5aVFZrWmxjaW5MX1VrUGhUN29vbU1tWldESnczYTNBQ1RPd1VTNGNJdjdJU3p3QXZQLVlDNkQ1cTh4Rk1WNnRMUi1DT3VGREFPa28xejc2NUl1dzJSa2hCTlJublBRNGkydlJVRjlFbFotOWtraWFqQkNNTXBpT1hZM0NXNEpObGMxQUNuS29rOExMSnMxT3NLbjNfLTdpQW1BcDMxR1RZdVRvbElGbENWbHJqRlVrTXhYbFdiMmItUzlxR2ZxT2FCWXpMVVJYZXBfSFVwNTczU3JHUVhET3hSWm80Ry1KcE9mV3FYejVHSEVSS0pxOUtCc3V2VHNFVkRqYk5Od20tM0ttdFQ1eGdsc091WGFYNFgybzNVd3ZvbzEwUDJ0T0hvTVd3YnlHNnpNWC0wbkJOQTIwQ3VYdlUzaXY5NFhDNlNOOW9UdGZNUk4zZ0VJakpwS21SZlJtQjVWLUxfejFYZFc1cjRwR3ZUOGdZb2VJaTdJUS1MYlRJb0ZFYW9uYzM3MDd4b09BR1pnTEh3RFpnaGhxZURQamllNUhqTHg0cHJfN08wMkdGSVQwQUlqWDhLVGViY3J5NlVFTzY3RGhGQ0R6aXNsb2w4dnBVYndTd1Jhd3IwS1BxY0h1X05RcGsySzVNbXR5YlBVQi1IOGFUNkh5QjhRZk5BQmZvcGF6ZTNXenZkdy1GRjFGdE1saGdMSnotUkIyX1VqTlZFWnJER1YyNGQtMFZHU3hmRVNPUWFCdXV3QUxzOGVSbF9EdEZGUFNxbTdiYm5oWHdYak5qa3Zoem5WY1ZUdDREVUxGX0VQeS1jckhqS2lRLXQ1Y2tyOFRjYnVhajNUZmZOUE9kbU9PYXdqdk5DYUtEOVFiMW9yZTYxMFNUaDdvUTExUFZ1bklYSkRKTnJ1RURvOTR3ODREcWdWeHpRS2RETjZqeXpvbUpxMW5lWl84RzVocmJFQ3JfZlpMd3RCZEo5RWZ0MzIxNWV6bHlwdWJJWXhoaWxlM2FHSjBhWG14Sk94ZV96cXFvU1JwWDdKZldmZWdvdWVKdXVfaS1jZjdENXQzSzNyb1d3eWhUMU53QzgxemRiTTlkdFRxZU1OdEN5c1kxOEd2MTJMcnBJWEE0eXdJdFpOYVNMQTNLR292UFlGb0Ztdz0= # Teamsbot Browser Bot Service # For local testing: run the bot locally with `npm run dev` in service-teams-browser-bot diff --git a/env-int.env b/env-int.env index 72099314..5e2a4e77 100644 --- a/env-int.env +++ b/env-int.env @@ -73,7 +73,7 @@ Connector_AiMistral_API_SECRET = INT_ENC:Z0FBQUFBQnFCdlFlU2tMLTFnQWhET2Nia2pTcVp Service_MSFT_TENANT_ID = common # Google Cloud Speech Services configuration -Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkNmVXZ1pWcHcydTF2MXF0ZGJoWHBydF85bTczTktiaEJ3Wk1vMW1mZVhDSG1yd0ZxR2ZuSGJTX0N3MWptWXFJTkNTWjh1SUVVTXI4UDVzcGdLMkU5SHJ2TUpkRlRoRWdnSldtYjNTQkh4UDJHY2xmdTdZQ1ZiMTZZcGZxS3RzaHdjV3dtVkZUcEpJcWx0b2xuQVR6ZmpoVFZPY1hNMTV2SnhDaC1IZEh4UUpLTy1ILXA4RG1zamJTbUJ4X0t2M2NkdzJPbEJxSmFpRzV3WC0wZThoVzlxcmpHZ3ZkLVlVY3REZk1vV19WQ05BOWN6cnJ4MWNYYnNiQ0FQSUVnUlpfM3BhMnlsVlZUOG5wM3pzM1lSN1UzWlZKUXRLczlHbjI1LTFvSUJ4SlVXMy1BNk43bE5Hb0RfTTVlWk9oZnFIaVg0SW5pbm9EcXRTTzU1RFlYY3dTcnpKWWNyNjN5T1BGZ0FmX253cEFncmhvZVRuM05KYzhkOEhFMFJsc2NBSEwzZVZ1R0JMOGxsekVwUE55alZaRXFrdzNWWVNGWXNmbnhKeWhQSFo2VXBTUlRPeHdvdVdncEFuOWgydEtsSUFneUN6cGVaTnBSdjNCdVJseGJFdmlMc203UFhLVlYyTENkaGg2dVN6Z2xwT1ZmTmN5bVZGUkM3ZWcyVkt2ckFUVVd3WFFwYnJjNVRobEh2SkVJbXRwUUpEOFJKQ1NUc0Q4NHNqUFhPSDh5cTV6MEcwSDEwRUJCQ2JiTTJlOE5nd3pMMkJaQ1dVYjMwZVVWWnlETmp2dkZ3aXEtQ29WNkxZTFkzYUkxdTlQUU1OTnhWWU12YU9MVnJQa1d2ZjRtUlhneTNubEMxTmp1eUNPOThSMlB3Y1F0T2tCdFNsNFlKalZPV25yR2QycVBUb096RmZ1V0FTaGsxLV9FWDBmenBIOXpMdGpLcUc0TWRoY2hlMFhYTzlET1ZRekw0ZHNwUVBQdVJBX2h6Q2ZzWVZJWTNybTJiekp3WmhmWF9SUFBXQzlqUjctcVlHWWVMZWVQallzR0JGTVF0WmtnWlg1aTM1bFprNVExZXY5dnNvWF93UjhwbkJ3RzNXaVJ2d2RRU3JJVlBvaVh4eTlBRUtqWkJia3dJQVVBV2Nqdm9FUTRUVW1TaHp2ZUwxT0N2ZndxQ2Nka1RYWXF0LWxIWFE0dTFQcVhncFFPM0hFdUUtYlFnemx3WkF4bjA1aDFULUdrZlVZbEJtRGRCdjJyVkdJSXozd0I0dF9zbWhOeHFqRDA4T1NVaWR5cjBwSVgwbllPU294NjZGTnM1bFhIdGpNQUxFOENWd3FCbGpSRFRmRXotQnU0N2lCVEU5RGF6Qi10S2U2NGdadDlrRjZtVE5oZkw5ZWFjXzhCTmxXQzNFTFgxRXVYY3J3YkxnbnlBSm9PY3h4MlM1NVFQbVNDRW5Ld1dvNWMxSmdoTXJuaE1pT2VFeXYwWXBHZ29MZDVlN2lwUUNIeGNCVVdQVi1rRXdJMWFncUlPTXR0MmZVQ1l0d09mZTdzWGFBWUJMUFd3b0RSOU8zeER2UWpNdzAxS0ZJWnB5S3FJdU9wUDJnTTNwMWw3VFVqVXQ3ZGZnU1RkUktkc0NhUHJ0SGFxZ0lVWDEzYjNtU2JfMGNWM1Y0dHlCTzNESEdENC1jUWF5MVppRzR1QlBNSUJySjFfRi1ENHEwcmJ4S3hQUFpXVHA0TG9DZWdoUlo5WnNSM1lCZm1KbEs2ak1yUUU4Wk9JcVJGUkJwc0NvUkMyTjhoTWxtZmVQeDREZVRKZkhYN2duLVNTeGZzdFdBVnhEandJSXB5QjM0azF0ckI3Tk1wSzFhNGVOUVRrNjU0cG9JQ29pN09xOFkwR1lMTlktaGp4TktxdTVtTnNEcldsV2pEZm5nQWpJc2hxY0hjQnVSWUR5VVdaUXBHWUloTzFZUC1oNzJ4UjZ1dnpLcDJxWEZtQlNIMWkzZ0hXWXdKeC1iLXdZWVJhcU04VFlpMU5pd2ZIdTdCdkVWVFVBdmJuRk16bEFFQTh4alBrcTV2RzliT2hGdTVPOXlRMjFuZktiRTZIamQ1VFVqS0hRTXhxcU1mdkgyQ1NjQmZfcjl4c3NJd0RIeDVMZUFBbHJqdEJxWWl3aWdGUEQxR3ZnMkNGdVB4RUxkZi1xOVlFQXh1NjRfbkFEaEJ5TVZlUGFrWVhSTVRPeGxqNlJDTHNsRWRrei1pYjhnUmZrb3BvWkQ2QXBzYjFHNXZoWU1LSExhLWtlYlJTZlJmYUM5Y1Rhb1pkMVYyWTByM3NTS0VXMG1ybm1BTVN2QXRYaXZqX2dKSkZrajZSS2cyVlNOQnd5Y29zMlVyaWlNbTJEb3FuUFFtbWNTNVpZTktUenFZSl91cVFXZjRkQUZyYmtPczU2S1RKQ19ONGFOTHlwX2hOOEE1UHZEVjhnT0xxRjMxTEE4SHhRbmlmTkZwVXJBdlJDbU5oZS05SzI4QVhEWDZaN2ZiSlFwUGRXSnB5TE9MZV9ia3pYcmZVa1dicG5FMHRXUFZXMWJQVDAwOEdDQzJmZEl0ZDhUOEFpZXZWWXl5Q2xwSmFienNCMldlb2NKb2ZRYV9KbUdHRzNUcjU1VUFhMzk1a2J6dDVuNTl6NTdpM0hGa3k0UWVtbF9pdDVsQVp2cndDLUU5dnNYOF9CLS0ySXhBSFdCSnpqV010bllBb3U0cEZZYVF5R2tSNFM5NlRhdS1fb1NqbDBKMkw0V2N0VEZhNExtQlR3ckZ3cVlCeHVXdXJ6X0s4cEtsaG5rVUxCN2RRbHQxTmcyVFBqYUxyOHJzeFBXVUJaRHpXbUoxdHZzMFBzQk1UTUFvX1pGNFNMNDFvZWdTdEUtMUNKMXNIeVlvQk1CeEdpZVdmN0tsSDVZZHJXSGt5c2o2MHdwSTZIMVBhRzM1eU43Q2FtcVNidExxczNJeUx5U2RuUG5EeHpCTlg2SV9WNk1ET3BRNXFuc0pNWlVvZUYtY21oRGtJSmwxQ09QbHBUV3BuS3B5NE9RVkhfellqZjJUQ0diSV94QlhQWmdaaC1TRWxsMUVWSXB0aE1McFZDZDNwQUVKZ2t5cXRTXzlRZVJwN0pZSnJSV21XMlh0TzFRVEl0c2I4QjBxOGRCYkNxek04a011X1lrb2poQ3h2LUhKTGJiUlhneHp5QWFBcE5nMElkNTVzM3JGOWtUQ19wNVBTaVVHUHFDNFJnNXJaWDNBSkMwbi1WbTdtSnFySkhNQl9ZQjZrR2xDcXhTRExhMmNHcGlyWjR3ZU9SSjRZd1l4ZjVPeHNiYk53SW5SYnZPTzNkd1lnZmFseV9tQ3BxM3lNYVBHT0J0elJnMTByZ3VHemxta0tVQzZZRllmQ2VLZ1ZCNDhUUTc3LWNCZXBMekFwWW1fQkQ1NktzNGFMYUdYTU0xbXprY1FONUNlUHNMY3h2NFJMMmhNa3VNdzF4TVFWQk9odnJUMjFJMVd3Z2N6Sms5aEM2SWlWZFViZ0JWTEpUWWM5NmIzOS1oQmRqdkt1NUUycFlVcUxERUZGbnZqTUxIYnJmMDBHZDEzbnJsWEEzSUo3UmNPUDg1dnRUU1FzcWtjTWZwUG9zM0JTY3RqMDdST2UxcXFTM0d0bGkwdFhnMk5LaUlxNWx3V1pLaVlLUFJXZzBzVl9Ia1V1OHdYUEFWOU50UndycGtCdzM0Q0NQamp2VTNqbFBLaGhsbUk5dUI5MjU5OHVySk1oY0drUWtXUloyVVRvOWJmbUVYRzFVeWNQczh2NXJCeVppRlZiWDNJaDhOSmRmX2lURTNVS3NXQXFZT1QtUmdvMWJoVWYxU3lqUUJhbzEyX3I3TXhwbm9wc1FoQ1ZUTlNBRjMyQTBTY2tzbHZ3RFUtTjVxQ0o1QXRTVks2WENwMGZCRGstNU1jN3FhUFJCQThyaFhhMVRsbnlSRXNGRmt3Yk01X21ldmV3bTItWm1JaGpZQWZROEFtT1d1UUtPQlhYVVFqT2NxLUxQenJHX3JfMEdscDRiMXcyZ1ZmU3NFMzVoelZJaDlvT0ZoRGQ2bmtlM0M5ZHlCd2ZMbnRZRkZUWHVBUEx4czNfTmtMckh5eXZrZFBzOEItOGRYOEhsMzBhZ0xlOWFjZzgteVBsdnpPT1pYdUxnbFNXYnhKaVB6QUxVdUJCOFpvU2x2c1FHZV94MDBOVWJhYkxISkswc0U5UmdPWFJLXzZNYklHTjN1QzRKaldKdEVHb0pOU284N3c2LXZGMGVleEZ5NGZ6OGV1dm1tM0J0aTQ3VFlNOEJrdEh3PT0= +Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQnFIc3YtSjhlcklrU2JCOW5mdHFHd0dLTUZZZk9PT3o5RWt5RjAxX2s3ekJRLUUzU0dNSnNseTE4bUpNTnZSTWg0QV9mWm5iX19aWjV4YnRXU1JBSm1INVB5dXNRT2JiYk1tLWRSS29pdTRMdS1lMDZxMkx4VTh3bU5aVWh3cEwyOE1QcXVockgtZWh5bzdNVXQyemFuSmZqRzZZYmNGN21JdjNwNWpPRXB6WU1qSU5rZUVSb3JBS0lhcThvakkwbTRUUHhBdjRZdWNsZ1Z1RmFaNGZLcEpaNVNLdFAxYzFXdTJydU9COWJ0bkNyYUF2X2FNc1BfT05teEs1SE9PeGhPd3VJSFY2VFJ5VEl6V3R3bzd6OTVKTEVRcmt5ZzdBMXBFY1A5dUFJRFJONFBlaDlJcjNBQnBraC0wMTBhNW8wYWZaeHNWclVTOVotLTdWSmVuYzJKcUZSUkdrdXB3VEVESzd4UTI0bGd6SzdCajdoazZXVTVCaGRiaWJaOHg5Z2thSWItcS05U25DbUdrT2M1QV81WEg2dlJfMlBtZU9Bc3V5bmtBWHRoRUVLR2lWNHY3M3hHcU1raFRFOWQwSEtUU1RDWDFRNFlkNHVnTkZDbk5zS3RZeGR2Z015RnRGc3NndFVEQjc4bVpNeE81bXc1MnQ2QjNZeHZCbUJJZVJ2TE5xWEd4M3hHT2hJWW5DOWMxQlNmZE9uMVRGVnRwTUlXZjZCRUZBLU9GWVZGWFpZbUE3WVlpZU1DX1Z0bWQ0bjlaRThHOE9WR3VOVzlYWS1JampTNmxkNmFxWG54WDJjallIT3UyT0tGSzJpeG1tX0JoQjZxbEpESHBhMWZFa205bjdvTVFwSVVidnVzdURZVDAzVVpkekJ2SVZTZmhxQVJ2OWpuRGR2WFE3elMtb3B2ZzhpQVNvRmkzbzRrY1BuamVzM0E2eVM0bXBHTHgtYmhsVG5jNlB1Q1JHZU9HUlNfaTJSQkcwS2FSZnZSOW9oZzdXa1RUVTVTZTgwY01GYXQyQ0xWX1Fnb0xaOTRQY3hTclgweVJ5clc5OVpRWWlDb0JQVXoxVDA0bW8zUE55aGowb1ZZNEpBN2UtSTZTY2llRGhISFFkYWFYVlVBQ0IzbGxzVTQ2V2dsUGV1Y2I5bEZLRnlwdXRHMWZVcnBaTXNzNzNkUVFqR2xnSEQ1VlpTdXpwMFVVYjQ0enFlUnk0d3dDQUtSS1dUVnNyYnBKQW9TRjJxN2JNY2NhRWNONWRpWU5RbzNNZVJBS3EzN2ZMZ1E5VXQtMDFTZklLY1JiSDNYRlFuOF9VYUktS0xoY2IyR0xkT19qTEpIV1p6RFExUWNCQTdqN1kyS0Jaa2lyMDluenc1MS1vdmhPVlE5OUphWEY2dXFYNE04Z3lBUG5DNGZjTUVnYzEzYWhzTHpMdVBzT0dzRGJaT2x5b0pVbWJtUzJxdEd2VGtrc01kTlNPNURoVHhwZzU1d3pTZGJiTUZIME5tQ0xqNWJ2QS1QSEJHV2FEOExHWDByV19rVnc2R2pibnNENEo1cTh4bGNMX2ZpSTBMcjRvQWRhbW5xYVBiZkZzWTRERlVESEU2aHpvdzNMTjlCazRYeEJhMmZwdXY5T25IYkFTaUM3SmdIV1FCX2xxRXctWHZQOHgxLXI1c1JkWmcydkFTUmxFSU03cGtnallnTXplOElQbEJRSEE2aW5KREU0YUxwX25wOFhuS2RIbms1dXNIRHBtNjFtb3B3UGVGb0hwOENKM1hMclBwa3NBa2pFYnZYbEtFbUF0Y3pmeFRmMDNMaTZrR1BZWnBrNUQ1WlU1NVZQSWUxN3dwcXhhcjdXNTl4LVVpYVF3Y0wtRmFyNXZRNTE3UUc2cHVaVVNpaVdHbXRqQVJNZWZmNjdQQ2lwTGd6RFFZN2tSY2NEdmxvaXk4MTZMcmg0VGo3MTN2R2V6cmV3YjdQVlNEZTQySUpaY2pkTHZzUzdJLVJ2WnlOQ3Vmem5FZXRaWjBMWjF4ZEF3ZHJ4VF8tMVNsRnljejVsaEpGOU5JbnhydjNVdzNMOENrWUVsbXp0ZEhuVE1Vd0RJcnp2N0RXUGFuNDM2OXBPbV9LRDUwTWk1NHYwaDhlVEhKUmtEa09INURwNjV5ZE1VWmpRSGdjeXJNc3FqcjZDdmx5WXluNWZ2VlpsWmR2TXVXVnBubEFmQlRfaGRwRndCVXVkMjkyLWVhaDQtZDN1cmFZLUoybGRwbGQ5MTExU2NnZ2lueVNfSjFDQ2NkWGtNX2M1T2I4YnVJOUFueGIxbG1EYlZOcFYtQlE3cm90SE40X0ZjalhLdXM5S2l5aW84ZUJPMlR4MU9EVkhZcHdrX1Zqc0NhWEJacDZHMzQwSzdkdi1Rd2s4Y1dfLS1ES0NfYTNxYl84UTN1S0lIM0pVTTNEYlJ0YW55Tk4yVjBONXNTQWtVZTJ2V3B5eHBJcG9IWGRMMklob0hMbVVZZzJKbTFMUExOQm5HSEZzWHU0VGVIWlJMVzFLeFB0NkkyWFkwWk0wdjdHRmxSWFFoSkJ2Vm5NUWNQQlp6YWlIc2NKLUdhOVVycHd5N3NFMDNVWlAxZGQ1NzRGbm9LcWxEb2tKR1RnVEtvRUc1d3l4aU1IOUQ5RldUT3Z0a3lpRHpVSWJ4MjU4RWY5MEpCQ0VFdHNMbnkxOGswcE44QzJwNXFCVGpIa0VGc2VNXy1qdzVNRU9DaXg2MW9VX3FjUk41QVFVLURwVGFLRTkyNWlENy1IcGZjNW9wY0Y5Q3d5eFg5emVUUF9hV3ZTQWNaNEN0VzdJRlFBR0picXJoUERacWNLbDZhTE8wdWlfZ3kxd2QzOXBOZV9uaUNGMkNJbGhNd3k0S2t3dTRGWVVxTTFRRlg3Ui1zLW1FLU1Mai1yaURjb2Fob2c4MDUyRHN5aldUVWMxLTVNbm5VQTdrYy0zLVFyOHRkNzZ3dGdhbXZXN3JHNkdfZ2RuRXFDM3R2TVB1cDNOdWZGTmpFNnNFTmMxTmFuZDdJUld5bERyQkJ0TGZXRk54NEdqN09hSmVMYV91NXUwNXFvMl9KV0hBNlB4bklNQ2U5WGZLUTdlX2dJenVGcDYwWHBsdTNpbE5mWGhWeXFuUkFPV0puR2h0RkhrR2MwTzJGUmp4bUR6UFlUWTlNbTJLa19hTUZZR0dscVpBbFBReTBRMDNseXo4SXNnZWt4VFdpOERqLV9ZczRkR0QwRFJQM0pqdHluWktDUlp6WU9XSjVNZi1tYnNzcVlGTDRFMzNlSmRTazFfTkNxSjAwM0wxNk9Sd2h1SWpfOW5MVWMtVXYyYlVZR0VuaHRpN1pnNnpHME5raVBMd2h2dDRyMV8yZGFJNnlkcmhtSWdmNlpLN19NcjNkc002dXFxQzhTaDZzRlgzNUJ1SzVpVnp6NVU1Y2luUlM4UEJoajNTOUJadnE1MlhzV0kxSzBObXkteVhNM3RKYW9heDVWWFJ1NGlDM0l0elRPbThwUU9oYkVkbC1PZFNLSHY3WHJiZWpEamNIVC00MlNNWV9qcHdjNDRjRlVhZXlrLTlicVBNaDlDeXdRb0Fwc3RmUGFvbURQZ29yckliaS1VUDNxcXVlYTJJRUhXNUVobk1KUDhHZE16UzBLeDViYVRwZWY3d2w0d253eEZYcExKRGpsaGlBUElaTzB3eUVadnROX1dabENGb3R4ZF9aS05KY0dHTVZaYzRFc1Z4TlZGbFd2NjdYRzJMTzVwU2NaN1Y3MzQ2Z2pzV2RSMzJBbjg0MEhaZmhoREloY0oxOFdjNDZNdVZfYlRKU1Q1M2hYdHgwUjVsTV9USjZCZXlQTTdNRWc3bUxOcXRDVkpTdnJxR0hkWWpaRUdrOEFyNHk4MENwVzdob0hUSkJvam4zZW1kcGxZUjg0RXFRNnBxSUg1MDVHdHRwVlFkWWhHM0ZyZVFvMF96R2V5YjBuMnVZTU5CQ3pVci16SGJlQTQtbnFLa1E2eHFncUg3UmYyYlZvOF82a3d2ZE4tbmxIUlNYYjlrck9QYk5CcV9faXludS1yem1JNjFBdVYyb21RQWFMMFkxX0s1TjQ4czZ2WXI3X0FzRWdNTlZndHl4bnVOTHl2YlZfaURQV053dHl4N1czRFdzaVFnRHB0MWRDV2ZuU2lzX1NZZkRQYzhsT3ItZWw0dVJlVmtFWUM5cEppOGxuYVdpQkN5dV9hQ2dodTJvV3REVkw2dVVDaGtvc0Zqd0V2dldLZEVNRVRRNVRUVmw5aHZmZEpHdk1wS0xwRFc5Vmx4dTdfdGZDRUtCU29qdEVIOW5VdjBmeGpFMFZHSUthamtVN1E2bDZqaEFackVSQnZMN0tyaUhIcUs1ZHMzMzl2TnhadGIwZW5QNS1BM3pSODY3WVFsLU1jeUpCMG1PWmhPVT0= # Teamsbot Browser Bot Service (service-main-teams-browser-bot on Infomaniak) TEAMSBOT_BROWSER_BOT_URL = http://teamsbot.poweron.swiss:4100 diff --git a/env-prod.env b/env-prod.env index d268450c..4395fd5d 100644 --- a/env-prod.env +++ b/env-prod.env @@ -72,7 +72,7 @@ Connector_AiMistral_API_SECRET = PROD_ENC:Z0FBQUFBQnFGdnVLRHplbzNheDhIdndsU0xUeG Service_MSFT_TENANT_ID = common # Google Cloud Speech Services configuration -Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQnBDM1Z4NFQxaF9uN3h1cVB6dnZid1c1R1VfNDlSQ1NHMEVDZWtKanpMQ29CLXc1MXBqRm1hQ0YtWVhaejBMY1ZTOEFEVlpWQ3hrYkFza1E2RDNsYkdMMndNR0VGNTMwVDRGdURJY3hyaVFxVjEtSEYwNHJzeWM3WmlpZW9jU2E3NTgycEV2allqQ3dJRTNyRFAzaDJ6dklKeXpNRkJhYjFzUkptN2dpbkNpMklrcGxuZl9vTkt3T0JvNm1YTXd5UlkwZWptUXdWVFpnV2J4X3J2WUhIUlFkSElFVnlqMnlJRnNHTnlpMWs2R1dZc2ROWjNYZG85cndmd1E5cUZnVmZRYnVjTG43dXFmSWd2bGFfVWFWSmtpWkpndWNlSUNwcnFNU2NqZXFaV0xsY3l3SElLRkVHcHZGZERKV1ltcGhTS0dhTko1VTJLYzNoZjRkSGVEX3dTMWVVTmdDczV5cE1JQUdSbUJGUm11eFhTVjJHbkt0SzB4UG1Dc2xmbnp1Y041Y2RTeWRuWGdmQy1sTGx0MGtnM2VJQ3EyLXViRlNhTU9ybzZkR1N1bXE5SXhlZENWRFpWSGlYOWx4SUQ3UlR0ZEVxQkxNakRUVFRiUmFnbklOalphLUZkRFVVaXBRUk5NZW5PaUZydTFmQkNPSTdTVTNZd0plWXllNVFJdmN4MVcyTGlwMGFtVjBzOGRxR1FjbzhfYW5zdTB0ZEZBTTJhakltazh1dktNMUZsOUItdFdTb1pIaUxySllXNkdlY20zUS0wTnpFNTB2SU5acG1VcXhyaHBmME8takw3RDh5T043T2VGOV92TzNya2pWSlpYVjZDdXlZcjM3a0hPTlhkaW9oQmxqQlpGRFYyTTY4WmZmT3k4Tk1tdXRuSGdTUVpNT2NKenhXb05PdXBfSEdhMTNxNjdpNXlKUUI2YUgydFFPX1VvXzVJb0UxWTU2YVNiNDQ0QndZanhMMHR1cGdHWGhvcEg1QXEtSXZJdTdZUE12ZEVVWkF4QmtsQS1GYnY3SFIxSHlsOGVfcEpGS1A4QUVEQWNEOFZYYlljQ3ByTU03YU16Y0UzUnJQZEprSWNjT1ZXVEtDWi03Y3ZzRVdYUTlabXJISEo5THRHVXVuM0xqbzA4bGVlZVpOMk1QMmptb21tV0pTMlVoOXdWVU95UW1iQmttc2w1RG9mMWwxXzg1T2IxYUVmTUJEZkpUdTFDTzZ3RlBFeUFiX01iRTZNWkNaSG45TkFOM2pzbUJRZ2N0VFpoejJUTG1RODY3TzZpSzVkYUQzaEpfY2pSTkRzU0VpanlkdXVQQmJ2WU5peno4QWNLTDVxZTlhSHI3NnNiM0k0Y3JkQ0xaOU05bGtsQl8zQklvaktWSDZ4aVp2MHlYelJuUDJyTU9CZC1OZjJxNFc1dDcwSUlxaVh1LTMyWWFwU0IwUU9kOUFpMWpnOERtLTh1VmJiNGVwcXBMbU5fMjVZc0hFbmxQT2puSFd1ZGpyTkphLU5sVlBZWWxrWEZrWGJQWmVkN19tZFZfZ1l1V3pSWlA0V0ZxM2lrWnl2NU9WeTdCbDROSmhfeENKTFhMVXk1d195S2JMUFJoRXZjcVo4V2g0MTNKRnZhUE1wRkNPM3FZOGdVazJPeW5PSGpuZnFGTTdJMkRnam5rUlV6NFlqODlIelRYaEN5VjdJNnVwbllNODNCTFRHMWlXbmM1VlRxbXB3Wm9LRjVrQUpjYzRNMThUMWwwSVhBMUlyamtPZnE4R0o4bEdHay1zMjR5RDJkZ1lYRHZaNHVHU2otR3ZpN25LZlEySEU0UmdTNzJGVHNWQXMyb0dVMV9WUE13ODhZWUFaakxGOWZieGNXZkNYRnV5djEyWTZLcmdrajRBLU1rS1Z0VVRkOWlDMU9fMGVmYXFhZXJGMUhpNkdmb2hkbzZ1OWV6VlNmVzNISjVYTFh6SjJNdWR5MWZidE8yVEo2dnRrZXhMRXBPczUwTG13OGhNUVpIQm0zQmRKRnJ0Nl8wNW1Ob0dHRDVpU0NWREV3TkY2SjktdVBkMFU1ZXBmSFpHQ3FHNTRZdTJvaExpZVEtLTU4YTVyeFBpNDdEajZtWUc4c1dBeUJqQ3NIY1NLS0FIMUxGZzZxNFNkOG9ORGNHWWJCVnZuNnJVTEtoQi1mRTZyUl81ZWJJMi1KOGdERzBhNVRZeHRYUUlqY2JvMFlaNHhWMU9pWFFiZjdaLUhkaG15TTBPZVlkS2R5UVdENTI4QVFiY1RJV0ZNZnlpVWxfZmlnN1BXbGdrbjFGUkhzYl9qeHBxVVJacUE4bjZETENHVFpSamh0NVpOM2hMYTZjYzBuS3J0a3hhZGxSM1V5UHd2OTU3ZHY0Yy1xWDBkWUk0Ymp0MWVrS3YzSktKODhQZnY3QTZ1Wm1VZkZJbS1jamdreks1ZlhpQjFOUDFiOHJ2Nm9NcmdTdU5LQXV2RkZWZEFNZnVKUjVwcVY3dDdhQnpmRVJ6SmlvVXpDM0ZiYXh5bGE2X04tTE9qZ3BiTnN3TF9ZaFRxSUpjNjB1dXZBcy1TZHRHTjFjSUR3WUl4cE9VNzB5Rkk4U3Z1SVZYTl9sYXlZVk83UnFrMlVmcnBpam9lRUlCY19DdVJwOXl2TVVDV1pMRFZTZk9MY3Z1eXA0MnhGazc5YllQaWtOeTc4NjlOa2lGY05RRzY1cG9nbGpYelc4c3FicWxWRkg0YzRSamFlQ19zOU14YWJreU9pNDREZVJ3a0REMUxGTzF1XzI1bEF3VXVZRjlBeWFiLXJsOXgza3VZem1WckhWSnVNbDBNcldadU8xQ3RwOTl5NGgtVlR0QklCLWl5WkE4V1FlQTBCOVU1RE9sQlRrYUNZOGdfUmEwbEZvUTFGUEFWVmQ4V1FhOU9VNjZqemRpZm1sUDhZQTJ0YVBRbWZldkF5THV4QXpfdUtNZ0tlcGdSRFM3c0lDOTNQbnBxdmxYYWNpTmI3MW9BMlZIdTQ5RldudHpNQWQ5NDNPLVVTLXVVNzdHZXh4UXpZa3dVa2J4dTFDV1RkYjRnWXU2M3lJekRYWGNMcWU5OVh6U2xZWDh6MmpqcnpiOHlnMjA5S3RFQm1NZjNSM21adkVnTUpSYVhkTzNkNnJCTmljY0x1cl9kMkx3UHhySjZEdHREanZERzNEUTFlTkR0NWlBczAtdmFGTjdZNVpTMlkxV2czYW5RN2lqemg4eUViZDV6RjdKNXdFcUlvcVhoNkJ6eVJkR1pua1hnNzQwOEs2TXJYSlpGcW9qRDU2QjBOWFFtdXBJRkRKbmdZUF9ZSmRPVEtvUjVhLTV1NjdXQjRhS0duaEtJb2FrQnNjUTRvdFMxdkdTNk1NYlFHUFhhYTJ1eUN3WHN4UlJ4UjdrZjY0SzFGYWVFN1k0cGJnc1RjNmFUenR4NHljbVhablZSWHZmUVN3cXRHNjhsX1BSZWEzdTJUZFA0S2pTaU9YMnZIQ1ZPcGhWMFJqZkVEMWRMR1h3SnU0Z2FzZ3VGM3puNzdhVjhaQXNIWHFsbjB0TDVYSFdSNV9rdWhUUUhSZHBGYkJIVDB5SDdlMC13QTVnS0g5Qkg5RGNxSGJlelVndUhPcEQ0QkRKMTJTZUM1OXJhVm0zYjU0OVY2dk9MQVBheklIQXpVNW9Yc0ROVjEzaFZTWmVxYlBWMlNlSzladzJ6TmNuMG5FVVZkN1VZN1pfS2ZHa0lQcE80S24wSnQtVlJVV09OVWJ3M09YMkZpV2ktVF9ENHhKU2dfYUQ2aUVyamk0VHJHQmVfVHU4clpUTFoteW5aSWRPV1M0RDRMTms4NGRoYmJfVE82aUl2X3VieVJOdDhBQmRwdzdnRTVBNzZwaW93dUlZb3ZRYUtOeG9ULWxvNVp5a0haSjdkcUhRb3d6UGIxRUpCVkVYX2d6TkRqQVozUWxkNGFoc1FXYVd2YWNkME9Qclo0bjYxMFRWTy1nbnI5NTBJNzRMMDluUXRKYTFqQUN4d0d5aHVlamN3Tkk3NWJXeXR0TW9BeUg5Vnp4Q2RnZUY3b3AtMDlrNmlrSGR0eGRtbUdUd2lFRWg4MklEeWJHN2wwZEpVSXMxNDNOWjRFS0tPdWxhMmFCckhfRENIY184aEFDZXNrRDl2dHQtQW12UnRuQXJjaDJoTUpiYkNWQUtfRG9GMUZoNWM4UnBYZ29RWWs2NHcyUm5kdTF3Vk1GeFpiRUJLaVZ2UGFjbi1jV3lMV0N2ZDl4VERPN295X01NNG56ZjZkRzZoYUtmY1E5NlVXemx2SnVfb19iSXg0R2M3Mjd1a2JRPT0= +Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQnFIc3YtNDZzenJuZEZiQnVMOWRmZjl3R29QOWZRaGlPdk56WG1DR0FSZU5DM3dENWdoMmRpaks1U1VDNDJkZ3d3UXhSbXlkZ2h3SGZfdk54WXVidF82VkdJQXZiRTk0UlhZaUY1b2kwNzNPSm52VFdsdkwtaHJBb2dpRDBVLXRwd19Bb0dUZDkyV1VWZDJ1TG5mZ0ktYXpuS3U1U0JkZUk5TXpMdnhOaUtMN3BIb0pEZ1N0SlpFN3NNby15VTRfWWtxaF9DYjlJcnVKb0ZualVMTUx2aVNGY0JJdE1oZy1xSVBUZDF1aDM0TGVlTzVrNkFHcjlhcEk0SmRIMTFGdDFTMVUxX1dERk9NTXZMb0tVTFRoc20xME1uRkdVV0Z5N200ZTQzSjVsVExoa2VRZmFBU21ZczF0Vm9Ib3BZM2ZneDkwak12UmFyWWd0eng3ZVVFTUFLVzNOazcxeUhLVWUxcEFIZWtNRi1mT29kM1pqNGJJUUh3UVBlNGY3SlotOWZFUk5aQXFXcUFVdnUzc0Z5bERXYUNPbG14VnBNenFvb2tiQ3lZeHNHUVBlQTdTdVdXOEkxaGxCX016WWktWmN2WFcwM0VmVHdvMHVnY212VFE2cjJwUjdENkFCZF9GcUktWWpmWlNXNWVTMHBPdzVxRi15d3FSRDFra2k0NEFmTmpUeVh3SHRuZWE3WGJ4eUNIcE5tdnRqX2NCZnJoMEI2emU4U0ZYN1Nmdlhva1NacFo3UFh3WnpSdGw5ZmNpSGhicFo0ZThReXl3LW9vUzZaMkFHX2lJalFEMWtjZVdqbVpIZGk0cEdEU01TMl9xQkdSNDllTS1GV3lXS0xROTJvSlhaTjlXenJhQ3lOd2p0VjR5ZjEyektUZGJ3UThJOVJuMzhsTTVBVW9BcDFtcjk5Y0pVeW0zX3R0Nk81R3VDRWEzZnRqSXhFUW5ONHFTSWlwQU4yazlDb01KYlFQRjBFVTljdEJIY29WdF9hUkRJOThVTVFfWlJQUXI0Z3RzWFlzR1ZxUWFBd2I1SW1EMWlKdVprT3dKYTlaREp6TkZEZmVsZGEyalZGc3dHaUkyamdmQWtUT2czNzBCZEg0Vk1HSHFpRnhRYzBRNnN3TFkyaE9uMTVXN1VJTmJwbTNUMTdZbVRyc2d6Yl9aaVBXNmFvanROQVhfbWpXTDRlR1RfbklnYnJUQTZPX2JfNnlrWDVDUWJ4Z3YwNXVsTkJFQlRhTG5DVHpwejdsMGl1bzRfRXRTU2dmb3BVMUo4VkQwa0hsTmFBZnVjVzRrQmNzS2R0ZHNGV24yQnktWENtMUp6eG1MQW1ENE1vWFpFUF9PMEpWZVlxX05hSW1QUGlVT1l3MFp4bDBDZVVldHlEUlVCY1VvVlBNTlBhWFlmcVRobDNqRHo0QjZvNDBqVUVKN3JOb2dtYXQxSWw5NERSeEVRdHNUWndzUkY5RjdBOG1FZFRiVTNVSzl5bDNwdTl2SVd5aW5Ub2Q1YlBDRnpBUDkteU44YnV5X05ONmNndm9teUpqaFZVcVlHdGVRcXRpZkJLVnRuMTJSUFhGWndibExqRW03YUJTWXZXUXJ5WXlvd01ISDFuUFpaMFJzNFVQbWRUb2h1Zi1rcXJXMkRQSUFPeWFJN3lzOFc1d3BjWG1kbWlQWGUwelNiSnJXbUpnajdlQTlQR19XNTF0Q3JYcUMzaGp3eU0yZGhKa3FtX0tleHBfekZaWlRJRlZlSzNDVU56cml0TnFJeUc3b09uYVlwbGxFVFR6WFJVMzRmak5yWjBhcjl5ZmJpQ3hpajRXV1dwbDF5N25tNnI2bWtFem1TS08yV3JybUF0enYxRXpkUVdTNVp4WVB0aldJUUN3TnhHcHdMczh5MTFETzNWLXZFSktsdU1vM1JSNXhraDlJRDl0MEhvR1NOQWRaQW1NdzhpZnFVa1hvdXNwY2FvaThHQjVMOXdySnNIcWJlWERfLXVOcHhpN2ZZOW4yVzB3VTI2a3hvVmFkc29aX2ZUZkY5bi04WEV4MTlxNXQ4cTcwaHE4X3hDWkQxelRwSUl2amZOQ0JXRlJjRFhJNVhjNjRmaXp5eG15LTN1MFRvN3BHTFRZQ1ZFVFYyNUxleFpKTHlIVzRnVHk1Y3ZUbV9RUDdqN1Z2M2ZqVG8wa2RoVHJPeENFRDNHV0wwdi1DbEdOVDFJZnRiZGEydlZyM2tQVExOVlo3LXhIUnhZUnB6a2UzZXNtTjR0S2NzUmFNOWNiSHhHTnJDWHowWk1tbVFKUC14M25aQ1hyYjhJM2pxOEtZY0J1WTZrU3l6cDJOdk5iSXpBUk41MFFVellVZFU4UWVDZXFkQnJFbGxQX2J0S3pReU8zZUdsZUgtTnJuSlpfTjdxR3UxWTBEV0JaRV93eE9qa2dNa2tVTHRxMWNyeUh2VWNrYkdKM3BZOURkUlBxUDA3R2M4NnlMTVR2dmNMZi1lZlhzalRJWlFocGRleVRJYXBBY2hCXzFGZEU4ZVFxbHNic3RDV2FYN1dNaWpkaGdwYTEzRkZYRlEtRXR1cERHdnJKX1Zzb1Q0MnVYZkVhb0VYU1JPdFhoV29TMlhTaEppR1lTTURLYmZnNS1pSzl4T1k5MXJ0YV9qX0ZyQ1R6RFFzRndrTW9IUVlxcG5jcTEyYVU3dkpIR0tZZTZiOXNIRFpIalRtUDFBLVNyd1NfNUMtLW52NVpFZGpQenJCOGw0UlJZNlZVT1ZXTm92R3k4c3hTQXFoNFE3TUFHcjRWc01zT082anJZT0laakl5VUk1WDdDaWlubjIwS3RNcjBjTTdpbUNxSmxNR05JaWtEQURlS1h6N2h0NE9CcW5rQ3NXWkwyNXVBUU5mLTU5MG8xX29xZ0t6Z2pKWmhMNG1BNXBhYWkzY0loSmluUXNKdURwQWRIV2laM2dHQTFxV19lbkZXWmdfWEdiWEZsMGVIWDdoMnJ5dzM0ZGtBM3BSRVp2QzFNbFJSWXBManN5WmFVMlp6aUpWMF9jMTRPbWptM1lsTE41NG1kUW4tT0ZqTzNaZnZ5ZzBLZzNNc1N1X2FMMVJ0N3o4a25LMkxKVUE0dTNhU3hZX3RFMUtKcEgtX1B0cTdEMmYyMzdPaEhoeWhaUGRITC11NzRWYTJnZldiUkFvdG95a1RwWnNKaERkT0kxN1RJMzZQZzFiSjl1SlJieTJjaHBMYmZDUlhTT2hvQnRPaTNhS3NzaVc1Tms0X0FyUHRsSXdCLW1OUWk1RkRKc3pqSjVQTFFROEN5M3pxUGVjZHI4SVM3Qmx1S1A2bEEzNWlVWkFndGpUSm4wcV9jRjQ5T0l1c3ZqN0w3Z1dMV2ZtbU9MbTVSOXphX3VLMko2ZEs3U0NIaFFIMVFIcnN0OGIxSjdxNGlHUHRnOEJDaGwzcXJYNFBnOGdFSVFuSGUyOWJ3WmtlVGhGQWk0THdZd1hUbGRydk83SWVzWUJrb21tSlNvVkJjdWYtcWo0aEc1Ri1XNTZoSENaRWJISmp3UlJNMU9vSnNzZ0VudXpxMDA3aGdfSDBNZlA0Y1gybkF4dGl6SzFOc1VMN0dzVkQxVllkSDhyby12SWNxTFRYdThJUm13S3p3cGFYc05TbVc2YVNtZEdCOFBCUXhadkIzNmdkbXpnc1pLYUhzOEtsY2kxVmNYZm9wOS1LOERLRHJhY2VhanNjaThUZW1rS01wUW05SFJxOGd1VF9STlJZWDRiTV92dXlQTkdxN3BYYTN1SUhRSjRNTy1PZWpGd0xhUlVES0hiWE5LUkM5dHNvenR3TVMySC1ueUZXUkxFY2VyRmhISGc2U2ZxeXY2VkJULV9pOTU1QkI5VUNndnVQcVItTW96VTBqRTdzem1IQ1UxVWtWdjhvTERFeGJ6M3dJNERUV1BTeUlRcG1fbUVjQ0lNREF5QkpLeHJHRkFxQS1kZEE4bXJ2aVVSckVoTkZwNGtoRElIcUktQjA1bkNRclM4dWlqUVRXXzdlQ0VjQWZGSTZlR01NQmU5bHQ3bGNtZWU1eHVvRVdQRVU4Rmx0OFRTaWF3cGgyeFJoM25sRk1GNXJtdEpfcEJmYVFrZXd4eXl0c0ZKVjQ3MkFNRjh5bDBTbFZNd256dmxpQlo5Z1FRM1ZmVTJSb3VrZTk3cXVQYmZ6SnNUWGhlSUhrUjVWUHFwemNmbW1scWVxTkcxT1p5dVlvUjhCSVJaSnBjU0dpc3YzVkt1WUtrd2xoQlVNQXh1eDhmTXNISWMyUnBUMmIwamxlS0tjMVRiWDlBcE03b1BHR1FmdmlsX2ZlMTNCaFNvNG1TeTNiQXRNZ2Y1eE1IaFAxTUZGZ1YyZjEzTG9PaGRCdHJzVlB5Mm12T1NiX2RyT2d2RERCRWFHT0dadW5DZjNtdXE4cHhEQlpub2l3bz0= # Teamsbot Browser Bot Service TEAMSBOT_BROWSER_BOT_URL = http://teamsbot.poweron.swiss:4100 diff --git a/modules/aicore/aicoreModelRegistry.py b/modules/aicore/aicoreModelRegistry.py index 1c50651d..813b3ac4 100644 --- a/modules/aicore/aicoreModelRegistry.py +++ b/modules/aicore/aicoreModelRegistry.py @@ -341,8 +341,8 @@ class ModelRegistry: modelRegistry = ModelRegistry() # Eager pre-warm on first import: ensures connectors are ready in this process. -# Critical for chatbot performance — avoids 4–8 s latency on first request. -# Runs when this module is first imported (lifespan or first chatbot request). +# Critical for AI/agent performance — avoids 4–8 s latency on first request. +# Runs when this module is first imported (lifespan or first AI request). def _eager_prewarm() -> None: try: modelRegistry.ensureConnectorsRegistered() diff --git a/modules/aicore/aicorePluginPrivateLlm.py b/modules/aicore/aicorePluginPrivateLlm.py index 20705e56..988ae9e4 100644 --- a/modules/aicore/aicorePluginPrivateLlm.py +++ b/modules/aicore/aicorePluginPrivateLlm.py @@ -16,11 +16,7 @@ Models (next-gen — RTX PRO 6000 96 GB, auto-activated when pulled in Ollama): - poweron-vision-general: Vision (llama4:scout); multimodal, long-context documents - poweron-embed: Embedding (nomic-embed-text); local RAG embedding -Pricing (CHF per call): -- Text models: CHF 0.010 -- Vision models: CHF 0.100 -- Reasoning models: CHF 0.050 -- Embedding: CHF 0.000 (flat rate) +Pricing: byte-based (~per-token via bytes/4), configured via the PRICE_* constants below. """ import logging @@ -43,11 +39,20 @@ from modules.datamodels.datamodelAi import ( # Configure logger logger = logging.getLogger(__name__) -# Pricing constants (CHF) -PRICE_TEXT_PER_CALL = 0.01 # CHF 0.010 per text model call -PRICE_VISION_PER_CALL = 0.10 # CHF 0.100 per vision model call -PRICE_REASONING_PER_CALL = 0.05 # CHF 0.050 per reasoning call (longer runtime) -PRICE_EMBED_PER_CALL = 0.00 # CHF 0.000 flat rate (local embedding) +# Pricing constants (CHF per 1k tokens; billed byte-based via bytes/4 ~ 1 token) +PRICE_INPUT_PER_1K = 0.0075 +PRICE_OUTPUT_PER_1K = 0.0375 +PRICE_EMBED_PER_1K = 0.0005 + + +def _calcPrivatePriceCHF(processingTime, bytesSent, bytesReceived): + """Byte-based price for private text/vision/reasoning models.""" + return (bytesSent / 4 / 1000) * PRICE_INPUT_PER_1K + (bytesReceived / 4 / 1000) * PRICE_OUTPUT_PER_1K + + +def _calcPrivateEmbedPriceCHF(processingTime, bytesSent, bytesReceived): + """Byte-based price for private embedding (input only).""" + return (bytesSent / 4 / 1000) * PRICE_EMBED_PER_1K # Private-LLM Service URL (fix, nicht via env konfigurierbar) @@ -242,8 +247,8 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.1, maxTokens=4096, contextLength=8192, # Reduced for RAM constraints - costPer1kTokensInput=0.0, # Flat rate pricing - costPer1kTokensOutput=0.0, # Flat rate pricing + costPer1kTokensInput=PRICE_INPUT_PER_1K, + costPer1kTokensOutput=PRICE_OUTPUT_PER_1K, speedRating=8, # Fast and efficient qualityRating=9, # High quality text model functionCall=self.callAiText, @@ -259,7 +264,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.AGENT, 8), ), version="qwen2.5:7b", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_TEXT_PER_CALL + calculatepriceCHF=_calcPrivatePriceCHF ), "ollamaModel": "qwen2.5:7b" }, @@ -273,8 +278,8 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.2, maxTokens=2048, contextLength=4096, # Reduced for RAM constraints (vision needs more) - costPer1kTokensInput=0.0, # Flat rate pricing - costPer1kTokensOutput=0.0, # Flat rate pricing + costPer1kTokensInput=PRICE_INPUT_PER_1K, + costPer1kTokensOutput=PRICE_OUTPUT_PER_1K, speedRating=7, qualityRating=9, functionCall=self.callAiVision, @@ -285,7 +290,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.NEUTRALIZATION_IMAGE, 9), ), version="qwen2.5vl:7b", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_VISION_PER_CALL + calculatepriceCHF=_calcPrivatePriceCHF ), "ollamaModel": "qwen2.5vl:7b" }, @@ -299,8 +304,8 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.1, maxTokens=2048, contextLength=4096, # Reduced for RAM constraints - costPer1kTokensInput=0.0, # Flat rate pricing - costPer1kTokensOutput=0.0, # Flat rate pricing + costPer1kTokensInput=PRICE_INPUT_PER_1K, + costPer1kTokensOutput=PRICE_OUTPUT_PER_1K, speedRating=9, # Fast due to small 2B model qualityRating=8, # Good for document understanding functionCall=self.callAiVision, @@ -311,7 +316,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.NEUTRALIZATION_IMAGE, 9), ), version="granite3.2-vision", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_VISION_PER_CALL + calculatepriceCHF=_calcPrivatePriceCHF ), "ollamaModel": "granite3.2-vision" }, @@ -326,8 +331,8 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.1, maxTokens=8192, contextLength=65536, - costPer1kTokensInput=0.0, - costPer1kTokensOutput=0.0, + costPer1kTokensInput=PRICE_INPUT_PER_1K, + costPer1kTokensOutput=PRICE_OUTPUT_PER_1K, speedRating=5, qualityRating=10, functionCall=self.callAiText, @@ -342,7 +347,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.AGENT, 9), ), version="deepseek-r1:70b", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_REASONING_PER_CALL + calculatepriceCHF=_calcPrivatePriceCHF ), "ollamaModel": "deepseek-r1:70b" }, @@ -356,8 +361,8 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.2, maxTokens=4096, contextLength=131072, - costPer1kTokensInput=0.0, - costPer1kTokensOutput=0.0, + costPer1kTokensInput=PRICE_INPUT_PER_1K, + costPer1kTokensOutput=PRICE_OUTPUT_PER_1K, speedRating=7, qualityRating=10, functionCall=self.callAiVision, @@ -368,7 +373,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.NEUTRALIZATION_IMAGE, 10), ), version="llama4:scout", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_VISION_PER_CALL + calculatepriceCHF=_calcPrivatePriceCHF ), "ollamaModel": "llama4:scout" }, @@ -382,7 +387,7 @@ class AiPrivateLlm(BaseConnectorAi): temperature=0.0, maxTokens=0, contextLength=8192, - costPer1kTokensInput=0.0, + costPer1kTokensInput=PRICE_EMBED_PER_1K, costPer1kTokensOutput=0.0, speedRating=10, qualityRating=8, @@ -393,7 +398,7 @@ class AiPrivateLlm(BaseConnectorAi): (OperationTypeEnum.EMBEDDING, 9), ), version="nomic-embed-text", - calculatepriceCHF=lambda processingTime, bytesSent, bytesReceived: PRICE_EMBED_PER_CALL + calculatepriceCHF=_calcPrivateEmbedPriceCHF ), "ollamaModel": "nomic-embed-text" }, diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 2126a65c..1f37e24a 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -230,6 +230,7 @@ _CONNECT_TIMEOUT_S = 10 # `_BORROW_WAIT_TIMEOUT_S` seconds before giving up. _BORROW_WAIT_TIMEOUT_S = 30.0 _BORROW_WAIT_BACKOFF_S = 0.05 +_shuttingDown = False def _resolvePoolMax() -> int: @@ -315,7 +316,13 @@ class _PoolRegistry: def closeAllPools() -> None: - """Public entry point for FastAPI lifespan shutdown hook.""" + """Public entry point for FastAPI lifespan shutdown hook. + + Sets the shutdown flag first so that any in-flight ``_acquireConn`` loops + abort immediately instead of polling for up to 30 s. + """ + global _shuttingDown + _shuttingDown = True _PoolRegistry.closeAll() @@ -590,7 +597,10 @@ class DatabaseConnector: psycopg2's pool throws on exhaustion instead of queueing — this helper polls with a short backoff so callers see queue semantics. + Aborts immediately when the application is shutting down. """ + if _shuttingDown: + raise psycopg2.pool.PoolError("Application is shutting down") deadline = time.monotonic() + _BORROW_WAIT_TIMEOUT_S attempt = 0 while True: @@ -598,6 +608,8 @@ class DatabaseConnector: return pool.getconn() except psycopg2.pool.PoolError as e: attempt += 1 + if _shuttingDown: + raise psycopg2.pool.PoolError("Application is shutting down") if time.monotonic() >= deadline: logger.error( "Connection pool exhausted after %.1fs wait (%d retries)", diff --git a/modules/connectors/providerGoogle/connectorGoogle.py b/modules/connectors/providerGoogle/connectorGoogle.py index 46fc6c54..1c5826de 100644 --- a/modules/connectors/providerGoogle/connectorGoogle.py +++ b/modules/connectors/providerGoogle/connectorGoogle.py @@ -2,7 +2,9 @@ # All rights reserved. """Google ProviderConnector -- Drive and Gmail via Google OAuth.""" +import asyncio import logging +import urllib.parse from typing import Any, Dict, List, Optional import aiohttp @@ -18,6 +20,41 @@ _CALENDAR_BASE = "https://www.googleapis.com/calendar/v3" _PEOPLE_BASE = "https://people.googleapis.com/v1" +def _parseGoogleDateRange(text: Optional[str]) -> tuple: + """Parse a date range from a filter/query string for Calendar timeMin/timeMax. + + Supports two ISO dates, a single ISO date (~31 day window) or a YYYY-MM + month pattern. Returns RFC3339 UTC strings (timeMin, timeMax) or (None, None). + """ + import re + from datetime import datetime, timedelta + if not text: + return (None, None) + + def _toRfc3339(value: str) -> str: + value = value.strip().rstrip("Z") + if "T" not in value: + value = f"{value}T00:00:00" + return f"{value}Z" + + isoMatch = re.findall(r'\d{4}-\d{2}-\d{2}(?:T[\d:]+)?', text) + if len(isoMatch) >= 2: + return (_toRfc3339(isoMatch[0]), _toRfc3339(isoMatch[1])) + if len(isoMatch) == 1: + try: + dt = datetime.fromisoformat(isoMatch[0]) + return (_toRfc3339(isoMatch[0]), _toRfc3339((dt + timedelta(days=31)).strftime('%Y-%m-%dT00:00:00'))) + except ValueError: + pass + monthMatch = re.match(r'^(\d{4})-(\d{2})$', text.strip()) + if monthMatch: + year, month = int(monthMatch.group(1)), int(monthMatch.group(2)) + start = f"{year}-{month:02d}-01T00:00:00" + end = f"{year + 1}-01-01T00:00:00" if month == 12 else f"{year}-{month + 1:02d}-01T00:00:00" + return (_toRfc3339(start), _toRfc3339(end)) + return (None, None) + + async def _googleGet(token: str, url: str) -> Dict[str, Any]: headers = {"Authorization": f"Bearer {token}"} timeout = aiohttp.ClientTimeout(total=20) @@ -33,6 +70,17 @@ async def _googleGet(token: str, url: str) -> Dict[str, Any]: return {"error": str(e)} +def _raiseGoogleError(result: Dict[str, Any], ctx: str) -> None: + """Raise a clear error for a failed Google API response. + + Browse/search must NOT swallow API failures into an empty result list, which + masks a real error as 'empty'. Callers wrap these in try/except. + """ + err = result.get("error") if isinstance(result, dict) else None + logger.warning("Google error (%s): %s", ctx, err or result) + raise RuntimeError(f"Google error ({ctx}): {err or result}") + + class DriveAdapter(ServiceAdapter): """Google Drive ServiceAdapter -- browse files and folders.""" @@ -53,8 +101,7 @@ class DriveAdapter(ServiceAdapter): result = await _googleGet(self._token, url) if "error" in result: - logger.warning(f"Google Drive browse failed: {result['error']}") - return [] + _raiseGoogleError(result, "Google Drive browse") entries = [] for f in result.get("files", []): @@ -125,27 +172,51 @@ class DriveAdapter(ServiceAdapter): path: Optional[str] = None, limit: Optional[int] = None, ) -> List[ExternalEntry]: - safeQuery = query.replace("'", "\\'") + safeQuery = query.replace("\\", "\\\\").replace("'", "\\'") folderId = (path or "").strip("/") - qParts = [f"name contains '{safeQuery}'", "trashed=false"] + # `fullText contains` matches file name AND content (and some metadata), + # which is what users expect from a search -- not just the file name. + qParts = [f"fullText contains '{safeQuery}'", "trashed=false"] if folderId: qParts.append(f"'{folderId}' in parents") qStr = " and ".join(qParts) - pageSize = max(1, min(int(limit or 100), 1000)) - url = f"{_DRIVE_BASE}/files?q={qStr}&fields=files(id,name,mimeType,size)&pageSize={pageSize}" + effectiveLimit = max(1, int(limit)) if limit is not None else None + pageSize = min(effectiveLimit or 100, 1000) logger.debug(f"Google Drive search: q={qStr}") - result = await _googleGet(self._token, url) - if "error" in result: - return [] - return [ - ExternalEntry( - name=f.get("name", ""), - path=f"/{f.get('id', '')}", - isFolder=f.get("mimeType") == "application/vnd.google-apps.folder", - size=int(f.get("size", 0)) if f.get("size") else None, - ) - for f in result.get("files", []) - ] + entries: List[ExternalEntry] = [] + pageToken: Optional[str] = None + hardCap = effectiveLimit or 1000 + while len(entries) < hardCap: + params = { + "q": qStr, + "fields": "nextPageToken,files(id,name,mimeType,size,modifiedTime)", + "pageSize": str(pageSize), + } + if pageToken: + params["pageToken"] = pageToken + url = f"{_DRIVE_BASE}/files?{urllib.parse.urlencode(params)}" + result = await _googleGet(self._token, url) + if "error" in result: + if not entries: + _raiseGoogleError(result, "Google Drive search") + break + for f in result.get("files", []): + entries.append(ExternalEntry( + name=f.get("name", ""), + path=f"/{f.get('id', '')}", + isFolder=f.get("mimeType") == "application/vnd.google-apps.folder", + size=int(f.get("size", 0)) if f.get("size") else None, + mimeType=f.get("mimeType"), + metadata={"id": f.get("id"), "modifiedTime": f.get("modifiedTime")}, + )) + if len(entries) >= hardCap: + break + pageToken = result.get("nextPageToken") + if not pageToken: + break + if effectiveLimit is not None: + entries = entries[:effectiveLimit] + return entries class GmailAdapter(ServiceAdapter): @@ -155,7 +226,8 @@ class GmailAdapter(ServiceAdapter): self._token = accessToken _DEFAULT_MESSAGE_LIMIT = 100 - _MAX_MESSAGE_LIMIT = 500 + _MAX_MESSAGE_LIMIT = 1000 + _METADATA_FETCH_CAP = 200 async def browse( self, @@ -169,8 +241,7 @@ class GmailAdapter(ServiceAdapter): url = f"{_GMAIL_BASE}/users/me/labels" result = await _googleGet(self._token, url) if "error" in result: - logger.warning(f"Gmail labels failed: {result['error']}") - return [] + _raiseGoogleError(result, "Gmail labels") _SYSTEM_LABELS = {"INBOX", "SENT", "DRAFT", "TRASH", "SPAM", "STARRED", "IMPORTANT"} labels = [] for lbl in result.get("labels", []): @@ -188,23 +259,116 @@ class GmailAdapter(ServiceAdapter): return labels effectiveLimit = self._DEFAULT_MESSAGE_LIMIT if limit is None else max(1, min(int(limit), self._MAX_MESSAGE_LIMIT)) - url = f"{_GMAIL_BASE}/users/me/messages?labelIds={cleanPath}&maxResults={effectiveLimit}" - result = await _googleGet(self._token, url) - if "error" in result: - return [] + labelId = await self._resolveLabelId(cleanPath) + if not labelId: + raise ValueError( + f"Gmail label not found: '{cleanPath}'. Browse the mailbox root ('/') " + f"to list available labels." + ) + msgIds, totalEstimate = await self._listMessageIds( + params={"labelIds": labelId}, limit=effectiveLimit, + ) + entries = await self._fetchMessageEntries( + msgIds[:self._METADATA_FETCH_CAP], labelPath=labelId, + ) + if totalEstimate and totalEstimate > len(msgIds): + entries.append(ExternalEntry( + name=f"(~{totalEstimate} total messages estimated, {len(msgIds)} listed)", + path=f"/{labelId}/_count", isFolder=False, + metadata={"totalEstimate": totalEstimate, "listed": len(msgIds)}, + )) + elif len(msgIds) > self._METADATA_FETCH_CAP: + entries.append(ExternalEntry( + name=f"({len(msgIds)} messages listed, metadata shown for first {self._METADATA_FETCH_CAP})", + path=f"/{labelId}/_count", isFolder=False, + metadata={"listed": len(msgIds), "metadataShown": self._METADATA_FETCH_CAP}, + )) + return entries - entries = [] - for msg in result.get("messages", [])[:effectiveLimit]: - msgId = msg.get("id", "") - detailUrl = f"{_GMAIL_BASE}/users/me/messages/{msgId}?format=metadata&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=Date" + async def _resolveLabelId(self, ref: str) -> Optional[str]: + """Resolve a Gmail label reference (display name / system name / id) to a + label id. Returns None if nothing matches so the caller can raise a clear + error instead of querying with an invalid label.""" + if not ref: + return None + r = ref.strip() + result = await _googleGet(self._token, f"{_GMAIL_BASE}/users/me/labels") + if "error" in result: + _raiseGoogleError(result, "Gmail labels") + labels = result.get("labels", []) + # 1) exact id match (already-resolved id passes through) + for lbl in labels: + if lbl.get("id") == r: + return r + # 2) case-insensitive display-name match + for lbl in labels: + if (lbl.get("name") or "").strip().lower() == r.lower(): + return lbl.get("id") + # 3) system label by uppercased name (INBOX, SENT, ...) + up = r.upper() + for lbl in labels: + if lbl.get("id") == up: + return up + return None + + async def _listMessageIds( + self, params: Dict[str, str], limit: int, + ) -> tuple: + """Page through ``messages.list`` and return (msgIds, totalEstimate). + + Gmail's ``maxResults`` caps at 500 per page, so we follow + ``nextPageToken`` until we have ``limit`` ids or there are no more pages. + ``resultSizeEstimate`` from the first page gives the agent an approximate + total count without having to download every message. + """ + msgIds: List[str] = [] + totalEstimate: Optional[int] = None + pageToken: Optional[str] = None + pageSize = min(limit, 500) + while len(msgIds) < limit: + p = {**params, "maxResults": str(pageSize)} + if pageToken: + p["pageToken"] = pageToken + url = f"{_GMAIL_BASE}/users/me/messages?{urllib.parse.urlencode(p)}" + result = await _googleGet(self._token, url) + if "error" in result: + if not msgIds: + _raiseGoogleError(result, "Gmail list messages") + break + if totalEstimate is None: + totalEstimate = result.get("resultSizeEstimate") + for m in result.get("messages", []): + mid = m.get("id", "") + if mid: + msgIds.append(mid) + if len(msgIds) >= limit: + break + pageToken = result.get("nextPageToken") + if not pageToken: + break + return msgIds, totalEstimate + + async def _fetchMessageEntries(self, msgIds: List[str], labelPath: str = "") -> List[ExternalEntry]: + """Resolve a list of Gmail message ids into ExternalEntries with + Subject/From/Date metadata. Detail fetches run concurrently to avoid a + slow sequential N+1 round-trip per message.""" + if not msgIds: + return [] + pathPrefix = f"/{labelPath}" if labelPath else "" + + async def _one(msgId: str) -> ExternalEntry: + detailUrl = ( + f"{_GMAIL_BASE}/users/me/messages/{msgId}" + f"?format=metadata&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=Date" + ) detail = await _googleGet(self._token, detailUrl) if "error" in detail: - entries.append(ExternalEntry(name=f"Message {msgId}", path=f"/{cleanPath}/{msgId}", isFolder=False)) - continue + return ExternalEntry(name=f"Message {msgId}", path=f"{pathPrefix}/{msgId}", isFolder=False, + metadata={"id": msgId}) headers = {h.get("name", ""): h.get("value", "") for h in detail.get("payload", {}).get("headers", [])} - entries.append(ExternalEntry( + return ExternalEntry( name=headers.get("Subject", "(no subject)"), - path=f"/{cleanPath}/{msgId}", + path=f"{pathPrefix}/{msgId}", isFolder=False, metadata={ "id": msgId, @@ -212,8 +376,9 @@ class GmailAdapter(ServiceAdapter): "date": headers.get("Date", ""), "snippet": detail.get("snippet", ""), }, - )) - return entries + ) + + return list(await asyncio.gather(*[_one(mid) for mid in msgIds])) async def download(self, path: str) -> DownloadResult: """Download a Gmail message as RFC 822 EML via format=raw.""" @@ -261,19 +426,34 @@ class GmailAdapter(ServiceAdapter): limit: Optional[int] = None, ) -> list: effectiveLimit = self._DEFAULT_MESSAGE_LIMIT if limit is None else max(1, min(int(limit), self._MAX_MESSAGE_LIMIT)) - url = f"{_GMAIL_BASE}/users/me/messages?q={query}&maxResults={effectiveLimit}" - result = await _googleGet(self._token, url) - if "error" in result: - return [] - return [ - ExternalEntry( - name=f"Message {m.get('id', '')}", - path=f"/{m.get('id', '')}", - isFolder=False, - metadata={"id": m.get("id")}, - ) - for m in result.get("messages", []) - ] + params: Dict[str, str] = {"q": query} + labelPath = (path or "").strip("/") + if labelPath: + labelId = await self._resolveLabelId(labelPath) + if not labelId: + raise ValueError( + f"Gmail label not found: '{labelPath}'. Browse the mailbox root ('/') " + f"to list available labels, or search without a label scope." + ) + labelPath = labelId + params["labelIds"] = labelId + msgIds, totalEstimate = await self._listMessageIds(params, limit=effectiveLimit) + entries = await self._fetchMessageEntries( + msgIds[:self._METADATA_FETCH_CAP], labelPath=labelPath, + ) + if totalEstimate and totalEstimate > len(msgIds): + entries.append(ExternalEntry( + name=f"(~{totalEstimate} total results estimated, {len(msgIds)} listed)", + path=f"/{labelPath or 'search'}/_count", isFolder=False, + metadata={"totalEstimate": totalEstimate, "listed": len(msgIds)}, + )) + elif len(msgIds) > self._METADATA_FETCH_CAP: + entries.append(ExternalEntry( + name=f"({len(msgIds)} results listed, metadata shown for first {self._METADATA_FETCH_CAP})", + path=f"/{labelPath or 'search'}/_count", isFolder=False, + metadata={"listed": len(msgIds), "metadataShown": self._METADATA_FETCH_CAP}, + )) + return entries class CalendarAdapter(ServiceAdapter): @@ -302,8 +482,7 @@ class CalendarAdapter(ServiceAdapter): url = f"{_CALENDAR_BASE}/users/me/calendarList?maxResults=250" result = await _googleGet(self._token, url) if "error" in result: - logger.warning(f"Google Calendar list failed: {result['error']}") - return [] + _raiseGoogleError(result, "Google Calendar list") calendars = result.get("items", []) if filter: f = filter.lower() @@ -331,10 +510,14 @@ class CalendarAdapter(ServiceAdapter): f"{_CALENDAR_BASE}/calendars/{quote(calendarId, safe='')}/events" f"?maxResults={effectiveLimit}&orderBy=startTime&singleEvents=true" ) + # Restrict to a date window when the filter is a date range, so large + # multi-year calendars only return the relevant period. + timeMin, timeMax = _parseGoogleDateRange(filter) + if timeMin and timeMax: + url += f"&timeMin={quote(timeMin, safe='')}&timeMax={quote(timeMax, safe='')}" result = await _googleGet(self._token, url) if "error" in result: - logger.warning(f"Google Calendar events failed: {result['error']}") - return [] + _raiseGoogleError(result, "Google Calendar events") events = result.get("items", []) return [ ExternalEntry( @@ -387,13 +570,23 @@ class CalendarAdapter(ServiceAdapter): from urllib.parse import quote calendarId = (path or "").strip("/").split("/", 1)[0] or "primary" effectiveLimit = self._DEFAULT_EVENT_LIMIT if limit is None else max(1, min(int(limit), self._MAX_EVENT_LIMIT)) - url = ( - f"{_CALENDAR_BASE}/calendars/{quote(calendarId, safe='')}/events" - f"?q={quote(query, safe='')}&maxResults={effectiveLimit}&singleEvents=true" - ) + # A date-range query maps to timeMin/timeMax (efficient window fetch); + # otherwise fall back to the free-text q parameter. + timeMin, timeMax = _parseGoogleDateRange(query) + if timeMin and timeMax: + url = ( + f"{_CALENDAR_BASE}/calendars/{quote(calendarId, safe='')}/events" + f"?timeMin={quote(timeMin, safe='')}&timeMax={quote(timeMax, safe='')}" + f"&maxResults={effectiveLimit}&orderBy=startTime&singleEvents=true" + ) + else: + url = ( + f"{_CALENDAR_BASE}/calendars/{quote(calendarId, safe='')}/events" + f"?q={quote(query, safe='')}&maxResults={effectiveLimit}&singleEvents=true" + ) result = await _googleGet(self._token, url) if "error" in result: - return [] + _raiseGoogleError(result, "Google Calendar search") return [ ExternalEntry( name=ev.get("summary", "(no title)"), @@ -479,8 +672,7 @@ class ContactsAdapter(ServiceAdapter): ) result = await _googleGet(self._token, url) if "error" in result: - logger.warning(f"Google People connections failed: {result['error']}") - return [] + _raiseGoogleError(result, "Google People connections") people = result.get("connections", []) else: groupResource = groupRef @@ -490,8 +682,7 @@ class ContactsAdapter(ServiceAdapter): ) grpResult = await _googleGet(self._token, grpUrl) if "error" in grpResult: - logger.warning(f"Google contactGroup detail failed: {grpResult['error']}") - return [] + _raiseGoogleError(grpResult, "Google contactGroup detail") memberResourceNames = grpResult.get("memberResourceNames") or [] if not memberResourceNames: return [] @@ -568,7 +759,7 @@ class ContactsAdapter(ServiceAdapter): ) result = await _googleGet(self._token, url) if "error" in result: - return [] + _raiseGoogleError(result, "Google Contacts search") entries: List[ExternalEntry] = [] for r in result.get("results", []): p = r.get("person") or {} @@ -581,6 +772,8 @@ class ContactsAdapter(ServiceAdapter): metadata={ "id": p.get("resourceName"), "emails": [e.get("value") for e in (p.get("emailAddresses") or []) if e.get("value")], + "phones": [pn.get("value") for pn in (p.get("phoneNumbers") or []) if pn.get("value")], + "organization": (p.get("organizations") or [{}])[0].get("name") if p.get("organizations") else None, }, ) ) diff --git a/modules/connectors/providerInfomaniak/connectorInfomaniak.py b/modules/connectors/providerInfomaniak/connectorInfomaniak.py index f8512893..94a54860 100644 --- a/modules/connectors/providerInfomaniak/connectorInfomaniak.py +++ b/modules/connectors/providerInfomaniak/connectorInfomaniak.py @@ -96,6 +96,17 @@ async def _infomaniakGet( return {"error": str(e)} +def _raiseInfomaniakError(result: Dict[str, Any], ctx: str) -> None: + """Raise a clear error for a failed Infomaniak API response. + + Browse/search must NOT swallow API failures into an empty result list, which + masks a real error as 'empty'. Callers wrap these in try/except. + """ + err = result.get("error") if isinstance(result, dict) else None + logger.warning("Infomaniak error (%s): %s", ctx, err or result) + raise RuntimeError(f"Infomaniak error ({ctx}): {err or result}") + + async def _infomaniakDownload( token: str, endpoint: str, @@ -358,10 +369,7 @@ class KdriveAdapter(ServiceAdapter): result = await _infomaniakGet(self._token, endpoint) if isinstance(result, dict) and result.get("error"): - logger.warning( - f"kDrive list-children {driveId}/{fileId or 'root'} failed: {result['error']}" - ) - return [] + _raiseInfomaniakError(result, f"kDrive list-children {driveId}/{fileId or 'root'}") data = _unwrapData(result) items = data if isinstance(data, list) else data.get("items", []) if isinstance(data, dict) else [] @@ -426,7 +434,7 @@ class KdriveAdapter(ServiceAdapter): endpoint = f"/2/drive/{driveId}/files/search?query={query}&per_page={pageSize}" result = await _infomaniakGet(self._token, endpoint) if isinstance(result, dict) and result.get("error"): - return [] + _raiseInfomaniakError(result, "kDrive search") data = _unwrapData(result) items = data if isinstance(data, list) else data.get("items", []) if isinstance(data, dict) else [] @@ -495,7 +503,7 @@ class CalendarAdapter(ServiceAdapter): if not segments: return await self._listCalendars() if len(segments) == 1: - return await self._listEvents(segments[0], limit=limit) + return await self._listEvents(segments[0], limit=limit, filter=filter) return [] async def _listCalendars(self) -> List[ExternalEntry]: @@ -503,8 +511,7 @@ class CalendarAdapter(ServiceAdapter): self._token, f"{_PIM_PREFIX}/calendar", baseUrl=_CALENDAR_BASE ) if isinstance(result, dict) and result.get("error"): - logger.warning(f"Calendar list-calendars failed: {result['error']}") - return [] + _raiseInfomaniakError(result, "Calendar list-calendars") data = _unwrapData(result) calendars = data.get("calendars", []) if isinstance(data, dict) else [] entries: List[ExternalEntry] = [] @@ -527,18 +534,64 @@ class CalendarAdapter(ServiceAdapter): )) return entries - def _eventWindow(self) -> tuple: + def _eventWindow(self, filter: Optional[str] = None) -> tuple: + # Honour an explicit date range from the agent (e.g. "2026-06" or + # "2026-06-01 2026-06-30"), clamped to the vendor's <3 month limit. + # Otherwise fall back to the default 90-day browsing window. + rng = self._parseFilterWindow(filter) + if rng: + return rng now = datetime.now(timezone.utc) fromStr = (now - timedelta(days=self._PAST_DAYS)).strftime("%Y-%m-%d %H:%M:%S") toStr = (now + timedelta(days=self._FUTURE_DAYS)).strftime("%Y-%m-%d %H:%M:%S") return fromStr, toStr + @staticmethod + def _parseFilterWindow(filter: Optional[str]) -> Optional[tuple]: + """Parse a date range from a filter string into Infomaniak's + 'Y-m-d H:i:s' from/to window, clamped to <3 months. Returns None when + the filter is not a parseable date range.""" + if not filter: + return None + iso = re.findall(r'\d{4}-\d{2}-\d{2}', filter) + start = end = None + if len(iso) >= 2: + start, end = iso[0], iso[1] + elif len(iso) == 1: + start = iso[0] + else: + month = re.match(r'^(\d{4})-(\d{2})$', filter.strip()) + if not month: + return None + year, mon = int(month.group(1)), int(month.group(2)) + start = f"{year}-{mon:02d}-01" + end = f"{year + 1}-01-01" if mon == 12 else f"{year}-{mon + 1:02d}-01" + try: + startDt = datetime.fromisoformat(start) + except ValueError: + return None + if end: + try: + endDt = datetime.fromisoformat(end) + except ValueError: + endDt = startDt + timedelta(days=31) + else: + endDt = startDt + timedelta(days=31) + # Clamp to vendor limit (<3 months). + if endDt - startDt > timedelta(days=85): + endDt = startDt + timedelta(days=85) + return ( + startDt.strftime("%Y-%m-%d %H:%M:%S"), + endDt.strftime("%Y-%m-%d %H:%M:%S"), + ) + async def _listEvents( self, calendarId: str, limit: Optional[int], + filter: Optional[str] = None, ) -> List[ExternalEntry]: - fromStr, toStr = self._eventWindow() + fromStr, toStr = self._eventWindow(filter) endpoint = ( f"{_PIM_PREFIX}/event" f"?calendar_id={calendarId}" @@ -547,8 +600,7 @@ class CalendarAdapter(ServiceAdapter): ) result = await _infomaniakGet(self._token, endpoint, baseUrl=_CALENDAR_BASE) if isinstance(result, dict) and result.get("error"): - logger.warning(f"Calendar list-events {calendarId} failed: {result['error']}") - return [] + _raiseInfomaniakError(result, f"Calendar list-events {calendarId}") data = _unwrapData(result) events = data if isinstance(data, list) else data.get("events", []) if isinstance(data, dict) else [] entries: List[ExternalEntry] = [] @@ -626,11 +678,14 @@ class CalendarAdapter(ServiceAdapter): ) if not calendars: return [] - needle = (query or "").strip().lower() + # A date-range query maps directly to the event window; a free-text + # query keeps the default window and filters on title/location. + dateWindow = self._parseFilterWindow(query) + needle = "" if dateWindow else (query or "").strip().lower() results: List[ExternalEntry] = [] for cal in calendars: calId = (cal.metadata or {}).get("id") or cal.path.strip("/") - for ev in await self._listEvents(calId, limit=limit): + for ev in await self._listEvents(calId, limit=limit, filter=query if dateWindow else None): hay = " ".join( str(v) for v in ( ev.name, @@ -768,8 +823,7 @@ class ContactAdapter(ServiceAdapter): self._token, f"{_PIM_PREFIX}/addressbook", baseUrl=_CONTACTS_BASE ) if isinstance(result, dict) and result.get("error"): - logger.warning(f"Contacts list-addressbooks failed: {result['error']}") - return [] + _raiseInfomaniakError(result, "Contacts list-addressbooks") data = _unwrapData(result) books = data.get("addressbooks", []) if isinstance(data, dict) else [] entries: List[ExternalEntry] = [] @@ -809,10 +863,7 @@ class ContactAdapter(ServiceAdapter): ) result = await _infomaniakGet(self._token, endpoint, baseUrl=_CONTACTS_BASE) if isinstance(result, dict) and result.get("error"): - logger.warning( - f"Contacts list-contacts {addressBookId} failed: {result['error']}" - ) - return [] + _raiseInfomaniakError(result, f"Contacts list-contacts {addressBookId}") data = _unwrapData(result) if isinstance(data, list): return [c for c in data if isinstance(c, dict)] diff --git a/modules/connectors/providerMsft/connectorMsft.py b/modules/connectors/providerMsft/connectorMsft.py index 4b60db55..24f6e25c 100644 --- a/modules/connectors/providerMsft/connectorMsft.py +++ b/modules/connectors/providerMsft/connectorMsft.py @@ -9,6 +9,7 @@ UserConnection (authority=msft). import logging import aiohttp import asyncio +import urllib.parse from typing import Dict, Any, List, Optional from modules.connectors.connectorProviderBase import ProviderConnector, ServiceAdapter, DownloadResult @@ -69,6 +70,8 @@ async def _makeGraphCall( "Authorization": f"Bearer {token}", "Content-Type": contentType, } + if "$count=true" in endpoint: + headers["ConsistencyLevel"] = "eventual" timeout = aiohttp.ClientTimeout(total=30) try: async with aiohttp.ClientSession(timeout=timeout) as session: @@ -124,6 +127,18 @@ def _stripGraphBase(url: str) -> str: return url +def _raiseGraphError(result: Dict[str, Any], ctx: str) -> None: + """Raise a clear error for a failed Graph response. + + Browse/search must NOT swallow API failures into an empty result list, which + makes a real error look like 'empty directory'. Callers (data-source tools, + tree-builder, sync jobs) already wrap these in try/except. + """ + err = result.get("error") if isinstance(result, dict) else None + logger.warning("Graph error (%s): %s", ctx, err or result) + raise RuntimeError(f"Graph error ({ctx}): {err or result}") + + def _graphItemToExternalEntry(item: Dict[str, Any], basePath: str = "") -> ExternalEntry: isFolder = "folder" in item # Graph exposes the driveItem content hash as ``eTag`` (quoted) or @@ -189,7 +204,8 @@ class SharepointAdapter(_GraphApiMixin, ServiceAdapter): while endpoint and len(items) < hardCap: result = await self._graphGet(endpoint) if "error" in result: - logger.warning(f"SharePoint browse failed: {result['error']}") + if not items: + _raiseGraphError(result, "SharePoint browse") break for raw in result.get("value", []) or []: items.append(raw) @@ -211,8 +227,7 @@ class SharepointAdapter(_GraphApiMixin, ServiceAdapter): """Discover accessible SharePoint sites.""" result = await self._graphGet("sites?search=*&$top=50") if "error" in result: - logger.warning(f"SharePoint site discovery failed: {result['error']}") - return [] + _raiseGraphError(result, "SharePoint site discovery") return [ ExternalEntry( name=s.get("displayName") or s.get("name", ""), @@ -253,17 +268,37 @@ class SharepointAdapter(_GraphApiMixin, ServiceAdapter): path: Optional[str] = None, limit: Optional[int] = None, ) -> List[ExternalEntry]: - siteId, _ = _parseSharepointPath(path or "") + siteId, folderPath = _parseSharepointPath(path or "") if not siteId: return [] safeQuery = query.replace("'", "''") - endpoint = f"sites/{siteId}/drive/root/search(q='{safeQuery}')" - result = await self._graphGet(endpoint) - if "error" in result: - return [] - entries = [_graphItemToExternalEntry(item) for item in result.get("value", [])] - if limit is not None: - entries = entries[: max(1, int(limit))] + cleanFolder = (folderPath or "").strip("/") + # Scope the search to the attached folder when one is given, so the agent + # does not get hits from unrelated parts of the site drive. + if cleanFolder: + endpoint: Optional[str] = f"sites/{siteId}/drive/root:/{cleanFolder}:/search(q='{safeQuery}')?$top=200" + else: + endpoint = f"sites/{siteId}/drive/root/search(q='{safeQuery}')?$top=200" + effectiveLimit = int(limit) if limit is not None else None + items: List[Dict[str, Any]] = [] + hardCap = 1000 + while endpoint and len(items) < hardCap: + result = await self._graphGet(endpoint) + if "error" in result: + if not items: + _raiseGraphError(result, "SharePoint search") + break + for raw in result.get("value", []) or []: + items.append(raw) + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + nextLink = result.get("@odata.nextLink") + endpoint = _stripGraphBase(nextLink) if nextLink else None + entries = [_graphItemToExternalEntry(item) for item in items] + if effectiveLimit is not None: + entries = entries[: max(1, effectiveLimit)] return entries @@ -273,6 +308,50 @@ class SharepointAdapter(_GraphApiMixin, ServiceAdapter): _CHARSET_META = '' + +def _parseDateRange(filterStr: Optional[str]) -> tuple: + """Parse a date range from a filter/query string. + + Supports two ISO dates ("2026-06-01 2026-06-30"), a single ISO date + (treated as a ~31 day window), or a YYYY-MM month pattern. Returns + (startDateTime, endDateTime) ISO strings, or (None, None) if not parseable. + """ + import re + from datetime import datetime, timedelta + if not filterStr: + return (None, None) + isoMatch = re.findall(r'\d{4}-\d{2}-\d{2}(?:T[\d:]+)?', filterStr) + if len(isoMatch) >= 2: + return (isoMatch[0], isoMatch[1]) + if len(isoMatch) == 1: + try: + dt = datetime.fromisoformat(isoMatch[0]) + return (isoMatch[0], (dt + timedelta(days=31)).strftime('%Y-%m-%dT00:00:00')) + except ValueError: + pass + monthMatch = re.match(r'^(\d{4})-(\d{2})$', filterStr.strip()) + if monthMatch: + year, month = int(monthMatch.group(1)), int(monthMatch.group(2)) + start = f"{year}-{month:02d}-01T00:00:00" + if month == 12: + end = f"{year + 1}-01-01T00:00:00" + else: + end = f"{year}-{month + 1:02d}-01T00:00:00" + return (start, end) + return (None, None) + + +def _toGraphUtc(isoStr: str) -> str: + """Normalise an ISO date/datetime to a Graph-compatible UTC string + (always 'YYYY-MM-DDTHH:MM:SSZ').""" + if not isoStr: + return isoStr + value = isoStr.strip().rstrip("Z") + if "T" not in value: + value = f"{value}T00:00:00" + return f"{value}Z" + + def _ensureHtmlCharset(html: str) -> str: """Ensure HTML body has a charset meta tag so Outlook renders UTF-8 correctly.""" if "charset" in html.lower(): @@ -350,25 +429,62 @@ class OutlookAdapter(_GraphApiMixin, ServiceAdapter): for f in folders ] - folderId = path.strip("/") + # The incoming path segment may be a display name ("MGB-Ablage"), a + # well-known shortcut ("inbox") or an already-resolved Graph folder id. + # Resolve it to a real id first; otherwise Graph rejects the URL with + # 400 ErrorInvalidIdMalformed. + folderRef = path.strip("/") + folderId = await self._resolveFolderId(folderRef) + if not folderId: + raise ValueError( + f"Outlook folder not found: '{folderRef}'. Browse the mailbox root " + f"(path '/') or call listMailFolders to obtain a valid folder id." + ) effectiveLimit = self._DEFAULT_MESSAGE_LIMIT if limit is None else max(1, min(int(limit), self._MAX_MESSAGE_LIMIT)) pageSize = min(self._PAGE_SIZE, effectiveLimit) - endpoint: Optional[str] = ( - f"me/mailFolders/{folderId}/messages" - f"?$top={pageSize}&$orderby=receivedDateTime desc" - ) + # Optional date-range filter (e.g. "2026-06" or "2026-06-01 2026-06-30") + # so only that period is fetched server-side instead of paging the whole + # folder. Falls back to a plain newest-first listing otherwise. + startDateTime, endDateTime = _parseDateRange(filter) + countParam = "&$count=true" + if startDateTime and endDateTime: + dateFilter = ( + f"receivedDateTime ge {_toGraphUtc(startDateTime)} and " + f"receivedDateTime lt {_toGraphUtc(endDateTime)}" + ) + endpoint: Optional[str] = ( + f"me/mailFolders/{folderId}/messages" + f"?$top={pageSize}&$orderby=receivedDateTime desc" + f"&$filter={urllib.parse.quote(dateFilter)}{countParam}" + ) + else: + endpoint = ( + f"me/mailFolders/{folderId}/messages" + f"?$top={pageSize}&$orderby=receivedDateTime desc{countParam}" + ) messages: List[Dict[str, Any]] = [] + totalCount: Optional[int] = None + firstPage = True while endpoint and len(messages) < effectiveLimit: result = await self._graphGet(endpoint) if "error" in result: + if firstPage: + err = result.get("error") or {} + raise RuntimeError( + f"Graph error listing messages in folder '{folderRef}': " + f"{err.get('message') or err}" + ) break + if firstPage and "@odata.count" in result: + totalCount = result["@odata.count"] + firstPage = False for m in result.get("value", []): messages.append(m) if len(messages) >= effectiveLimit: break nextLink = result.get("@odata.nextLink") endpoint = _stripGraphBase(nextLink) if nextLink else None - return [ + entries = [ ExternalEntry( name=m.get("subject", "(no subject)"), path=f"{path}/{m.get('id', '')}", @@ -382,6 +498,13 @@ class OutlookAdapter(_GraphApiMixin, ServiceAdapter): ) for m in messages ] + if totalCount is not None and totalCount > len(entries): + entries.append(ExternalEntry( + name=f"({totalCount} total messages in folder, {len(entries)} listed)", + path=f"{path}/_count", isFolder=False, + metadata={"totalCount": totalCount, "listed": len(entries)}, + )) + return entries async def download(self, path: str) -> DownloadResult: """Download a mail message as RFC 822 EML via Graph API $value endpoint.""" @@ -412,14 +535,28 @@ class OutlookAdapter(_GraphApiMixin, ServiceAdapter): path: Optional[str] = None, limit: Optional[int] = None, ) -> List[ExternalEntry]: - safeQuery = query.replace("'", "''") + safeQuery = query.replace('"', '\\"') effectiveLimit = self._DEFAULT_MESSAGE_LIMIT if limit is None else max(1, min(int(limit), self._MAX_MESSAGE_LIMIT)) + # Scope the search to the attached folder when one is given, so the agent + # gets hits only from e.g. the Inbox instead of the whole mailbox. Resolve + # the folder reference (display name / well-known / id) to a real id first. + folderRef = (path or "").strip("/") + base = "me/messages" + if folderRef: + folderId = await self._resolveFolderId(folderRef) + if not folderId: + raise ValueError( + f"Outlook folder not found: '{folderRef}'. Call listMailFolders " + f"to obtain a valid folder id, or search without a folder scope." + ) + base = f"me/mailFolders/{folderId}/messages" # NOTE: Graph $search does not support $orderby and may return a single # page (no @odata.nextLink). We still pass $top to lift the implicit 25. - endpoint = f"me/messages?$search=\"{safeQuery}\"&$top={effectiveLimit}" + endpoint = f"{base}?$search=\"{safeQuery}\"&$top={effectiveLimit}" result = await self._graphGet(endpoint) if "error" in result: - return [] + err = result.get("error") or {} + raise RuntimeError(f"Graph error searching mail: {err.get('message') or err}") return [ ExternalEntry( name=m.get("subject", "(no subject)"), @@ -774,8 +911,7 @@ class TeamsAdapter(_GraphApiMixin, ServiceAdapter): if not cleanPath: result = await self._graphGet("me/joinedTeams") if "error" in result: - logger.warning(f"Teams browse failed: {result['error']}") - return [] + _raiseGraphError(result, "Teams browse") return [ ExternalEntry( name=t.get("displayName", ""), @@ -791,7 +927,7 @@ class TeamsAdapter(_GraphApiMixin, ServiceAdapter): if len(parts) == 1: result = await self._graphGet(f"teams/{teamId}/channels") if "error" in result: - return [] + _raiseGraphError(result, "Teams channels") return [ ExternalEntry( name=ch.get("displayName", ""), @@ -834,18 +970,33 @@ class OneDriveAdapter(_GraphApiMixin, ServiceAdapter): ) -> List[ExternalEntry]: cleanPath = (path or "").strip("/") if not cleanPath: - endpoint = "me/drive/root/children" + endpoint: Optional[str] = "me/drive/root/children?$top=200" else: - endpoint = f"me/drive/root:/{cleanPath}:/children" + endpoint = f"me/drive/root:/{cleanPath}:/children?$top=200" - result = await self._graphGet(endpoint) - if "error" in result: - return [] - entries = [_graphItemToExternalEntry(item, path) for item in result.get("value", [])] + effectiveLimit = int(limit) if limit is not None else None + items: List[Dict[str, Any]] = [] + hardCap = 5000 + while endpoint and len(items) < hardCap: + result = await self._graphGet(endpoint) + if "error" in result: + if not items: + _raiseGraphError(result, "OneDrive browse") + break + for raw in result.get("value", []) or []: + items.append(raw) + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + nextLink = result.get("@odata.nextLink") + endpoint = _stripGraphBase(nextLink) if nextLink else None + + entries = [_graphItemToExternalEntry(item, path) for item in items] if filter: entries = [e for e in entries if _matchFilter(e, filter)] - if limit is not None: - entries = entries[: max(1, int(limit))] + if effectiveLimit is not None: + entries = entries[: max(1, effectiveLimit)] return entries async def download(self, path: str) -> bytes: @@ -868,13 +1019,32 @@ class OneDriveAdapter(_GraphApiMixin, ServiceAdapter): limit: Optional[int] = None, ) -> List[ExternalEntry]: safeQuery = query.replace("'", "''") - endpoint = f"me/drive/root/search(q='{safeQuery}')" - result = await self._graphGet(endpoint) - if "error" in result: - return [] - entries = [_graphItemToExternalEntry(item) for item in result.get("value", [])] - if limit is not None: - entries = entries[: max(1, int(limit))] + cleanPath = (path or "").strip("/") + # Scope to the attached folder if given, otherwise search the whole drive. + if cleanPath: + endpoint: Optional[str] = f"me/drive/root:/{cleanPath}:/search(q='{safeQuery}')?$top=200" + else: + endpoint = f"me/drive/root/search(q='{safeQuery}')?$top=200" + effectiveLimit = int(limit) if limit is not None else None + items: List[Dict[str, Any]] = [] + hardCap = 1000 + while endpoint and len(items) < hardCap: + result = await self._graphGet(endpoint) + if "error" in result: + if not items: + _raiseGraphError(result, "OneDrive search") + break + for raw in result.get("value", []) or []: + items.append(raw) + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + if effectiveLimit is not None and len(items) >= effectiveLimit: + break + nextLink = result.get("@odata.nextLink") + endpoint = _stripGraphBase(nextLink) if nextLink else None + entries = [_graphItemToExternalEntry(item) for item in items] + if effectiveLimit is not None: + entries = entries[: max(1, effectiveLimit)] return entries @@ -908,8 +1078,7 @@ class CalendarAdapter(_GraphApiMixin, ServiceAdapter): if not cleanPath: result = await self._graphGet("me/calendars?$top=100") if "error" in result: - logger.warning(f"MSFT Calendar list failed: {result['error']}") - return [] + _raiseGraphError(result, "MSFT Calendar list") calendars = result.get("value", []) if filter: calendars = [c for c in calendars if filter.lower() in (c.get("name") or "").lower()] @@ -929,7 +1098,15 @@ class CalendarAdapter(_GraphApiMixin, ServiceAdapter): for c in calendars ] - calendarId = cleanPath.split("/", 1)[0] + # The path segment may be a calendar display name or an already-resolved + # calendar id; resolve first so a name does not produce a malformed URL. + calendarRef = cleanPath.split("/", 1)[0] + calendarId = await self._resolveCalendarId(calendarRef) + if not calendarId: + raise ValueError( + f"Calendar not found: '{calendarRef}'. Browse the root ('/') to list " + f"calendars and use the returned id." + ) effectiveLimit = self._DEFAULT_EVENT_LIMIT if limit is None else max(1, min(int(limit), self._MAX_EVENT_LIMIT)) pageSize = min(self._PAGE_SIZE, effectiveLimit) @@ -952,7 +1129,8 @@ class CalendarAdapter(_GraphApiMixin, ServiceAdapter): while endpoint and len(events) < effectiveLimit: result = await self._graphGet(endpoint) if "error" in result: - logger.warning(f"MSFT Calendar events failed: {result['error']}") + if not events: + _raiseGraphError(result, "MSFT Calendar events") break for ev in result.get("value", []): events.append(ev) @@ -980,32 +1158,34 @@ class CalendarAdapter(_GraphApiMixin, ServiceAdapter): for ev in events ] + async def _resolveCalendarId(self, ref: str) -> Optional[str]: + """Resolve a calendar reference (display name / 'default' / id) to a Graph + calendar id. Returns None if nothing matches.""" + if not ref: + return None + r = ref.strip() + # Heuristic: Graph ids are long URL-safe strings without spaces. + if len(r) > 60 and " " not in r: + return r + result = await self._graphGet("me/calendars?$top=100") + if "error" in result: + _raiseGraphError(result, "MSFT Calendar list") + cals = result.get("value", []) + for c in cals: + if c.get("id") == r: + return r + if r.lower() in ("default", "primary", "calendar", "kalender"): + for c in cals: + if c.get("isDefaultCalendar"): + return c.get("id") + for c in cals: + if (c.get("name") or "").strip().lower() == r.lower(): + return c.get("id") + return None + @staticmethod def _parseDateRange(filterStr: Optional[str]) -> tuple: - """Parse date range from filter string. Supports ISO dates or YYYY-MM patterns.""" - import re - from datetime import datetime, timedelta - if not filterStr: - return (None, None) - isoMatch = re.findall(r'\d{4}-\d{2}-\d{2}(?:T[\d:]+)?', filterStr) - if len(isoMatch) >= 2: - return (isoMatch[0], isoMatch[1]) - if len(isoMatch) == 1: - try: - dt = datetime.fromisoformat(isoMatch[0]) - return (isoMatch[0], (dt + timedelta(days=31)).strftime('%Y-%m-%dT00:00:00')) - except ValueError: - pass - monthMatch = re.match(r'^(\d{4})-(\d{2})$', filterStr.strip()) - if monthMatch: - year, month = int(monthMatch.group(1)), int(monthMatch.group(2)) - start = f"{year}-{month:02d}-01T00:00:00" - if month == 12: - end = f"{year + 1}-01-01T00:00:00" - else: - end = f"{year}-{month + 1:02d}-01T00:00:00" - return (start, end) - return (None, None) + return _parseDateRange(filterStr) async def download(self, path: str) -> DownloadResult: cleanPath = (path or "").strip("/") @@ -1050,7 +1230,7 @@ class CalendarAdapter(_GraphApiMixin, ServiceAdapter): result = await self._graphGet(endpoint) if "error" in result: - return [] + _raiseGraphError(result, "MSFT Calendar search") calendarId = (path or "").strip("/").split("/")[0] if path else "search" return [ ExternalEntry( @@ -1126,7 +1306,15 @@ class ContactsAdapter(_GraphApiMixin, ServiceAdapter): logger.warning(f"MSFT contactFolders list failed: {result['error']}") return folders - folderId = cleanPath.split("/", 1)[0] + # The path segment may be a contact-folder display name or an already- + # resolved folder id (or the virtual 'default'); resolve first. + folderRef = cleanPath.split("/", 1)[0] + folderId = await self._resolveContactFolderId(folderRef) + if not folderId: + raise ValueError( + f"Contact folder not found: '{folderRef}'. Browse the root ('/') to " + f"list folders and use the returned id." + ) effectiveLimit = self._DEFAULT_CONTACT_LIMIT if limit is None else max(1, min(int(limit), self._MAX_CONTACT_LIMIT)) pageSize = min(self._PAGE_SIZE, effectiveLimit) if folderId == self._DEFAULT_FOLDER_ID: @@ -1138,7 +1326,8 @@ class ContactsAdapter(_GraphApiMixin, ServiceAdapter): while endpoint and len(contacts) < effectiveLimit: result = await self._graphGet(endpoint) if "error" in result: - logger.warning(f"MSFT contacts list failed: {result['error']}") + if not contacts: + _raiseGraphError(result, "MSFT contacts list") break for c in result.get("value", []): contacts.append(c) @@ -1166,6 +1355,28 @@ class ContactsAdapter(_GraphApiMixin, ServiceAdapter): for c in contacts ] + async def _resolveContactFolderId(self, ref: str) -> Optional[str]: + """Resolve a contact-folder reference (display name / 'default' / id) to a + folder id. Returns None if nothing matches.""" + if not ref: + return None + r = ref.strip() + if r == self._DEFAULT_FOLDER_ID or r.lower() in ("kontakte", "contacts", "default"): + return self._DEFAULT_FOLDER_ID + # Heuristic: Graph ids are long URL-safe strings without spaces. + if len(r) > 60 and " " not in r: + return r + result = await self._graphGet("me/contactFolders?$top=100") + if "error" in result: + _raiseGraphError(result, "MSFT contactFolders list") + for f in result.get("value", []): + if f.get("id") == r: + return r + for f in result.get("value", []): + if (f.get("displayName") or "").strip().lower() == r.lower(): + return f.get("id") + return None + async def download(self, path: str) -> DownloadResult: cleanPath = (path or "").strip("/") if "/" not in cleanPath: @@ -1193,19 +1404,27 @@ class ContactsAdapter(_GraphApiMixin, ServiceAdapter): path: Optional[str] = None, limit: Optional[int] = None, ) -> List[ExternalEntry]: - safeQuery = query.replace("'", "''") + safeQuery = query.replace('"', '\\"') effectiveLimit = self._DEFAULT_CONTACT_LIMIT if limit is None else max(1, min(int(limit), self._MAX_CONTACT_LIMIT)) endpoint = f"me/contacts?$search=\"{safeQuery}\"&$top={effectiveLimit}" result = await self._graphGet(endpoint) if "error" in result: - return [] + _raiseGraphError(result, "MSFT contacts search") return [ ExternalEntry( name=c.get("displayName") or _personLabel(c) or "(no name)", path=f"/search/{c.get('id', '')}", isFolder=False, mimeType="text/vcard", - metadata={"id": c.get("id")}, + metadata={ + "id": c.get("id"), + "givenName": c.get("givenName"), + "surname": c.get("surname"), + "companyName": c.get("companyName"), + "emailAddresses": [e.get("address") for e in (c.get("emailAddresses") or []) if e.get("address")], + "businessPhones": c.get("businessPhones") or [], + "mobilePhone": c.get("mobilePhone"), + }, ) for c in result.get("value", []) ] diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index 6546f6d9..93eae82a 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -111,7 +111,6 @@ class ChatMessage(PowerOnModel): class WorkflowModeEnum(str, Enum): WORKFLOW_DYNAMIC = "Dynamic" WORKFLOW_AUTOMATION = "Automation" - WORKFLOW_CHATBOT = "Chatbot" @i18nModel("Chat-Workflow") class ChatWorkflow(PowerOnModel): @@ -169,10 +168,6 @@ class ChatWorkflow(PowerOnModel): "value": WorkflowModeEnum.WORKFLOW_AUTOMATION.value, "label": "Automation", }, - { - "value": WorkflowModeEnum.WORKFLOW_CHATBOT.value, - "label": "Chatbot", - }, ]}) maxSteps: int = Field(default=10, description="Maximum number of iterations in dynamic mode", json_schema_extra={"label": "Max. Schritte", "frontend_type": "integer", "frontend_readonly": False, "frontend_required": False}) expectedFormats: Optional[List[str]] = Field(None, description="List of expected file format extensions from user request (e.g., ['xlsx', 'pdf']). Extracted during intent analysis.", json_schema_extra={"label": "Erwartete Formate", "frontend_type": "text", "frontend_readonly": True, "frontend_required": False}) diff --git a/modules/datamodels/datamodelDataSource.py b/modules/datamodels/datamodelDataSource.py index de32bdf3..1cfaa933 100644 --- a/modules/datamodels/datamodelDataSource.py +++ b/modules/datamodels/datamodelDataSource.py @@ -77,19 +77,14 @@ class DataSource(PowerOnModel): description="Timestamp of last successful RAG indexing run", json_schema_extra={"label": "Letzte Indexierung", "frontend_type": "timestamp"}, ) + # scope was removed (privacy, 2026-06). Personal sources must not be + # shared across scopes. Only Files (folder-files) retain scope. + # The DB column is kept as deprecated-nullable to avoid a migration; + # it is never read or written by UDB/ingest/knowledge anymore. scope: Optional[str] = Field( default=None, - description=( - "Data visibility scope with inherit semantics. " - "None = inherit; values: personal, featureInstance, mandate, global. " - "Cascade-reset on parent toggle." - ), - json_schema_extra={"label": "Sichtbarkeit", "frontend_type": "select", "frontend_readonly": False, "frontend_required": False, "frontend_options": [ - {"value": "personal", "label": "Persönlich"}, - {"value": "featureInstance", "label": "Feature-Instanz"}, - {"value": "mandate", "label": "Mandant"}, - {"value": "global", "label": "Global"}, - ]}, + description="DEPRECATED (2026-06, privacy). Always None. Use Files scope instead.", + json_schema_extra={"frontend_readonly": True, "frontend_hidden": True}, ) neutralize: Optional[bool] = Field( default=None, diff --git a/modules/datamodels/datamodelFeatures.py b/modules/datamodels/datamodelFeatures.py index 138ab0dd..67532fe3 100644 --- a/modules/datamodels/datamodelFeatures.py +++ b/modules/datamodels/datamodelFeatures.py @@ -12,9 +12,9 @@ from modules.datamodels.datamodelUtils import TextMultilingual @i18nModel("Feature") class Feature(PowerOnModel): - """Feature-Definition (global, z.B. 'trustee', 'chatbot'). Verfuegbare Funktionalitaeten der Plattform.""" + """Feature-Definition (global, z.B. 'trustee', 'commcoach'). Verfuegbare Funktionalitaeten der Plattform.""" code: str = Field( - description="Unique feature code (Primary Key), z.B. 'trustee', 'chatbot'", + description="Unique feature code (Primary Key), z.B. 'trustee', 'commcoach'", json_schema_extra={"label": "Code", "frontend_type": "text", "frontend_readonly": False, "frontend_required": True} ) label: TextMultilingual = Field( diff --git a/modules/datamodels/datamodelJson.py b/modules/datamodels/datamodelJson.py index 0228fbad..6c7793c4 100644 --- a/modules/datamodels/datamodelJson.py +++ b/modules/datamodels/datamodelJson.py @@ -16,6 +16,9 @@ supportedSectionTypes: List[str] = [ "paragraph", "code_block", "image", + # Layout primitives (A3): type-specific document layout. + "cover_page", # centered title page (subtitle/author/date/logo), ends with page break + "image_grid", # N-column arrangement of images (marketing-style layouts) ] class InlineRun(TypedDict, total=False): diff --git a/modules/features/chatbot/__init__.py b/modules/features/chatbot/__init__.py deleted file mode 100644 index 30b57e1f..00000000 --- a/modules/features/chatbot/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Chatbot feature - LangGraph-based chatbot implementation. -Lazy-loaded to avoid importing langgraph/langchain at boot time. -""" - - -async def chatProcess(*args, **kwargs): - """Lazy wrapper - imports the real chatProcess on first call to defer langgraph loading.""" - from .service import chatProcess as _chatProcess - return await _chatProcess(*args, **kwargs) - - -__all__ = ['chatProcess'] diff --git a/modules/features/chatbot/bridges/__init__.py b/modules/features/chatbot/bridges/__init__.py deleted file mode 100644 index 6eb29559..00000000 --- a/modules/features/chatbot/bridges/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Bridges to external systems (AI models, database, tools).""" diff --git a/modules/features/chatbot/bridges/ai.py b/modules/features/chatbot/bridges/ai.py deleted file mode 100644 index 1962fa8f..00000000 --- a/modules/features/chatbot/bridges/ai.py +++ /dev/null @@ -1,727 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -AI Center to LangChain bridge. -Implements LangChain BaseChatModel interface using AI center models. -""" - -import logging -import asyncio -import time -from typing import Any, AsyncIterator, Callable, Dict, List, Optional -from datetime import datetime - -from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import ( - BaseMessage, - HumanMessage, - SystemMessage, - AIMessage, - ToolMessage, - convert_to_openai_messages, -) -from langchain_core.outputs import ChatGeneration, ChatResult -from langchain_core.runnables import RunnableConfig - -from modules.aicore.aicoreModelRegistry import modelRegistry -from modules.aicore.aicoreModelSelector import modelSelector -from modules.datamodels.datamodelAi import ( - AiModel, - AiModelCall, - AiModelResponse, - AiCallResponse, - AiCallOptions, - OperationTypeEnum, - ProcessingModeEnum, -) -from modules.datamodels.datamodelUam import User - -logger = logging.getLogger(__name__) - -# Workflow-level store for allowed_providers and RBAC context (survives LangGraph/bind_tools -# execution context where instance attributes may be lost when model is wrapped or serialized) -_workflow_allowed_providers: Dict[str, List[str]] = {} -_workflow_rbac_context: Dict[str, tuple] = {} # workflow_id -> (mandateId, featureInstanceId) - - -def clear_workflow_allowed_providers(workflow_id: str) -> None: - """Remove workflow from registry when stream completes to avoid memory growth.""" - _workflow_allowed_providers.pop(workflow_id, None) - - -class AICenterChatModel(BaseChatModel): - """ - LangChain-compatible chat model that uses AI center models. - Bridges AI center model selection and calling to LangChain's BaseChatModel interface. - """ - - def __init__( - self, - user: User, - operation_type: OperationTypeEnum = OperationTypeEnum.DATA_ANALYSE, - processing_mode: ProcessingModeEnum = ProcessingModeEnum.DETAILED, - billing_callback: Optional[Callable[[AiCallResponse], None]] = None, - workflow_id: Optional[str] = None, - allowed_providers: Optional[List[str]] = None, - prefer_fast_model: bool = False, - mandate_id: Optional[str] = None, - feature_instance_id: Optional[str] = None, - **kwargs - ): - """ - Initialize the AI center chat model bridge. - - Args: - user: Current user for RBAC and model selection - operation_type: Operation type for model selection - processing_mode: Processing mode for model selection - billing_callback: Optional callback invoked after each _agenerate with AiCallResponse for billing - workflow_id: Optional workflow/conversation ID for billing context - allowed_providers: Optional list of allowed provider connector types (empty/None = all) - prefer_fast_model: When True, strongly prefer faster models (e.g. gpt-4o-mini for planner) - **kwargs: Additional arguments passed to BaseChatModel - """ - super().__init__(**kwargs) - # Use object.__setattr__ to bypass Pydantic validation for custom attributes - object.__setattr__(self, "user", user) - object.__setattr__(self, "operation_type", operation_type) - object.__setattr__(self, "processing_mode", processing_mode) - object.__setattr__(self, "_selected_model", None) - object.__setattr__(self, "_billing_callback", billing_callback) - object.__setattr__(self, "_workflow_id", workflow_id) - object.__setattr__(self, "_allowed_providers", allowed_providers or []) - object.__setattr__(self, "_prefer_fast_model", prefer_fast_model) - object.__setattr__(self, "_mandate_id", mandate_id) - object.__setattr__(self, "_feature_instance_id", feature_instance_id) - # Store in workflow-level registry so it survives when instance attrs are lost (e.g. bind_tools) - if workflow_id and allowed_providers: - _workflow_allowed_providers[workflow_id] = list(allowed_providers) - if workflow_id and (mandate_id is not None or feature_instance_id is not None): - _workflow_rbac_context[workflow_id] = (mandate_id, feature_instance_id) - - @property - def _llm_type(self) -> str: - """Return type of LLM.""" - return "aicenter" - - def _select_model(self, messages: List[BaseMessage]) -> AiModel: - """ - Select the best AI center model for the given messages. - Uses caching to avoid repeated model selection within same session. - - Args: - messages: List of LangChain messages - - Returns: - Selected AI model - """ - # Return cached model if already selected (significant performance improvement) - if self._selected_model is not None: - return self._selected_model - - # Convert messages to prompt/context format for model selector - prompt_parts = [] - context_parts = [] - - for msg in messages: - if isinstance(msg, SystemMessage): - prompt_parts.append(msg.content) - elif isinstance(msg, HumanMessage): - prompt_parts.append(msg.content) - elif isinstance(msg, AIMessage): - context_parts.append(msg.content) - elif isinstance(msg, ToolMessage): - context_parts.append(f"Tool {msg.name}: {msg.content}") - - prompt = "\n".join(prompt_parts) - context = "\n".join(context_parts) if context_parts else "" - - # Get available models with RBAC filtering - # Use cached/singleton interfaces for better performance - from modules.interfaces.interfaceDbApp import getRootInterface - - workflow_id = getattr(self, "_workflow_id", None) - rootInterface = getRootInterface() - rbac_instance = rootInterface.rbac - - mandate_id = getattr(self, "_mandate_id", None) - feature_instance_id = getattr(self, "_feature_instance_id", None) - if workflow_id and (mandate_id is None and feature_instance_id is None): - ctx = _workflow_rbac_context.get(workflow_id) - if ctx: - mandate_id, feature_instance_id = ctx - available_models = modelRegistry.getAvailableModels( - currentUser=self.user, - rbacInstance=rbac_instance, - mandateId=mandate_id, - featureInstanceId=feature_instance_id, - ) - - # Allowed providers: instance attr or workflow store (lost in LangGraph/bind_tools context) - allowed = ( - (_workflow_allowed_providers.get(workflow_id) if workflow_id else None) - or getattr(self, '_allowed_providers', None) - or [] - ) - if allowed: - logger.info(f"AICenterChatModel _select_model: applying allowedProviders={allowed}") - filtered = [m for m in available_models if m.connectorType in allowed] - if filtered: - available_models = filtered - else: - logger.warning(f"No models match allowedProviders {allowed}, using all RBAC-permitted models") - options = AiCallOptions( - operationType=self.operation_type, - processingMode=self.processing_mode, - allowedProviders=allowed if allowed else None, - preferFastModel=getattr(self, "_prefer_fast_model", False), - ) - - # Select model - selected_model = modelSelector.selectModel( - prompt=prompt, - context=context, - options=options, - availableModels=available_models - ) - - if not selected_model: - raise ValueError(f"No suitable model found for operation type {self.operation_type.value}") - - logger.info(f"Selected AI center model: {selected_model.displayName} ({selected_model.name})") - object.__setattr__(self, "_selected_model", selected_model) - return selected_model - - def _convert_messages_to_ai_format(self, messages: List[BaseMessage]) -> List[Dict[str, Any]]: - """ - Convert LangChain messages to AI center format (OpenAI-style). - - Args: - messages: List of LangChain messages - - Returns: - List of messages in OpenAI format - """ - # Use LangChain's built-in conversion - openai_messages = convert_to_openai_messages(messages) - return openai_messages - - def _convert_ai_response_to_langchain( - self, - response: AiModelResponse, - tool_calls: Optional[List[Dict[str, Any]]] = None - ) -> AIMessage: - """ - Convert AI center response to LangChain AIMessage. - - Args: - response: AI center response - tool_calls: Optional tool calls from the response (format: [{"id": "...", "name": "...", "args": {...}}]) - - Returns: - LangChain AIMessage with tool_calls if present - """ - # LangChain expects tool_calls in format: [{"id": "...", "name": "...", "args": {...}}] - # The tool_calls parameter should already be in this format - - kwargs = {} - if tool_calls: - kwargs["tool_calls"] = tool_calls - - return AIMessage(content=response.content or "", **kwargs) - - def _generate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager: Optional[Any] = None, - **kwargs: Any, - ) -> ChatResult: - """ - Synchronous generate method required by BaseChatModel. - Wraps the async _agenerate method. - - Args: - messages: List of LangChain messages - stop: Optional stop sequences - run_manager: Optional callback manager - **kwargs: Additional arguments - - Returns: - ChatResult with generations - """ - # Try to get the current event loop - try: - loop = asyncio.get_event_loop() - if loop.is_running(): - # If we're in an async context, raise an error - raise RuntimeError( - "AICenterChatModel._generate() called from async context. " - "Use _agenerate() instead." - ) - except RuntimeError: - # No event loop, we can create one - pass - - # Run the async method synchronously - return asyncio.run(self._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)) - - async def _call_openai_streaming( - self, - ai_messages: List[dict], - run_manager: Optional[Any], - model_call: "AiModelCall", - input_bytes: int, - start_time: float, - ) -> "AiModelResponse": - """Call OpenAI/Ollama with stream=True, emit tokens via run_manager, return full response.""" - import httpx - import json as _json - from modules.shared.configuration import APP_CONFIG - - if self._selected_model.connectorType == "openai": - api_url = getattr(self._selected_model, "apiUrl", None) or "https://api.openai.com/v1/chat/completions" - api_key = APP_CONFIG.get("Connector_AiOpenai_API_SECRET") - if not api_key: - raise ValueError("OpenAI API key not configured") - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} - ollama_model = self._selected_model.name - else: - base_url = getattr(self._selected_model, "apiUrl", "").replace("/api/analyze", "") - api_url = f"{base_url.rstrip('/')}/v1/chat/completions" - api_key = APP_CONFIG.get("Connector_AiPrivateLlm_API_SECRET") - headers = {"Content-Type": "application/json"} - if api_key: - headers["X-API-Key"] = api_key - ollama_model = getattr(self._selected_model, "version", None) or self._selected_model.name - - payload = { - "model": ollama_model, - "messages": ai_messages, - "temperature": self._selected_model.temperature, - "max_tokens": self._selected_model.maxTokens, - "stream": True, - } - content_parts: List[str] = [] - async with httpx.AsyncClient(timeout=600.0) as client: - async with client.stream("POST", api_url, headers=headers, json=payload) as resp: - if resp.status_code != 200: - raise ValueError(f"OpenAI stream error: {resp.status_code} - {await resp.aread()}") - buffer = "" - async for chunk in resp.aiter_text(): - buffer += chunk - while "\n" in buffer or "\r\n" in buffer: - line, _, buffer = buffer.partition("\n") - line = line.strip() - if line.startswith("data: "): - data_str = line[6:].strip() - if data_str == "[DONE]": - break - try: - data = _json.loads(data_str) - choices = data.get("choices") or [] - if choices: - delta = choices[0].get("delta") or {} - token = delta.get("content") or "" - if token and run_manager and hasattr(run_manager, "on_llm_new_token"): - run_manager.on_llm_new_token(token) - content_parts.append(token) - except _json.JSONDecodeError: - pass - - content = "".join(content_parts) - processing_time = time.time() - start_time - output_bytes = len(content.encode("utf-8")) - price_chf = 0.0 - if getattr(self._selected_model, "calculatepriceCHF", None): - try: - price_chf = self._selected_model.calculatepriceCHF(processing_time, input_bytes, output_bytes) - except Exception: - pass - billing_callback = getattr(self, "_billing_callback", None) - if billing_callback: - try: - billing_callback(AiCallResponse( - content=content, - modelName=self._selected_model.name, - provider=self._selected_model.connectorType or "unknown", - priceCHF=price_chf, - processingTime=processing_time, - bytesSent=input_bytes, - bytesReceived=output_bytes, - errorCount=0, - )) - except Exception as e: - logger.error(f"Billing callback error: {e}") - - return AiModelResponse(content=content, success=True, modelId=self._selected_model.name, metadata={}) - - async def _agenerate( - self, - messages: List[BaseMessage], - stop: Optional[List[str]] = None, - run_manager: Optional[Any] = None, - **kwargs: Any, - ) -> ChatResult: - """ - Async generate method required by BaseChatModel. - - Args: - messages: List of LangChain messages - stop: Optional stop sequences - run_manager: Optional callback manager - **kwargs: Additional arguments (may include tools for tool calling) - - Returns: - ChatResult with generations - """ - # Select model if not already selected - if not self._selected_model: - self._select_model(messages) - - # Check if tools are bound (for tool calling) - tools = getattr(self, "_bound_tools", None) - - # Convert messages to AI center format - ai_messages = self._convert_messages_to_ai_format(messages) - - # Compute input bytes for billing (sum of message content lengths) - input_bytes = sum( - len((m.get("content") or "").encode("utf-8")) - for m in ai_messages - if isinstance(m.get("content"), str) - ) - start_time = time.time() - - # If tools are bound, add tool definitions to the system message - # This ensures the model knows about available tools - # Some models need explicit tool definitions to enable tool calling - if tools: - # Find or create system message - system_message_idx = None - for i, msg in enumerate(ai_messages): - if msg.get("role") == "system": - system_message_idx = i - break - - # Build tool descriptions for the system message - tool_descriptions = [] - for tool in tools: - if hasattr(tool, "name") and hasattr(tool, "description"): - # Get tool parameters for better description - args_schema = getattr(tool, "args_schema", None) - params_info = "" - if args_schema: - try: - if hasattr(args_schema, "model_json_schema"): - schema = args_schema.model_json_schema() - if "properties" in schema: - params = list(schema["properties"].keys()) - params_info = f" (Parameter: {', '.join(params)})" - except: - pass - tool_descriptions.append(f"- {tool.name}: {tool.description}{params_info}") - - if tool_descriptions: - tools_text = "\n".join(tool_descriptions) - tools_note = f"\n\n⚠️⚠️⚠️ KRITISCH - TOOL-NUTZUNG ⚠️⚠️⚠️\n\nVERFÜGBARE TOOLS:\n{tools_text}\n\nABSOLUT VERBINDLICH:\n- Du MUSST diese Tools verwenden, um Anfragen zu bearbeiten!\n- Für Status-Updates MUSST du IMMER das Tool 'send_streaming_message' verwenden!\n- VERBOTEN: Normale Text-Nachrichten für Status-Updates!\n- Du MUSST Tools aufrufen, nicht nur darüber sprechen!\n\nBeispiel FALSCH: \"Ich werde die Datenbank durchsuchen...\"\nBeispiel RICHTIG: Rufe das Tool 'send_streaming_message' mit \"Durchsuche Datenbank...\" auf!" - - if system_message_idx is not None: - # Append to existing system message - ai_messages[system_message_idx]["content"] += tools_note - else: - # Add new system message at the beginning - ai_messages.insert(0, { - "role": "system", - "content": tools_note.strip() - }) - - # Convert LangChain tools to OpenAI/function-calling format (used by OpenAI and Ollama-compatible APIs) - openai_tools = None - if tools and self._selected_model.connectorType in ("openai", "privatellm"): - # Build tool schema in OpenAI format (Ollama uses same format for tool calling) - openai_tools = [] - for tool in tools: - if hasattr(tool, "name") and hasattr(tool, "description"): - # Get tool parameters schema - args_schema = getattr(tool, "args_schema", None) - parameters = {} - if args_schema: - # Check if it's a Pydantic model class or instance - from pydantic import BaseModel - - # Check if it's a class (not an instance) - if isinstance(args_schema, type) and issubclass(args_schema, BaseModel): - # It's a Pydantic model class - get JSON schema - if hasattr(args_schema, "model_json_schema"): - # Pydantic v2 - parameters = args_schema.model_json_schema() - elif hasattr(args_schema, "schema"): - # Pydantic v1 - parameters = args_schema.schema() - elif isinstance(args_schema, BaseModel): - # It's a Pydantic model instance - if hasattr(args_schema, "model_dump"): - parameters = args_schema.model_dump() - elif hasattr(args_schema, "schema"): - # Has schema method (might be a class) - try: - parameters = args_schema.schema() - except TypeError: - # If schema() requires instance, try model_json_schema - if hasattr(args_schema, "model_json_schema"): - parameters = args_schema.model_json_schema() - else: - parameters = {} - elif isinstance(args_schema, dict): - # Already a dict - parameters = args_schema - - tool_schema = { - "type": "function", - "function": { - "name": tool.name, - "description": tool.description or "", - "parameters": parameters - } - } - openai_tools.append(tool_schema) - - # Store tools for potential use by connector - # Note: The connector may need to access tools from the model_call - # This is a workaround since AiModelCall doesn't have a tools field - # Tools are added to system message above to ensure model knows about them - - # Create model call - model_call = AiModelCall( - messages=ai_messages, - model=self._selected_model, - options=AiCallOptions( - operationType=self.operation_type, - processingMode=self.processing_mode, - temperature=self._selected_model.temperature - ) - ) - - # If tools are bound, use OpenAI-compatible API (OpenAI or Private-LLM Ollama endpoint) - if openai_tools and self._selected_model.connectorType in ("openai", "privatellm"): - import httpx - import json as _json - from modules.shared.configuration import APP_CONFIG - - if self._selected_model.connectorType == "openai": - api_url = self._selected_model.apiUrl - api_key = APP_CONFIG.get("Connector_AiOpenai_API_SECRET") - if not api_key: - raise ValueError("OpenAI API key not configured") - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} - ollama_model = self._selected_model.name - else: - # privatellm: use Ollama OpenAI-compatible /v1/chat/completions (same service, same provider) - base_url = self._selected_model.apiUrl.replace("/api/analyze", "") - api_url = f"{base_url}/v1/chat/completions" - api_key = APP_CONFIG.get("Connector_AiPrivateLlm_API_SECRET") - headers = {"Content-Type": "application/json"} - if api_key: - headers["X-API-Key"] = api_key - # Ollama needs the underlying model name (e.g. qwen2.5:7b), not poweron-text-general - ollama_model = getattr(self._selected_model, "version", None) or self._selected_model.name - - payload = { - "model": ollama_model, - "messages": ai_messages, - "tools": openai_tools, - "tool_choice": "auto", - "temperature": self._selected_model.temperature, - "max_tokens": self._selected_model.maxTokens, - } - - use_connector_fallback = False - async with httpx.AsyncClient(timeout=600.0) as client: - response_obj = await client.post(api_url, headers=headers, json=payload) - - if response_obj.status_code == 404 and self._selected_model.connectorType == "privatellm": - logger.warning( - "Private-LLM /v1/chat/completions not found (404). Falling back to /api/analyze. " - "Tool calling will not work until the service exposes an OpenAI-compatible endpoint." - ) - use_connector_fallback = True - elif response_obj.status_code != 200: - error_msg = f"AI API error ({self._selected_model.connectorType}): {response_obj.status_code} - {response_obj.text}" - logger.error(error_msg) - raise ValueError(error_msg) - - if use_connector_fallback: - if not self._selected_model.functionCall: - raise ValueError(f"Model {self._selected_model.displayName} has no functionCall defined") - response = await self._selected_model.functionCall(model_call) - else: - response_json = response_obj.json() - choice = response_json["choices"][0] - message = choice["message"] - - content = message.get("content", "") - tool_calls_raw = message.get("tool_calls") - - tool_calls = None - if tool_calls_raw: - tool_calls = [] - for tc in tool_calls_raw: - func_data = tc.get("function", {}) - func_name = func_data.get("name") - func_args_str = func_data.get("arguments", "{}") - try: - func_args = _json.loads(func_args_str) if isinstance(func_args_str, str) else func_args_str - except Exception: - func_args = {} - tool_calls.append({ - "id": tc.get("id"), - "name": func_name, - "args": func_args, - }) - - response = AiModelResponse( - content=content or "", - success=True, - modelId=self._selected_model.name, - metadata={ - "response_id": response_json.get("id", ""), - "tool_calls": tool_calls, - }, - ) - elif not tools and self._selected_model.connectorType in ("openai", "privatellm"): - # Streaming path for OpenAI/Ollama without tools (ChatGPT-like token streaming) - response = await self._call_openai_streaming( - ai_messages, run_manager, model_call, input_bytes, start_time - ) - else: - # No tools or not OpenAI - use connector normally - if not self._selected_model.functionCall: - raise ValueError(f"Model {self._selected_model.displayName} has no functionCall defined") - - response: AiModelResponse = await self._selected_model.functionCall(model_call) - - if not response.success: - raise ValueError(f"AI model call failed: {response.error or 'Unknown error'}") - - # Billing: compute price and invoke callback - output_bytes = len((response.content or "").encode("utf-8")) - processing_time = time.time() - start_time - price_chf = 0.0 - if getattr(self._selected_model, "calculatepriceCHF", None): - try: - price_chf = self._selected_model.calculatepriceCHF( - processing_time, input_bytes, output_bytes - ) - except Exception as e: - logger.warning(f"Billing: price calculation failed: {e}") - billing_callback = getattr(self, "_billing_callback", None) - if billing_callback: - try: - ai_response = AiCallResponse( - content=response.content or "", - modelName=self._selected_model.name, - provider=getattr(self._selected_model, "connectorType", "unknown") or "unknown", - priceCHF=price_chf, - processingTime=processing_time, - bytesSent=input_bytes, - bytesReceived=output_bytes, - errorCount=0, - ) - billing_callback(ai_response) - except Exception as e: - logger.error(f"Billing callback error: {e}") - - # Extract tool calls from response metadata if present - tool_calls = None - if response.metadata: - # Check for tool calls in metadata (format may vary by connector) - tool_calls = response.metadata.get("tool_calls") or response.metadata.get("function_calls") - - # Convert response to LangChain format with tool calls - ai_message = self._convert_ai_response_to_langchain(response, tool_calls=tool_calls) - - # Create generation and result - generation = ChatGeneration(message=ai_message) - return ChatResult(generations=[generation]) - - def bind_tools(self, tools: List[Any], **kwargs: Any) -> "AICenterChatModel": - """ - Bind tools to the model (required for LangGraph tool calling). - - Args: - tools: List of LangChain tools - **kwargs: Additional arguments - - Returns: - New instance with tools bound - """ - # Create a new instance with tools bound - # Note: The actual tool binding happens in LangGraph's ToolNode - # This method is called by LangGraph to prepare the model - bound_model = AICenterChatModel( - user=self.user, - operation_type=self.operation_type, - processing_mode=self.processing_mode, - billing_callback=getattr(self, "_billing_callback", None), - workflow_id=getattr(self, "_workflow_id", None), - ) - object.__setattr__(bound_model, "_selected_model", self._selected_model) - # Store tools for potential use in message conversion - object.__setattr__(bound_model, "_bound_tools", tools) - return bound_model - - def invoke( - self, - input: List[BaseMessage], - config: Optional[RunnableConfig] = None, - **kwargs: Any, - ) -> BaseMessage: - """ - Synchronous invoke method (required by BaseChatModel). - Note: This is a wrapper around async _agenerate. - - Args: - input: List of LangChain messages - config: Optional runnable config - **kwargs: Additional arguments - - Returns: - AIMessage response - """ - import asyncio - - # Try to get existing event loop - try: - loop = asyncio.get_event_loop() - if loop.is_running(): - # If loop is running, we need to use a different approach - # This shouldn't happen in LangGraph context, but handle it gracefully - raise RuntimeError("Cannot use synchronous invoke in async context. Use ainvoke instead.") - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - # Run async generation - result = loop.run_until_complete(self._agenerate(input, **kwargs)) - return result.generations[0].message - - async def ainvoke( - self, - input: List[BaseMessage], - config: Optional[RunnableConfig] = None, - **kwargs: Any, - ) -> BaseMessage: - """ - Async invoke method (required by BaseChatModel). - - Args: - input: List of LangChain messages - config: Optional runnable config - **kwargs: Additional arguments - - Returns: - AIMessage response - """ - result = await self._agenerate(input, **kwargs) - return result.generations[0].message diff --git a/modules/features/chatbot/bridges/memory.py b/modules/features/chatbot/bridges/memory.py deleted file mode 100644 index 2d9251c1..00000000 --- a/modules/features/chatbot/bridges/memory.py +++ /dev/null @@ -1,576 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Custom LangGraph checkpointer using existing database interface. -Maps LangGraph state to existing message storage format. -""" - -import contextvars -import logging -import uuid -from typing import Any, Dict, List, Optional, Tuple, NamedTuple -from datetime import datetime - -from langgraph.checkpoint.base import BaseCheckpointSaver, Checkpoint, CheckpointMetadata - -# CheckpointTuple might not be directly importable, so we define it as a NamedTuple -# Based on LangGraph's usage, it needs config, checkpoint, metadata, parent_config, and pending_writes -class CheckpointTuple(NamedTuple): - """Tuple containing config, checkpoint, metadata, parent_config, and pending_writes.""" - config: Dict[str, Any] - checkpoint: Checkpoint - metadata: CheckpointMetadata - parent_config: Optional[Dict[str, Any]] = None - pending_writes: Optional[List[Tuple[str, Any]]] = None -from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, SystemMessage, ToolMessage - -from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface -from modules.features.chatbot.interfaceFeatureChatbot import ChatbotMessage -from modules.datamodels.datamodelUam import User -from modules.shared.timeUtils import getUtcTimestamp - -logger = logging.getLogger(__name__) - - -def _sanitize_llm_response(text: str) -> str: - """Strip chat template tokens and trailing junk that some models leak.""" - if not text or not isinstance(text, str): - return text or "" - for sentinel in ("<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|user|>", "<|assistant|>"): - if sentinel in text: - text = text.split(sentinel)[0] - return text.strip() - - -class DatabaseCheckpointer(BaseCheckpointSaver): - """ - Custom LangGraph checkpointer that uses the chatbot's own database interface. - Maps LangGraph thread_id to conversation.id; stores messages via interface (workflowId maps to conversationId). - """ - - def __init__( - self, - user: User, - workflow_id: str, - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - *, - interface=None, - ): - """ - Initialize the database checkpointer. - - Args: - user: Current user for database access - workflow_id: Workflow ID (maps to LangGraph thread_id) - mandateId: Mandate ID for proper data isolation - featureInstanceId: Feature instance ID for proper data isolation - interface: Optional pre-created chatbot interface (avoids extra getInterface + DB init) - """ - self.user = user - self.workflow_id = workflow_id - self.interface = interface if interface is not None else getChatbotInterface( - user, mandateId=mandateId, featureInstanceId=featureInstanceId - ) - - def _convert_langchain_to_db_message( - self, - msg: BaseMessage, - sequence_nr: int, - round_number: int - ) -> Dict[str, Any]: - """ - Convert LangChain message to database message format. - - Args: - msg: LangChain message - sequence_nr: Sequence number for ordering - round_number: Round number in workflow - - Returns: - Dictionary in database message format - """ - import uuid - - role = "user" - content = "" - - if isinstance(msg, HumanMessage): - role = "user" - content = msg.content if isinstance(msg.content, str) else str(msg.content) - elif isinstance(msg, AIMessage): - role = "assistant" - content = msg.content if isinstance(msg.content, str) else str(msg.content) - elif isinstance(msg, SystemMessage): - # System messages are stored but marked as system - role = "system" - content = msg.content if isinstance(msg.content, str) else str(msg.content) - elif isinstance(msg, ToolMessage): - # Tool messages are stored as assistant messages with tool info - role = "assistant" - content = f"Tool {msg.name}: {msg.content}" - - return { - "id": f"msg_{uuid.uuid4()}", - "workflowId": self.workflow_id, - "message": content, - "role": role, - "status": "step" if sequence_nr > 1 else "first", - "sequenceNr": sequence_nr, - "publishedAt": getUtcTimestamp(), - "roundNumber": round_number, - "taskNumber": 0, - "actionNumber": 0 - } - - def _convert_db_to_langchain_messages( - self, - messages: List[ChatbotMessage] - ) -> List[BaseMessage]: - """ - Convert database messages to LangChain messages. - - Args: - messages: List of database ChatMessage objects - - Returns: - List of LangChain BaseMessage objects - """ - langchain_messages = [] - - for msg in messages: - if msg.role == "user": - langchain_messages.append(HumanMessage(content=msg.message)) - elif msg.role == "assistant": - langchain_messages.append(AIMessage(content=msg.message)) - elif msg.role == "system": - langchain_messages.append(SystemMessage(content=msg.message)) - # Skip other roles for now - - return langchain_messages - - def put( - self, - config: Dict[str, Any], - checkpoint: Checkpoint, - metadata: CheckpointMetadata, - new_versions: Dict[str, int], - ) -> None: - """ - Store a checkpoint in the database. - - Args: - config: LangGraph config (contains thread_id) - checkpoint: Checkpoint to store - metadata: Checkpoint metadata - new_versions: New version numbers - """ - try: - # Extract thread_id from config (maps to workflow_id) - thread_id = config.get("configurable", {}).get("thread_id", self.workflow_id) - - # Get current workflow to determine round number - workflow = self.interface.getWorkflow(thread_id) - if not workflow: - logger.warning(f"Workflow {thread_id} not found, cannot store checkpoint") - return - - round_number = workflow.currentRound if workflow else 1 - - # Extract messages from checkpoint - state = checkpoint.get("channel_values", {}) - messages = state.get("messages", []) - - if not messages: - logger.debug(f"No messages in checkpoint for workflow {thread_id}") - return - - # Get existing messages to determine what's already stored - existing_messages = self.interface.getMessages(thread_id) - existing_count = len(existing_messages) if existing_messages else 0 - - # Create a set of existing message content+role for quick lookup - existing_content_set = set() - if existing_messages: - for existing_msg in existing_messages: - # Create a unique key from role and message content - content_key = (existing_msg.role, existing_msg.message) - existing_content_set.add(content_key) - - # Filter checkpoint messages to only user/assistant (skip system) - # Skip intermediate AIMessages with tool_calls (these are tool call requests, not final answers) - checkpoint_user_assistant_messages = [] - for msg in messages: - if isinstance(msg, HumanMessage): - # Always store user messages - checkpoint_user_assistant_messages.append(msg) - elif isinstance(msg, AIMessage): - # Check if this message has tool_calls - tool_calls = getattr(msg, "tool_calls", None) - if tool_calls and len(tool_calls) > 0: - logger.debug(f"Skipping intermediate AIMessage with tool_calls for workflow {thread_id}") - continue - # Skip agent_sql_plan output (raw SQL block) - only store agent_formulate final answer - content = msg.content if isinstance(msg.content, str) else str(msg.content) - cu = (content or "").strip().upper() - if content and ( - content.strip().startswith("```") - or (cu.startswith("SELECT") and ("FROM" in cu or "JOIN" in cu)) - ): - logger.debug(f"Skipping intermediate SQL AIMessage for workflow {thread_id}") - continue - checkpoint_user_assistant_messages.append(msg) - - # Only store new messages that aren't already in the database - new_messages_to_store = [] - for msg in checkpoint_user_assistant_messages: - role = "user" if isinstance(msg, HumanMessage) else "assistant" - content = msg.content if isinstance(msg.content, str) else str(msg.content) - if isinstance(msg, AIMessage): - content = _sanitize_llm_response(content) - if not content or not content.strip(): - continue - content_key = (role, content) - if content_key not in existing_content_set: - if isinstance(msg, AIMessage) and msg.content != content: - msg = AIMessage(content=content) - new_messages_to_store.append(msg) - existing_content_set.add(content_key) - - # Store only the new messages - if new_messages_to_store: - for i, msg in enumerate(new_messages_to_store, 1): - sequence_nr = existing_count + i - # Convert to database format - db_message_data = self._convert_langchain_to_db_message( - msg, - sequence_nr, - round_number - ) - - # Store the message - try: - self.interface.createMessage(db_message_data) - logger.debug(f"Stored message {db_message_data['id']} for workflow {thread_id}") - existing_count += 1 # Update count for next message - except Exception as e: - logger.error(f"Error storing message: {e}", exc_info=True) - else: - logger.debug(f"No new messages to store for workflow {thread_id} (existing: {existing_count}, checkpoint: {len(checkpoint_user_assistant_messages)})") - - # Update workflow last activity - self.interface.updateWorkflow(thread_id, { - "lastActivity": getUtcTimestamp() - }) - - except Exception as e: - logger.error(f"Error storing checkpoint: {e}", exc_info=True) - raise - - def get( - self, - config: Dict[str, Any], - ) -> Optional[Checkpoint]: - """ - Retrieve a checkpoint from the database. - - Args: - config: LangGraph config (contains thread_id) - - Returns: - Checkpoint if found, None otherwise - """ - try: - # Extract thread_id from config (maps to workflow_id) - thread_id = config.get("configurable", {}).get("thread_id", self.workflow_id) - - # Get workflow - workflow = self.interface.getWorkflow(thread_id) - if not workflow: - logger.debug(f"Workflow {thread_id} not found") - return None - - # Get messages - messages = self.interface.getMessages(thread_id) - - checkpoint_id = str(uuid.uuid4()) - - if not messages: - # Return empty checkpoint for new workflow - return { - "id": checkpoint_id, - "v": 1, - "ts": getUtcTimestamp(), - "channel_values": { - "messages": [] - }, - "channel_versions": {}, - "versions_seen": {} - } - - # Convert to LangChain messages - langchain_messages = self._convert_db_to_langchain_messages(messages) - - # Build checkpoint - checkpoint = { - "id": checkpoint_id, - "v": 1, - "ts": getUtcTimestamp(), - "channel_values": { - "messages": langchain_messages - }, - "channel_versions": {}, - "versions_seen": {} - } - - return checkpoint - - except Exception as e: - logger.error(f"Error retrieving checkpoint: {e}", exc_info=True) - return None - - def list( - self, - config: Dict[str, Any], - filter: Optional[Dict[str, Any]] = None, - before: Optional[str] = None, - limit: Optional[int] = None, - ) -> List[Checkpoint]: - """ - List checkpoints (not fully implemented - returns current checkpoint). - - Args: - config: LangGraph config - filter: Optional filter - before: Optional timestamp before which to list - limit: Optional limit on number of results - - Returns: - List of checkpoints - """ - checkpoint = self.get(config) - if checkpoint: - return [checkpoint] - return [] - - def put_writes( - self, - config: Dict[str, Any], - writes: List[Tuple[str, Any]], - task_id: str, - ) -> None: - """ - Store checkpoint writes (not used in current implementation). - - Args: - config: LangGraph config - writes: List of write operations - task_id: Task ID - """ - # Not implemented - using put() instead - pass - - async def aget_tuple( - self, - config: Dict[str, Any], - ) -> Optional[CheckpointTuple]: - """ - Async version of get that returns tuple of (config, checkpoint, metadata). - - Args: - config: LangGraph config (contains thread_id) - - Returns: - CheckpointTuple with config, checkpoint and metadata if found, None otherwise - """ - checkpoint = self.get(config) - if checkpoint: - # Return checkpoint with metadata including step - # CheckpointMetadata is typically a TypedDict - # LangGraph expects 'step' in metadata - metadata: CheckpointMetadata = { - "step": 0 # Start at step 0, LangGraph will increment - } - return CheckpointTuple( - config=config, - checkpoint=checkpoint, - metadata=metadata, - parent_config=None, # No parent checkpoint for our implementation - pending_writes=None # No pending writes in our implementation - ) - return None - - async def aput( - self, - config: Dict[str, Any], - checkpoint: Checkpoint, - metadata: CheckpointMetadata, - new_versions: Dict[str, int], - ) -> None: - """ - Async version of put. - - Args: - config: LangGraph config (contains thread_id) - checkpoint: Checkpoint to store - metadata: Checkpoint metadata - new_versions: New version numbers - """ - self.put(config, checkpoint, metadata, new_versions) - - async def alist( - self, - config: Dict[str, Any], - filter: Optional[Dict[str, Any]] = None, - before: Optional[str] = None, - limit: Optional[int] = None, - ) -> List[Checkpoint]: - """ - Async version of list. - - Args: - config: LangGraph config - filter: Optional filter - before: Optional timestamp before which to list - limit: Optional limit on number of results - - Returns: - List of checkpoints - """ - return self.list(config, filter, before, limit) - - async def aput_writes( - self, - config: Dict[str, Any], - writes: List[Tuple[str, Any]], - task_id: str, - ) -> None: - """ - Async version of put_writes. - Store checkpoint writes (not used in current implementation). - - Args: - config: LangGraph config - writes: List of write operations - task_id: Task ID - """ - # Not implemented - using aput() instead - # This method is called by LangGraph but we handle writes through aput() - pass - - -# ContextVar for per-request checkpointer (used by CheckpointerResolver for graph caching) -_current_checkpointer: contextvars.ContextVar[Optional[BaseCheckpointSaver]] = contextvars.ContextVar( - "chatbot_current_checkpointer", default=None -) - - -def set_checkpointer(checkpointer: BaseCheckpointSaver) -> contextvars.Token: - """Set the current request's checkpointer. Returns token to reset later.""" - return _current_checkpointer.set(checkpointer) - - -def reset_checkpointer(token: contextvars.Token) -> None: - """Reset checkpointer to prior value. Safe when called from a different async context.""" - try: - _current_checkpointer.reset(token) - except ValueError: - # Token was created in a different context (e.g. after yield, generator cleanup) - pass - - -class CheckpointerResolver(BaseCheckpointSaver): - """ - Delegating checkpointer that reads the real checkpointer from context. - Used for graph caching: the compiled graph uses this resolver; at invoke time - the per-request checkpointer is set via set_checkpointer(). - """ - - def _get_checkpointer(self) -> BaseCheckpointSaver: - cp = _current_checkpointer.get() - if cp is None: - raise RuntimeError( - "CheckpointerResolver: no checkpointer in context. " - "Call set_checkpointer() before invoking the cached graph." - ) - return cp - - def put( - self, - config: Dict[str, Any], - checkpoint: Checkpoint, - metadata: CheckpointMetadata, - new_versions: Dict[str, int], - ) -> None: - self._get_checkpointer().put(config, checkpoint, metadata, new_versions) - - def get(self, config: Dict[str, Any]) -> Optional[Checkpoint]: - return self._get_checkpointer().get(config) - - def list( - self, - config: Dict[str, Any], - filter: Optional[Dict[str, Any]] = None, - before: Optional[str] = None, - limit: Optional[int] = None, - ) -> List[Checkpoint]: - return self._get_checkpointer().list(config, filter, before, limit) - - def put_writes( - self, - config: Dict[str, Any], - writes: List[Tuple[str, Any]], - task_id: str, - ) -> None: - self._get_checkpointer().put_writes(config, writes, task_id) - - async def aget_tuple(self, config: Dict[str, Any]) -> Optional[CheckpointTuple]: - inner = self._get_checkpointer() - if hasattr(inner, "aget_tuple"): - return await inner.aget_tuple(config) - checkpoint = inner.get(config) - if checkpoint: - metadata: CheckpointMetadata = {"step": 0} - return CheckpointTuple( - config=config, - checkpoint=checkpoint, - metadata=metadata, - parent_config=None, - pending_writes=None, - ) - return None - - async def aput( - self, - config: Dict[str, Any], - checkpoint: Checkpoint, - metadata: CheckpointMetadata, - new_versions: Dict[str, int], - ) -> None: - inner = self._get_checkpointer() - if hasattr(inner, "aput"): - await inner.aput(config, checkpoint, metadata, new_versions) - else: - inner.put(config, checkpoint, metadata, new_versions) - - async def alist( - self, - config: Dict[str, Any], - filter: Optional[Dict[str, Any]] = None, - before: Optional[str] = None, - limit: Optional[int] = None, - ) -> List[Checkpoint]: - inner = self._get_checkpointer() - if hasattr(inner, "alist"): - return await inner.alist(config, filter, before, limit) - return inner.list(config, filter, before, limit) - - async def aput_writes( - self, - config: Dict[str, Any], - writes: List[Tuple[str, Any]], - task_id: str, - ) -> None: - inner = self._get_checkpointer() - if hasattr(inner, "aput_writes"): - await inner.aput_writes(config, writes, task_id) diff --git a/modules/features/chatbot/bridges/tools.py b/modules/features/chatbot/bridges/tools.py deleted file mode 100644 index 5117d7db..00000000 --- a/modules/features/chatbot/bridges/tools.py +++ /dev/null @@ -1,390 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Chatbot tools for LangGraph integration. -Includes SQL query tool, Tavily search tool, and streaming status tool. - -Tools can be created with factory functions for dynamic configuration: -- create_sql_query_tool(connector_type) - SQL query tool with configurable connector -- create_tavily_search_tool() - Tavily web search tool -- create_send_streaming_message_tool(event_manager) - Streaming status updates -""" - -import logging -import asyncio -from typing import Optional, Callable, Dict, Any -from langchain_core.tools import tool - -from modules.connectors.connectorPreprocessor import PreprocessorConnector -from modules.shared.configuration import APP_CONFIG - -logger = logging.getLogger(__name__) - -# ============================================================================= -# Connection pool for preprocessor connector (performance optimization) -# ============================================================================= - -class ConnectorPool: - """Simple connection pool for preprocessor connectors.""" - _instance: Optional['ConnectorPool'] = None - _connector: Optional[PreprocessorConnector] = None - _lock: asyncio.Lock = None - - @classmethod - def get_instance(cls) -> 'ConnectorPool': - if cls._instance is None: - cls._instance = ConnectorPool() - cls._lock = asyncio.Lock() - return cls._instance - - async def get_connector(self) -> PreprocessorConnector: - """Get or create a connector instance.""" - if self._connector is None: - self._connector = PreprocessorConnector() - return self._connector - - async def close(self): - """Close the connector.""" - if self._connector: - try: - await self._connector.close() - except Exception as e: - logger.debug(f"Error closing connector: {e}") - self._connector = None - -# Global pool instance -_connector_pool = ConnectorPool.get_instance() - - -# ============================================================================= -# Factory functions for configurable tools -# ============================================================================= - -def create_sql_query_tool(connector_type: str = "preprocessor"): - """ - Create a SQL query tool with a specific connector type. - - Args: - connector_type: Type of database connector to use (e.g., "preprocessor") - - Returns: - LangChain tool for executing SQL queries - """ - @tool - async def sqlite_query(query: str) -> str: - """ - Execute a SQL SELECT query on the database. - - This tool allows you to query the database to find articles, prices, - inventory levels, and other product information. - - Args: - query: A valid SQL SELECT query. Must use double quotes for column names - with spaces or special characters (e.g., "Artikelnummer", "S_IST_BESTAND"). - Only SELECT queries are allowed. - - Returns: - Query results as a formatted string, or an error message if the query fails. - - Examples: - - Find articles by name: - SELECT a."Artikelnummer", a."Artikelbezeichnung", a."Lieferant" - FROM Artikel a - WHERE a."Artikelbezeichnung" LIKE '%Motor%' - LIMIT 20 - - - Find articles with price and inventory: - SELECT a."Artikelnummer", a."Artikelbezeichnung", e."EP_CHF", - lp."Lagerplatz" as "Lagerplatzname", l."S_IST_BESTAND", - l."S_RESERVIERTER__BESTAND", - CASE WHEN l."S_IST_BESTAND" != 'Unbekannt' - THEN CAST(l."S_IST_BESTAND" AS INTEGER) - COALESCE(l."S_RESERVIERTER__BESTAND", 0) - ELSE NULL END as "Verfügbarer Bestand" - FROM Artikel a - LEFT JOIN Einkaufspreis e ON a."I_ID" = e."m_Artikel" - LEFT JOIN Lagerplatz_Artikel l ON a."I_ID" = l."R_ARTIKEL" - LEFT JOIN Lagerplatz lp ON l."R_LAGERPLATZ" = lp."I_ID" - WHERE a."Artikelbezeichnung" LIKE '%Netzgerät%' - LIMIT 20 - """ - try: - # Use connection pool for better performance - connector = await _connector_pool.get_connector() - - result = await connector.executeQuery(query, return_json=True) - - if result.get("text", "").startswith(("Error:", "Query failed:")): - error_msg = result.get("text", "Query failed") - logger.error(f"SQL query failed: {error_msg}") - return error_msg - - # Format results - data = result.get("data", []) - row_count = result.get("row_count", len(data)) - - if not data: - return f"Query executed successfully. Returned {row_count} rows (no data)." - - # Format as readable string - optimized for faster output - lines = [f"Query returned {row_count} rows:"] - - # Show column headers from first row - if data and isinstance(data[0], dict): - headers = list(data[0].keys()) - lines.append("Columns: " + ", ".join(headers)) - - # Show first 30 rows (reduced for faster response) - max_rows = min(30, len(data)) - for i, row in enumerate(data[:max_rows], 1): - row_str = ", ".join([f"{k}: {v}" for k, v in row.items()]) - lines.append(f"{i}. {row_str}") - - if row_count > max_rows: - lines.append(f"(Showing first {max_rows} of {row_count} rows)") - else: - # Fallback for non-dict rows - for i, row in enumerate(data[:30], 1): - lines.append(f"{i}. {row}") - - return "\n".join(lines) - # Note: Connection is reused, not closed after each query - - except Exception as e: - error_msg = f"Error executing SQL query: {str(e)}" - logger.error(error_msg, exc_info=True) - return error_msg - - return sqlite_query - - -def create_tavily_search_tool(): - """ - Create a Tavily web search tool. - - Returns: - LangChain tool for executing Tavily web searches - """ - @tool - async def tavily_search(query: str) -> str: - """ - Search the internet for comprehensive information using Tavily search via AI Center. - - Use this tool when you need to find detailed product information, datasheets, - certifications, technical specifications, market trends, or other comprehensive - information that is not in the database. - - IMPORTANT: This tool returns FULL content from search results (not truncated). - Use all available information to provide comprehensive, detailed answers with - specific facts, numbers, dates, and technical details. - - Args: - query: Search query string. Be specific and include product names, - model numbers, or other relevant keywords. For comprehensive - research, use broad queries like "latest developments in LED technology 2026" - - Returns: - Comprehensive search results with full content, titles, URLs, and sources. - Results include up to 15 sources with complete content for detailed analysis. - - Examples: - - Search for comprehensive product information: - tavily_search("latest LED technology developments 2026") - - - Search for product datasheet: - tavily_search("Siemens 6AV2 181-8XP00-0AX0 datasheet") - - - Search for market trends: - tavily_search("LED market trends efficiency breakthroughs 2025") - """ - try: - # Use AI Center Tavily plugin instead of direct langchain-tavily - from modules.aicore.aicoreModelRegistry import modelRegistry - from modules.aicore.aicoreModelSelector import modelSelector - from modules.datamodels.datamodelAi import ( - AiModelCall, - AiModelResponse, - AiCallOptions, - OperationTypeEnum, - ProcessingModeEnum, - AiCallPromptWebSearch - ) - import json - - # Discover and register connectors if not already registered - if not modelRegistry._connectors: - discovered_connectors = modelRegistry.discoverConnectors() - for connector in discovered_connectors: - modelRegistry.registerConnector(connector) - - # Refresh models to ensure Tavily is available - modelRegistry.refreshModels() - - # Get available Tavily models (without RBAC filtering since tools don't have user context) - available_models = modelRegistry.getAvailableModels() - tavily_models = [m for m in available_models if m.connectorType == "tavily"] - - if not tavily_models: - return "Error: Tavily model not available in AI Center. Please check configuration." - - # Select the best Tavily model for web search - options = AiCallOptions( - operationType=OperationTypeEnum.WEB_SEARCH_DATA, - processingMode=ProcessingModeEnum.BASIC - ) - - # Use model selector to choose the best Tavily model - # Since we only have Tavily models, we can just pick the first one - # or use selector if multiple Tavily models exist - if len(tavily_models) == 1: - selected_model = tavily_models[0] - else: - selected_model = modelSelector.selectModel( - prompt=query, - context="", - options=options, - availableModels=tavily_models - ) - - if not selected_model: - return "Error: Could not select Tavily model for web search." - - # Create web search prompt with more results and deeper research - web_search_prompt = AiCallPromptWebSearch( - instruction=query, - maxNumberPages=15, # Request more results for comprehensive information - country=None, # No country filter by default - language=None, # No language filter by default - researchDepth="deep" # Deep research for comprehensive results - ) - - # Create model call with JSON prompt - model_call = AiModelCall( - messages=[ - { - "role": "user", - "content": json.dumps(web_search_prompt.model_dump()) - } - ], - model=selected_model, - options=options - ) - - # Call the model's functionCall (which routes to _routeWebOperation) - if not selected_model.functionCall: - return "Error: Tavily model has no functionCall defined." - - response: AiModelResponse = await selected_model.functionCall(model_call) - - if not response.success: - error_msg = response.error or "Unknown error" - logger.error(f"Tavily search failed: {error_msg}") - return f"Error performing Tavily search: {error_msg}" - - # Parse response content (should be JSON with URLs and content) - try: - result_data = json.loads(response.content) if response.content else {} - - # Handle different response formats - if isinstance(result_data, list): - # List of URLs or results - results = result_data - elif isinstance(result_data, dict): - # Dictionary with URLs or results key - results = result_data.get("urls", []) or result_data.get("results", []) or [] - else: - results = [] - - if not results: - return f"No results found for query: {query}" - - # Format results with full content (not truncated) - lines = [f"Internet search results for: {query}\n"] - - # Return all results with full content (up to 15 results) - for i, result in enumerate(results[:15], 1): - if isinstance(result, str): - # Simple URL string - lines.append(f"{i}. {result}") - lines.append(f" URL: {result}") - elif isinstance(result, dict): - # Dictionary with url, title, content - url = result.get("url", "") - title = result.get("title", url) - content = result.get("content", "") - - lines.append(f"{i}. {title}") - lines.append(f" URL: {url}") - if content: - # Return FULL content, not truncated - let the LLM decide what to use - lines.append(f" Content: {content}") - else: - # Fallback - lines.append(f"{i}. {str(result)}") - lines.append("") - - return "\n".join(lines) - - except json.JSONDecodeError: - # If response is not JSON, try to parse as plain text - if response.content: - return f"Internet search results for: {query}\n\n{response.content}" - return f"No results found for query: {query}" - - except Exception as e: - error_msg = f"Error performing Tavily search via AI Center: {str(e)}" - logger.error(error_msg, exc_info=True) - return error_msg - - return tavily_search - - -# ============================================================================= -# Legacy tool definitions (kept for backwards compatibility) -# ============================================================================= - -# Legacy sqlite_query tool using default preprocessor connector -sqlite_query = create_sql_query_tool("preprocessor") - -# Legacy tavily_search tool -tavily_search = create_tavily_search_tool() - - -# ============================================================================= -# Streaming message tool factory -# ============================================================================= - -def create_send_streaming_message_tool(event_manager=None): - """ - Create the send_streaming_message tool with access to event manager. - - Args: - event_manager: Event manager instance for emitting events (not used directly, - events are captured via LangGraph tool events) - - Returns: - LangChain tool for sending streaming messages - """ - @tool - async def send_streaming_message(message: str) -> str: - """ - Send a streaming status update to the user. - - Use this tool frequently to keep the user informed about what you are doing. - This helps provide a better user experience by showing progress updates. - - Args: - message: A short message describing what you are currently doing. - Examples: - - "Durchsuche Datenbank nach Lampen, LED, Leuchten, und Ähnlichem." - - "Suche im Internet nach Produktinformationen." - - "Analysiere Suchergebnisse und bereite Antwort vor." - - Returns: - Confirmation that the message was sent. - """ - # This tool doesn't actually do anything in the tool execution - # The actual event emission happens in the streaming bridge - # This is just for LangGraph to recognize it as a tool call - return f"Status-Update gesendet: {message}" - - return send_streaming_message diff --git a/modules/features/chatbot/chatbot.py b/modules/features/chatbot/chatbot.py deleted file mode 100644 index 431248ec..00000000 --- a/modules/features/chatbot/chatbot.py +++ /dev/null @@ -1,1301 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Chatbot domain logic.""" - -import contextvars -import re -import logging -import threading -from dataclasses import dataclass, field -from typing import Annotated, AsyncIterator, Any, List, Optional, TYPE_CHECKING -from pydantic import BaseModel - -from langchain_core.messages import ( - AIMessage, - BaseMessage, - HumanMessage, - SystemMessage, - ToolMessage, - trim_messages, -) -from langgraph.graph.message import add_messages -from langgraph.graph import StateGraph, START, END -from langgraph.graph.state import CompiledStateGraph - -from modules.features.chatbot.bridges.ai import AICenterChatModel -from modules.features.chatbot.bridges.memory import ( - CheckpointerResolver, - DatabaseCheckpointer, - set_checkpointer, - reset_checkpointer, -) -from modules.features.chatbot.bridges.tools import ( - create_sql_query_tool, - create_tavily_search_tool, - create_send_streaming_message_tool, -) -from modules.datamodels.datamodelUam import User - -if TYPE_CHECKING: - from modules.features.chatbot.config import ChatbotConfig - -logger = logging.getLogger(__name__) - - -def _tool_output_to_markdown_table(raw: str) -> str: - """ - Convert sqlite_query tool output to a markdown table for deterministic display. - Reduces model hallucination by providing a ready-to-copy table. - Format: "Query returned N rows:\\nColumns: A, B, C\\n1. A: x, B: y, C: z\\n..." - """ - if not raw or not raw.strip(): - return raw - lines = [ln.strip() for ln in raw.strip().split("\n") if ln.strip()] - if len(lines) < 2: - return raw - # Parse header - row_count_line = lines[0] # "Query returned 20 rows:" - cols_line = next((ln for ln in lines if ln.lower().startswith("columns:")), None) - if not cols_line: - return raw - headers = [h.strip() for h in cols_line.replace("Columns:", "").split(",")] - if not headers: - return raw - # Parse data rows (1. Col: val, Col: val) - rows = [] - for ln in lines: - if re.match(r"^\d+\.\s+", ln): - rest = re.sub(r"^\d+\.\s+", "", ln) - row = {} - for part in rest.split(", "): - if ": " in part: - k, v = part.split(": ", 1) - row[k.strip()] = str(v).strip() - if row: - rows.append([row.get(h, "") for h in headers]) - if not rows: - return raw - # Build markdown table - sep = " | " - header_row = sep.join(headers) - div_row = sep.join(["---"] * len(headers)) - data_rows = [sep.join(str(c) for c in r) for r in rows] - table = "\n".join([header_row, div_row] + data_rows) - suffix = "" - if "(Showing first" in raw or "of " in raw: - m = re.search(r"\(Showing first (\d+) of (\d+) rows\)", raw) - if m: - suffix = f"\n\nZeige {m.group(1)} von {m.group(2)} Artikeln." - return f"{row_count_line}\n\n{table}{suffix}" - - -def _sanitize_llm_response(text: str) -> str: - """Strip chat template tokens and trailing junk that some models leak.""" - if not text or not isinstance(text, str): - return text or "" - for sentinel in ("<|im_start|>", "<|im_end|>", "<|endoftext|>", "<|user|>", "<|assistant|>"): - if sentinel in text: - text = text.split(sentinel)[0] - return text.strip() - - -# Natural language markers to split system prompt into context sections -_SPLIT_MARKERS = { - "schema_start": "Die Datenbank enthält", - "schema_start_alt": "Die Datenbank enthält die Tabellen", - "response_structure_start": "Antwortstruktur ist strikt", - "response_structure_alt": "Antwortstruktur:", - "response_structure_fallback": "Antwortstruktur", -} - - -def _split_system_prompt(prompt: str) -> dict: - """ - Split system prompt by natural language section markers. - Returns: {intro, schema, response_structure} - - intro: Role, tools, general instructions (before schema) - - schema: Database tables, SQL rules, column definitions (for SQL generation) - - response_structure: Mandatory answer format (Einleitungssatz, Tabelle, etc.) - """ - if not prompt or not isinstance(prompt, str): - return {"intro": "", "schema": "", "response_structure": ""} - - text = prompt.strip() - intro_end = len(text) - schema_start_idx = -1 - schema_end = len(text) - response_start_idx = -1 - - # Find schema start - for marker in (_SPLIT_MARKERS["schema_start"], _SPLIT_MARKERS["schema_start_alt"]): - idx = text.find(marker) - if idx >= 0: - schema_start_idx = idx - intro_end = idx - break - - # Find response structure start - for marker in ( - _SPLIT_MARKERS["response_structure_start"], - _SPLIT_MARKERS["response_structure_alt"], - _SPLIT_MARKERS["response_structure_fallback"], - ): - idx = text.find(marker) - if idx >= 0: - response_start_idx = idx - schema_end = idx if schema_start_idx >= 0 else len(text) - break - - intro = text[:intro_end].strip() if intro_end > 0 else "" - schema = ( - text[schema_start_idx:schema_end].strip() - if schema_start_idx >= 0 and schema_end > schema_start_idx - else "" - ) - response_structure = ( - text[response_start_idx:].strip() - if response_start_idx >= 0 - else "" - ) - - # Fallback: if no markers found, use full prompt for intro - if not intro and not schema and not response_structure: - intro = text - elif not response_structure and intro: - response_structure = intro # Use intro's format hints as fallback - - return {"intro": intro, "schema": schema, "response_structure": response_structure} - - -class ChatState(BaseModel): - """Represents the state of a chat session.""" - - messages: Annotated[List[BaseMessage], add_messages] - plan: Optional[str] = None # Planner routing: "SQL", "TAVILY", "BOTH", "NONE" - - -@dataclass -class ChatbotGraphContext: - """Per-request context for cached graph execution. Nodes read model/tools from here.""" - - model: AICenterChatModel - planner_model: AICenterChatModel - tools: List[Any] - tools_by_name: dict - sql_tool: Any - tavily_tool: Any - streaming_tool: Any - prompt_sections: dict - system_prompt: str - - -_graph_context: contextvars.ContextVar[Optional[ChatbotGraphContext]] = contextvars.ContextVar( - "chatbot_graph_context", default=None -) - - -def _get_graph_context() -> ChatbotGraphContext: - ctx = _graph_context.get() - if ctx is None: - raise RuntimeError( - "ChatbotGraphContext not set. Ensure graph context is set before invoking cached graph." - ) - return ctx - - -def _set_graph_context(ctx: ChatbotGraphContext) -> contextvars.Token: - return _graph_context.set(ctx) - - -def _reset_graph_context(token: contextvars.Token) -> None: - """Reset graph context. Safe when called from a different async context (e.g. generator cleanup).""" - try: - _graph_context.reset(token) - except ValueError: - # Token was created in a different context (e.g. after yield, generator cleanup) - pass - - -# Cached compiled graph; lock for thread-safe cache access -_compiled_graph_cache: Optional[CompiledStateGraph] = None -_compiled_graph_lock = threading.Lock() - - -def _get_or_build_cached_graph() -> CompiledStateGraph: - """Return cached compiled graph or build and cache it. Thread-safe.""" - global _compiled_graph_cache - with _compiled_graph_lock: - if _compiled_graph_cache is not None: - return _compiled_graph_cache - _compiled_graph_cache = _build_cached_graph() - logger.info("Chatbot: compiled graph cached for reuse") - return _compiled_graph_cache - - -def _build_cached_graph() -> CompiledStateGraph: - """Build the chatbot graph with context-resolved nodes and CheckpointerResolver.""" - checkpointer = CheckpointerResolver() - - PLANNER_SYSTEM = ( - "Du bist ein Assistent. Antworte NUR mit einem Wort: SQL, TAVILY, BOTH oder NONE.\n" - "SQL = Fragen zu Lager, Bestand, Artikel, Preisen, wie viele, Anzahl (Datenbankabfrage).\n" - "TAVILY = Internetsuche, Produktinfos außerhalb der DB, Markttrends.\n" - "BOTH = beides nötig. NONE = nur Begrüßung oder Danksagung, keine Daten nötig.\n" - "Beispiele: 'wie viele X auf Lager' -> SQL, 'Infos zu Produkt Y' -> TAVILY." - ) - SCHEMA_TRUNCATION_SUFFIX = ( - "\n\n[... Schema gekürzt. Wichtige Tabellen: Artikel, Lagerplatz_Artikel, Einkaufspreis, Lagerplatz. " - "Artikel-Spalte: a.\"Artikelbezeichnung\". " - "JOIN: Artikel a, Lagerplatz_Artikel l ON a.I_ID = l.R_ARTIKEL, Lagerplatz lp ON l.R_LAGERPLATZ = lp.I_ID.]" - ) - SQL_PLAN_SUFFIX = ( - "\n\n--- AUSGABEFORMAT (PFLICHT) ---\n" - "Antworte NUR mit einer SQL SELECT-Abfrage in diesem Format:\n" - "```sql\nDEINE_SQL_QUERY\n```\n" - "KRITISCH bei 'wie viele X auf Lager': Liefere ARTIKELZEILEN (Artikelnummer, Artikelbezeichnung, Bestand) " - "mit LIMIT 20, NICHT nur SELECT COUNT(*). Ohne Detailzeilen kann keine Tabelle angezeigt werden. " - "Gesamtanzahl optional via Unterabfrage im SELECT." - ) - FORMULATE_TASK = ( - "\n\n--- AKTUELLE AUFGABE ---\n" - "Du erhältst eine Benutzerfrage und die exakten Datenbankergebnisse. " - "KRITISCH: Nutze NUR die gelieferten Daten. Erfinde NIEMALS Daten (keine LED-A01, LED Rot, etc.). " - "Wenn die Ergebnisse NUR eine Zahl enthalten (z.B. '1. COUNT(*): 806'): Reportiere NUR diese Zahl, KEINE erfundene Tabelle. " - "Eine Tabelle darf NUR erstellt werden, wenn echte Zeilen '1. Spalte: Wert, ...' in den Daten stehen. " - "Beachte die obige ANTWORTSTRUKTUR." - ) - bytes_per_token = 3 - reserved_tokens = 3000 - _SQL_KEYWORDS = ( - "lager", "bestand", "artikel", "wie viele", "anzahl", "preis", - "lieferant", "lieferanten", "bestellen", "verfügbar", "inventar" - ) - - def _get_context_length(ctx: ChatbotGraphContext) -> int: - if hasattr(ctx.model, "_selected_model") and ctx.model._selected_model: - return getattr(ctx.model._selected_model, "contextLength", 128000) - return 128000 - - def _truncate_system_prompt(full_prompt: str, max_chars: int, suffix: str = "") -> str: - if len(full_prompt) <= max_chars: - return full_prompt - return full_prompt[: max_chars - len(suffix)] + suffix - - async def planner_node(state: ChatState) -> dict: - ctx = _get_graph_context() - human_msgs = [m for m in state.messages if isinstance(m, HumanMessage)] - last_human = human_msgs[-1].content if human_msgs else "" - window = [SystemMessage(content=PLANNER_SYSTEM), HumanMessage(content=last_human)] - plan = "SQL" - try: - response = await ctx.planner_model.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - logger.warning(f"Planner model selection failed: {exc}") - return {"plan": plan} - raise - content = (response.content or "").strip().upper() - for keyword in ("SQL", "TAVILY", "BOTH", "NONE"): - if keyword in content: - plan = keyword - break - return {"plan": plan} - - def route_by_plan(state: ChatState) -> str: - ctx = _get_graph_context() - plan = (state.plan or "SQL").upper() - if plan == "NONE" and ctx.sql_tool: - last_user = "" - for m in reversed(state.messages): - if isinstance(m, HumanMessage): - last_user = (m.content or "").lower() - break - if any(kw in last_user for kw in _SQL_KEYWORDS): - logger.info("Planner returned NONE but user asked inventory question - routing to SQL") - plan = "SQL" - if plan in ("SQL", "BOTH") and ctx.sql_tool: - return "agent_sql_plan" - if plan == "TAVILY" and ctx.tavily_tool: - return "agent_tavily" - return "agent_answer" - - def select_window(ctx: ChatbotGraphContext, msgs: List[BaseMessage], max_tokens_override: Optional[int] = None) -> List[BaseMessage]: - def approx_counter(items: List[BaseMessage]) -> int: - return sum(len(getattr(m, "content", "") or "") for m in items) - max_tokens = max_tokens_override or _get_context_length(ctx) - return trim_messages( - msgs, - strategy="last", - token_counter=approx_counter, - max_tokens=int(max_tokens * 0.8), - start_on="human", - end_on=("human", "tool"), - include_system=True, - ) - - async def _agent_common(state: ChatState, system_content: str, llm: Any, node_name: str) -> dict: - ctx = _get_graph_context() - msgs = select_window(ctx, state.messages) - if not msgs or not isinstance(msgs[0], SystemMessage): - window = [SystemMessage(content=system_content)] + msgs - else: - window = [SystemMessage(content=system_content)] + [m for m in msgs if not isinstance(m, SystemMessage)] - try: - response = await llm.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - logger.warning(f"{node_name} model selection failed: {exc}") - response = AIMessage( - content="Es tut mir leid, derzeit steht kein passendes KI-Modell für diese Anfrage zur Verfügung. " - "Bitte versuchen Sie es später erneut oder wenden Sie sich an den Administrator." - ) - else: - raise - return {"messages": [response]} - - def _parse_sql_from_content(content: str) -> Optional[str]: - if not content: - return None - match = re.search(r"```(?:sql)?\s*([\s\S]*?)```", content) - if match: - sql = match.group(1).strip() - if sql and sql.upper().strip().startswith("SELECT"): - return sql - for line in content.split("\n"): - line = line.strip() - if line.upper().startswith("SELECT"): - return line - return None - - def _sanitize_sql_typos(sql: str) -> str: - if not sql: - return sql - sql = re.sub(r"WHEN([A-Za-z_][A-Za-z0-9_.\"]*)", r"WHEN \1", sql, flags=re.IGNORECASE) - sql = re.sub(r"\bLAGerplatz_Artikel\b", "Lagerplatz_Artikel", sql) - sql = re.sub(r"\bLAGerplatz\b", "Lagerplatz", sql) - sql = sql.replace('"Einkaufspreis_neu"', '"Einkaufspreis"') - sql = sql.replace("Einkaufspreis_neu.", "Einkaufspreis.") - sql = re.sub(r'"Einkaufspreis"\."ARTIKEL"', '"Einkaufspreis"."m_Artikel"', sql) - return sql - - async def agent_sql_plan_node(state: ChatState) -> dict: - ctx = _get_graph_context() - ctx_len = _get_context_length(ctx) - max_system_chars = max(1000, int(ctx_len * 0.8 - reserved_tokens) * bytes_per_token) - len(SQL_PLAN_SUFFIX) - schema_part = ctx.prompt_sections.get("schema") or ctx.prompt_sections.get("intro", "") - intro_part = (ctx.prompt_sections.get("intro", "") or "")[:400] - combined = f"{intro_part}\n\n{schema_part}" if intro_part else schema_part - system_content = _truncate_system_prompt(combined, max_system_chars, SCHEMA_TRUNCATION_SUFFIX) + SQL_PLAN_SUFFIX - llm = ctx.model - return await _agent_common(state, system_content, llm, "agent_sql_plan") - - async def parse_execute_sql_node(state: ChatState) -> dict: - ctx = _get_graph_context() - sql_t = ctx.sql_tool - last_msg = state.messages[-1] if state.messages else None - if not isinstance(last_msg, AIMessage): - return {"messages": [ToolMessage(content="Fehler: Keine AI-Antwort zum Parsen.", tool_call_id="parse_0", name="sqlite_query")]} - sql = _parse_sql_from_content(last_msg.content or "") - if not sql or not sql_t: - return {"messages": [ToolMessage(content="Konnte keine SQL-Abfrage aus der Antwort extrahieren.", tool_call_id="parse_0", name="sqlite_query")]} - sql = _sanitize_sql_typos(sql) - try: - result = await sql_t.ainvoke({"query": sql}) - except Exception as e: - logger.error(f"SQL execution failed: {e}") - result = f"Fehler bei der Ausführung: {e}" - return {"messages": [ToolMessage(content=str(result), tool_call_id="parse_0", name="sqlite_query")]} - - async def agent_formulate_node(state: ChatState) -> dict: - ctx = _get_graph_context() - human_content = "" - tool_content = "" - for m in state.messages: - if isinstance(m, HumanMessage): - human_content = m.content or "" - if isinstance(m, ToolMessage) and getattr(m, "name", "") == "sqlite_query": - tool_content = m.content or "" - if not tool_content or not tool_content.strip(): - logger.warning("agent_formulate: no tool_content (sqlite_query) in state.messages") - return {"messages": [AIMessage(content="Die Datenbankabfrage konnte keine Ergebnisse liefern. Bitte versuchen Sie es mit einer anderen Formulierung.")]} - if "Query failed" in tool_content or tool_content.strip().startswith("Error"): - err_summary = "Die Datenbankabfrage ist fehlgeschlagen." - if "no such column" in tool_content: - err_summary += " Ein Spaltenname scheint nicht zu passen. Bitte die Anfrage anders formulieren." - return {"messages": [AIMessage(content=err_summary)]} - formatted_data = _tool_output_to_markdown_table(tool_content) - logger.debug(f"agent_formulate: tool_content length={len(tool_content)}, formatted={len(formatted_data)}") - ctx_len = _get_context_length(ctx) - max_system_chars = max(3000, int(ctx_len * 0.5) * bytes_per_token) - len(FORMULATE_TASK) - resp_struct = ctx.prompt_sections.get("response_structure") or ctx.prompt_sections.get("intro", "") - intro_formulate = ctx.prompt_sections.get("intro", "") - combined = f"{intro_formulate}\n\n{resp_struct}" if intro_formulate != resp_struct else resp_struct - if len(combined) + len(FORMULATE_TASK) > max_system_chars: - combined = _truncate_system_prompt(combined, max_system_chars - len(FORMULATE_TASK), "") - system_content = combined + FORMULATE_TASK - prompt = ( - f"Benutzerfrage: {human_content}\n\n" - "--- VORGEGEBENE DATEN (diese Tabelle/Zahlen UNVERÄNDERT in die Antwort übernehmen): ---\n" - f"{formatted_data}\n\n" - "Die obige Tabelle bzw. Zahlen sind die EINZIGEN erlaubten Daten. Kopiere sie 1:1. " - "Berechne keine eigenen Summen/Anzahlen - nutze die gelieferten Werte. Formuliere die Antwort:" - ) - window = [SystemMessage(content=system_content), HumanMessage(content=prompt)] - try: - response = await ctx.model.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - response = AIMessage(content="Es gab einen Fehler bei der Formulierung. Bitte versuchen Sie es erneut.") - else: - raise - if response.content: - response = AIMessage(content=_sanitize_llm_response(response.content)) - return {"messages": [response]} - - async def agent_tavily_node(state: ChatState) -> dict: - ctx = _get_graph_context() - resp_struct = ctx.prompt_sections.get("response_structure") or "" - intro_tavily = ctx.prompt_sections.get("intro", "") - combined = f"{intro_tavily}\n\n{resp_struct}" if resp_struct else intro_tavily - system_content = _truncate_system_prompt(combined, 6000, "") - tools_tavily = [t for t in [ctx.tavily_tool, ctx.streaming_tool] if t is not None] - llm_tavily = ctx.model.bind_tools(tools=tools_tavily) if tools_tavily else ctx.model - return await _agent_common(state, system_content, llm_tavily, "agent_tavily") - - async def agent_answer_node(state: ChatState) -> dict: - ctx = _get_graph_context() - resp_struct = ctx.prompt_sections.get("response_structure") or "" - intro_answer = ctx.prompt_sections.get("intro", "") - combined = f"{intro_answer}\n\n{resp_struct}" if resp_struct else intro_answer - system_content = _truncate_system_prompt(combined, 6000, "") - llm = ctx.planner_model if ctx.planner_model else ctx.model - return await _agent_common(state, system_content, llm, "agent_answer") - - def should_continue_tavily(state: ChatState) -> str: - last = state.messages[-1] - return "tools" if getattr(last, "tool_calls", None) else END - - def route_back(state: ChatState) -> str: - ctx = _get_graph_context() - return "agent_tavily" if ctx.tavily_tool else "agent_answer" - - async def tools_with_retry(state: ChatState) -> dict: - import asyncio - ctx = _get_graph_context() - last_message = state.messages[-1] - tool_calls = getattr(last_message, "tool_calls", []) - if not tool_calls: - return {"messages": []} - tools_by_name = ctx.tools_by_name - - async def execute_single_tool(tool_call): - tool_name = tool_call.get("name") or tool_call.get("function", {}).get("name") - tool_id = tool_call.get("id", f"call_{tool_name}") - args = tool_call.get("args") or tool_call.get("function", {}).get("arguments", {}) - if isinstance(args, str): - import json - try: - args = json.loads(args) - except Exception: - args = {"input": args} - tool = tools_by_name.get(tool_name) - if not tool: - return ToolMessage(content=f"Error: Tool '{tool_name}' not found", tool_call_id=tool_id, name=tool_name) - try: - if hasattr(tool, "coroutine") and asyncio.iscoroutinefunction(tool.coroutine): - result = await tool.coroutine(**args) - elif hasattr(tool, "ainvoke"): - result = await tool.ainvoke(args) - else: - result = tool.invoke(args) - return ToolMessage(content=str(result), tool_call_id=tool_id, name=tool_name) - except Exception as e: - logger.error(f"Tool {tool_name} failed: {e}") - return ToolMessage(content=f"Error executing {tool_name}: {str(e)}", tool_call_id=tool_id, name=tool_name) - - tool_messages = await asyncio.gather( - *[execute_single_tool(tc) for tc in tool_calls], - return_exceptions=True - ) - result_messages = [] - for i, msg in enumerate(tool_messages): - if isinstance(msg, Exception): - tool_call = tool_calls[i] - tool_name = tool_call.get("name", "unknown") - tool_id = tool_call.get("id", f"call_{i}") - result_messages.append(ToolMessage(content=f"Error: {str(msg)}", tool_call_id=tool_id, name=tool_name)) - else: - result_messages.append(msg) - result = {"messages": result_messages} - no_results_keywords = [ - "returned 0 rows", "no data", "keine artikel gefunden", "keine ergebnisse" - ] - for msg in result.get("messages", []): - content = getattr(msg, "content", "") - if isinstance(content, str): - content_lower = content.lower() - if any(keyword in content_lower for keyword in no_results_keywords): - retry_count = sum(1 for m in state.messages if "retry" in str(getattr(m, "content", "")).lower()) - if retry_count < 2: - logger.info("No results found in tool output, adding retry instruction") - retry_message = HumanMessage( - content="WICHTIG: Die vorherige Suche hat keine Ergebnisse gefunden. " - "Bitte versuche eine alternative Suchstrategie:\n" - "1. Wenn die Frage im Format 'X von Y' war (z.B. 'Lampen von Eaton'), " - "verwende IMMER eine Kombination aus Lieferanten-Filter (WHERE a.\"Lieferant\" LIKE '%Y%') " - "UND Produkttyp-Filter (WHERE a.\"Artikelbezeichnung\" LIKE '%X%' OR ...)\n" - "2. Verwende mehrere Synonyme für den Produkttyp (z.B. bei 'Lampen': Lampe, LED, Beleuchtung, Licht, Leuchte, Strahler)\n" - "3. Führe zuerst eine COUNT-Abfrage durch, dann die Detail-Abfrage mit Lagerbeständen\n" - "4. Verwende LIKE '%Lieferant%' für den Lieferanten-Filter, um auch Varianten zu finden" - ) - result["messages"].append(retry_message) - break - return result - - workflow = StateGraph(ChatState) - workflow.add_node("planner", planner_node) - workflow.add_node("agent_sql_plan", agent_sql_plan_node) - workflow.add_node("parse_execute_sql", parse_execute_sql_node) - workflow.add_node("agent_formulate", agent_formulate_node) - workflow.add_node("tools", tools_with_retry) - workflow.add_node("agent_tavily", agent_tavily_node) - workflow.add_node("agent_answer", agent_answer_node) - workflow.add_edge(START, "planner") - workflow.add_conditional_edges("planner", route_by_plan) - workflow.add_edge("agent_sql_plan", "parse_execute_sql") - workflow.add_edge("parse_execute_sql", "agent_formulate") - workflow.add_edge("agent_formulate", END) - workflow.add_conditional_edges("agent_tavily", should_continue_tavily) - workflow.add_edge("agent_answer", END) - workflow.add_conditional_edges("tools", route_back) - return workflow.compile(checkpointer=checkpointer) - - -@dataclass -class Chatbot: - """Represents a chatbot.""" - - model: AICenterChatModel - memory: DatabaseCheckpointer - planner_model: Optional[AICenterChatModel] = None # Fast model for routing (SQL/TAVILY/NONE) - app: CompiledStateGraph = None - _tools: List[Any] = field(default_factory=list) # Configured tools (for cached graph context) - system_prompt: str = "You are a helpful assistant." - workflow_id: str = "default" - config: Optional["ChatbotConfig"] = None - _event_manager: Any = None - _chat_streaming_helper: Any = None # From service center streaming service - - @classmethod - async def create( - cls, - model: AICenterChatModel, - memory: DatabaseCheckpointer, - system_prompt: str, - workflow_id: str = "default", - config: Optional["ChatbotConfig"] = None, - event_manager=None, - planner_model: Optional[AICenterChatModel] = None, - chat_streaming_helper=None, - ) -> "Chatbot": - """Factory method to create and configure a Chatbot instance. - - Args: - model: The chat model to use (AICenterChatModel). - memory: The chat memory to use (DatabaseCheckpointer). - system_prompt: The system prompt to initialize the chatbot. - workflow_id: The workflow ID (maps to thread_id). - config: Optional chatbot configuration for dynamic tool enablement. - event_manager: Optional event manager for streaming (passed from route). - planner_model: Optional fast model for planner/routing (default: same as model). - chat_streaming_helper: ChatStreamingHelper from service center streaming service. - - Returns: - A configured Chatbot instance. - """ - instance = Chatbot( - model=model, - memory=memory, - system_prompt=system_prompt, - workflow_id=workflow_id, - config=config, - _event_manager=event_manager, - planner_model=planner_model, - _chat_streaming_helper=chat_streaming_helper, - ) - configured_tools = await instance._configure_tools() - instance._tools = configured_tools - instance.app = _get_or_build_cached_graph() - return instance - - async def _configure_tools(self) -> List[Any]: - """Configure tools for the chatbot based on config. - - Returns: - List of configured tools based on config settings. - """ - tools = [] - - # Get tool enablement from config (use defaults if no config) - sql_enabled = True - tavily_enabled = False - streaming_enabled = True - connector_type = "preprocessor" - - if self.config: - sql_enabled = self.config.tools.is_sql_enabled() - tavily_enabled = self.config.tools.is_tavily_enabled() - streaming_enabled = self.config.tools.is_streaming_enabled() - connector_type = self.config.database.connector - - logger.info(f"Chatbot tools config - SQL: {sql_enabled}, Tavily: {tavily_enabled}, " - f"Streaming: {streaming_enabled}, Connector: {connector_type}") - - # SQL query tool (if enabled) - if sql_enabled: - sql_tool = create_sql_query_tool(connector_type=connector_type) - tools.append(sql_tool) - logger.debug(f"Added SQL query tool with connector: {connector_type}") - - # Tavily search tool (if enabled) - if tavily_enabled: - tavily_tool = create_tavily_search_tool() - tools.append(tavily_tool) - logger.debug("Added Tavily search tool") - - # Streaming status tool (if enabled and event_manager available) - if streaming_enabled and self._event_manager: - send_streaming_message = create_send_streaming_message_tool(self._event_manager) - tools.append(send_streaming_message) - logger.debug("Added streaming status tool") - - logger.info(f"Configured {len(tools)} tools for chatbot workflow {self.workflow_id}") - return tools - - def _build_app( - self, memory: DatabaseCheckpointer, tools: List[Any] - ) -> CompiledStateGraph[ChatState, None, ChatState, ChatState]: - """Builds the chatbot application workflow using LangGraph. - - Supports small context windows via planning phase and tiered prompts. - - Args: - memory: The chat memory to use. - tools: The list of tools the chatbot can use. - - Returns: - A compiled state graph representing the chatbot application. - """ - # Build tool subsets per agent type - tools_by_name = {t.name: t for t in tools} - sql_tool = tools_by_name.get("sqlite_query") - tavily_tool = tools_by_name.get("tavily_search") - streaming_tool = tools_by_name.get("send_streaming_message") - tools_sql = [t for t in [sql_tool, tavily_tool, streaming_tool] if t is not None] - tools_tavily = [t for t in [tavily_tool, streaming_tool] if t is not None] - llm_plain = self.model - llm_planner = self.planner_model if self.planner_model else self.model - # SQL path uses structured prompts + parse/execute (no native tool calling) - fits /api/analyze - llm_tavily = self.model.bind_tools(tools=tools_tavily) if tools_tavily else self.model - - # Minimal planner prompt (~250 tokens) - fits any 8K+ model - # Explicit: Lager, Bestand, Artikel, wie viele = SQL (Datenbank) - PLANNER_SYSTEM = ( - "Du bist ein Assistent. Antworte NUR mit einem Wort: SQL, TAVILY, BOTH oder NONE.\n" - "SQL = Fragen zu Lager, Bestand, Artikel, Preisen, wie viele, Anzahl (Datenbankabfrage).\n" - "TAVILY = Internetsuche, Produktinfos außerhalb der DB, Markttrends.\n" - "BOTH = beides nötig. NONE = nur Begrüßung oder Danksagung, keine Daten nötig.\n" - "Beispiele: 'wie viele X auf Lager' -> SQL, 'Infos zu Produkt Y' -> TAVILY." - ) - - # Truncation suffix for schema when prompt is cut - SCHEMA_TRUNCATION_SUFFIX = ( - "\n\n[... Schema gekürzt. Wichtige Tabellen: Artikel, Lagerplatz_Artikel, Einkaufspreis, Lagerplatz. " - "Artikel-Spalte: a.\"Artikelbezeichnung\". " - "JOIN: Artikel a, Lagerplatz_Artikel l ON a.I_ID = l.R_ARTIKEL, Lagerplatz lp ON l.R_LAGERPLATZ = lp.I_ID.]" - ) - - # Structured output for /api/analyze (no tool calls): model outputs SQL in code block, we parse and execute - SQL_PLAN_SUFFIX = ( - "\n\n--- AUSGABEFORMAT (PFLICHT) ---\n" - "Antworte NUR mit einer SQL SELECT-Abfrage in diesem Format:\n" - "```sql\nDEINE_SQL_QUERY\n```\n" - "KRITISCH bei 'wie viele X auf Lager': Liefere ARTIKELZEILEN (Artikelnummer, Artikelbezeichnung, Bestand) " - "mit LIMIT 20, NICHT nur SELECT COUNT(*). Ohne Detailzeilen kann keine Tabelle angezeigt werden. " - "Gesamtanzahl optional via Unterabfrage im SELECT." - ) - - bytes_per_token = 3 # Balanced estimate for mixed content - reserved_tokens = 3000 # Tools block + conversation overhead - - def _get_context_length() -> int: - """Get selected model's context length; pre-select if needed.""" - if hasattr(self.model, "_selected_model") and self.model._selected_model: - return getattr(self.model._selected_model, "contextLength", 128000) - return 128000 - - def _truncate_system_prompt(full_prompt: str, max_chars: int, suffix: str = "") -> str: - """Truncate system prompt to fit context budget.""" - if len(full_prompt) <= max_chars: - return full_prompt - return full_prompt[: max_chars - len(suffix)] + suffix - - # Split system prompt by natural language sections for targeted context - _prompt_sections = _split_system_prompt(self.system_prompt) - - def select_window(msgs: List[BaseMessage], max_tokens_override: Optional[int] = None) -> List[BaseMessage]: - """Selects a window of messages that fit within the context window size.""" - - def approx_counter(items: List[BaseMessage]) -> int: - return sum(len(getattr(m, "content", "") or "") for m in items) - - max_tokens = max_tokens_override or _get_context_length() - return trim_messages( - msgs, - strategy="last", - token_counter=approx_counter, - max_tokens=int(max_tokens * 0.8), - start_on="human", - end_on=("human", "tool"), - include_system=True, - ) - - async def planner_node(state: ChatState) -> dict: - """Planner: minimal prompt, no tools. Outputs SQL/TAVILY/BOTH/NONE. - Does NOT add planner message to chat - only sets state.plan for routing. - Uses llm_planner (fast model) when available for lower latency.""" - human_msgs = [m for m in state.messages if isinstance(m, HumanMessage)] - last_human = human_msgs[-1].content if human_msgs else "" - window = [ - SystemMessage(content=PLANNER_SYSTEM), - HumanMessage(content=last_human), - ] - plan = "SQL" - try: - response = await llm_planner.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - logger.warning(f"Planner model selection failed: {exc}") - return {"plan": plan} - raise - content = (response.content or "").strip().upper() - for keyword in ("SQL", "TAVILY", "BOTH", "NONE"): - if keyword in content: - plan = keyword - break - return {"plan": plan} - - # Keywords that indicate database/inventory query - override NONE to SQL - _SQL_KEYWORDS = ( - "lager", "bestand", "artikel", "wie viele", "anzahl", "preis", - "lieferant", "lieferanten", "bestellen", "verfügbar", "inventar" - ) - - def route_by_plan(state: ChatState) -> str: - """Route from planner to agent_sql_plan, agent_tavily, or agent_answer.""" - plan = (state.plan or "SQL").upper() - # Override NONE when user clearly asks for inventory/data (e.g. "wie viele LEDs auf Lager") - if plan == "NONE" and sql_tool: - last_user = "" - for m in reversed(state.messages): - if isinstance(m, HumanMessage): - last_user = (m.content or "").lower() - break - if any(kw in last_user for kw in _SQL_KEYWORDS): - logger.info("Planner returned NONE but user asked inventory question - routing to SQL") - plan = "SQL" - if plan in ("SQL", "BOTH") and sql_tool: - return "agent_sql_plan" - if plan == "TAVILY" and tavily_tool: - return "agent_tavily" - return "agent_answer" - - async def _agent_common( - state: ChatState, - system_content: str, - llm, - node_name: str, - ) -> dict: - """Shared logic for agent nodes.""" - msgs = select_window(state.messages) - if not msgs or not isinstance(msgs[0], SystemMessage): - window = [SystemMessage(content=system_content)] + msgs - else: - window = [SystemMessage(content=system_content)] + [m for m in msgs if not isinstance(m, SystemMessage)] - try: - response = await llm.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - logger.warning(f"{node_name} model selection failed: {exc}") - response = AIMessage( - content=( - "Es tut mir leid, derzeit steht kein passendes KI-Modell für diese Anfrage zur Verfügung. " - "Bitte versuchen Sie es später erneut oder wenden Sie sich an den Administrator." - ) - ) - else: - raise - return {"messages": [response]} - - async def agent_sql_plan_node(state: ChatState) -> dict: - """Generate SQL. Uses schema section + minimal intro. Output: ```sql...``` for parse/execute.""" - ctx_len = _get_context_length() - max_system_chars = max(1000, int(ctx_len * 0.8 - reserved_tokens) * bytes_per_token) - len(SQL_PLAN_SUFFIX) - # Prefer schema section; add short intro if space allows - schema_part = _prompt_sections["schema"] or _prompt_sections["intro"] - intro_part = _prompt_sections["intro"][:400] if _prompt_sections["intro"] else "" - combined = f"{intro_part}\n\n{schema_part}" if intro_part else schema_part - system_content = _truncate_system_prompt( - combined, max_system_chars, SCHEMA_TRUNCATION_SUFFIX - ) + SQL_PLAN_SUFFIX - return await _agent_common(state, system_content, llm_plain, "agent_sql_plan") - - def _parse_sql_from_content(content: str) -> Optional[str]: - """Extract SQL from ```sql...``` or ```...``` code block. Only allows SELECT.""" - if not content: - return None - match = re.search(r"```(?:sql)?\s*([\s\S]*?)```", content) - if match: - sql = match.group(1).strip() - if sql and sql.upper().strip().startswith("SELECT"): - return sql - # Fallback: find line starting with SELECT - for line in content.split("\n"): - line = line.strip() - if line.upper().startswith("SELECT"): - return line - return None - - def _sanitize_sql_typos(sql: str) -> str: - """Fix common LLM SQL typos that cause syntax errors.""" - if not sql: - return sql - # Fix "CASE WHENLAGerplatz" - missing space after WHEN when followed directly by identifier - sql = re.sub(r"WHEN([A-Za-z_][A-Za-z0-9_.\"]*)", r"WHEN \1", sql, flags=re.IGNORECASE) - # Fix "LAGerplatz_Artikel" / "LAGerplatz" -> correct casing - sql = re.sub(r"\bLAGerplatz_Artikel\b", "Lagerplatz_Artikel", sql) - sql = re.sub(r"\bLAGerplatz\b", "Lagerplatz", sql) - # Preprocessor uses Einkaufspreis (not Einkaufspreis_neu) and m_Artikel (not ARTIKEL) - sql = sql.replace('"Einkaufspreis_neu"', '"Einkaufspreis"') - sql = sql.replace("Einkaufspreis_neu.", "Einkaufspreis.") - sql = re.sub( - r'"Einkaufspreis"\."ARTIKEL"', - '"Einkaufspreis"."m_Artikel"', - sql, - ) - return sql - - async def parse_execute_sql_node(state: ChatState) -> dict: - """Parse SQL from last AIMessage, execute via preprocessor, add ToolMessage.""" - last_msg = state.messages[-1] if state.messages else None - if not isinstance(last_msg, AIMessage): - return {"messages": [ToolMessage(content="Fehler: Keine AI-Antwort zum Parsen.", tool_call_id="parse_0", name="sqlite_query")]} - sql = _parse_sql_from_content(last_msg.content or "") - if not sql or not sql_tool: - return {"messages": [ToolMessage(content="Konnte keine SQL-Abfrage aus der Antwort extrahieren.", tool_call_id="parse_0", name="sqlite_query")]} - sql = _sanitize_sql_typos(sql) - try: - result = await sql_tool.ainvoke({"query": sql}) - except Exception as e: - logger.error(f"SQL execution failed: {e}") - result = f"Fehler bei der Ausführung: {e}" - return {"messages": [ToolMessage(content=str(result), tool_call_id="parse_0", name="sqlite_query")]} - - FORMULATE_TASK = ( - "\n\n--- AKTUELLE AUFGABE ---\n" - "Du erhältst eine Benutzerfrage und die exakten Datenbankergebnisse. " - "KRITISCH: Nutze NUR die gelieferten Daten. Erfinde NIEMALS Daten (keine LED-A01, LED Rot, etc.). " - "Wenn die Ergebnisse NUR eine Zahl enthalten (z.B. '1. COUNT(*): 806'): Reportiere NUR diese Zahl, KEINE erfundene Tabelle. " - "Eine Tabelle darf NUR erstellt werden, wenn echte Zeilen '1. Spalte: Wert, ...' in den Daten stehen. " - "Beachte die obige ANTWORTSTRUKTUR." - ) - - async def agent_formulate_node(state: ChatState) -> dict: - """Formulate final answer. Uses intro + response_structure sections (not schema).""" - human_content = "" - tool_content = "" - for m in state.messages: - if isinstance(m, HumanMessage): - human_content = m.content or "" - if isinstance(m, ToolMessage) and getattr(m, "name", "") == "sqlite_query": - tool_content = m.content or "" - if not tool_content or not tool_content.strip(): - logger.warning("agent_formulate: no tool_content (sqlite_query) in state.messages") - return {"messages": [AIMessage(content="Die Datenbankabfrage konnte keine Ergebnisse liefern. Bitte versuchen Sie es mit einer anderen Formulierung.")]} - # When SQL failed, return error directly - don't let model hallucinate success - if "Query failed" in tool_content or tool_content.strip().startswith("Error"): - err_summary = "Die Datenbankabfrage ist fehlgeschlagen." - if "no such column" in tool_content: - err_summary += " Ein Spaltenname scheint nicht zu passen. Bitte die Anfrage anders formulieren." - return {"messages": [AIMessage(content=err_summary)]} - # Convert to markdown table so model copies exact values instead of reformatting/hallucinating - formatted_data = _tool_output_to_markdown_table(tool_content) - logger.debug(f"agent_formulate: tool_content length={len(tool_content)}, formatted={len(formatted_data)}") - ctx_len = _get_context_length() - max_system_chars = max(3000, int(ctx_len * 0.5) * bytes_per_token) - len(FORMULATE_TASK) - # Use intro + response_structure (mandatory format) - resp_struct = _prompt_sections["response_structure"] or _prompt_sections["intro"] - intro_formulate = _prompt_sections["intro"] - combined = f"{intro_formulate}\n\n{resp_struct}" if intro_formulate != resp_struct else resp_struct - # Fit within context; prefer keeping response_structure intact - if len(combined) + len(FORMULATE_TASK) > max_system_chars: - combined = _truncate_system_prompt(combined, max_system_chars - len(FORMULATE_TASK), "") - system_content = combined + FORMULATE_TASK - prompt = ( - f"Benutzerfrage: {human_content}\n\n" - "--- VORGEGEBENE DATEN (diese Tabelle/Zahlen UNVERÄNDERT in die Antwort übernehmen): ---\n" - f"{formatted_data}\n\n" - "Die obige Tabelle bzw. Zahlen sind die EINZIGEN erlaubten Daten. Kopiere sie 1:1. " - "Berechne keine eigenen Summen/Anzahlen - nutze die gelieferten Werte. Formuliere die Antwort:" - ) - window = [SystemMessage(content=system_content), HumanMessage(content=prompt)] - try: - response = await llm_plain.ainvoke(window) - except ValueError as exc: - if "No suitable model found" in str(exc): - response = AIMessage(content="Es gab einen Fehler bei der Formulierung. Bitte versuchen Sie es erneut.") - else: - raise - # Sanitize: strip leaked chat template tokens (<|im_start|> etc.) and trailing junk - if response.content: - response = AIMessage(content=_sanitize_llm_response(response.content)) - return {"messages": [response]} - - async def agent_tavily_node(state: ChatState) -> dict: - """Agent with Tavily only. Uses intro + response_structure (no schema).""" - resp_struct = _prompt_sections["response_structure"] or "" - intro_tavily = _prompt_sections["intro"] - combined = f"{intro_tavily}\n\n{resp_struct}" if resp_struct else intro_tavily - system_content = _truncate_system_prompt(combined, 6000, "") - return await _agent_common(state, system_content, llm_tavily, "agent_tavily") - - async def agent_answer_node(state: ChatState) -> dict: - """Agent with no tools (plan NONE). Uses fast model for lower latency.""" - resp_struct = _prompt_sections["response_structure"] or "" - intro_answer = _prompt_sections["intro"] - combined = f"{intro_answer}\n\n{resp_struct}" if resp_struct else intro_answer - system_content = _truncate_system_prompt(combined, 6000, "") - return await _agent_common(state, system_content, llm_planner, "agent_answer") - - def should_continue_tavily(state: ChatState) -> str: - last = state.messages[-1] - return "tools" if getattr(last, "tool_calls", None) else END - - def route_back(state: ChatState) -> str: - """Route from tools back to agent_tavily (SQL path uses parse_execute_sql, no tools loop).""" - # Tools node is only reached from agent_tavily when it returns tool_calls - return "agent_tavily" if tavily_tool else "agent_answer" - - async def tools_with_retry(state: ChatState) -> dict: - """Tools node with parallel execution and retry logic. - - Args: - state: The current chat state. - - Returns: - The updated chat state after tool execution. - """ - import asyncio - - # Get tool calls from the last message - last_message = state.messages[-1] - tool_calls = getattr(last_message, "tool_calls", []) - - if not tool_calls: - return {"messages": []} - - # Create a lookup for tools by name - tools_by_name = {t.name: t for t in tools} - - async def execute_single_tool(tool_call): - """Execute a single tool call.""" - tool_name = tool_call.get("name") or tool_call.get("function", {}).get("name") - tool_id = tool_call.get("id", f"call_{tool_name}") - args = tool_call.get("args") or tool_call.get("function", {}).get("arguments", {}) - - if isinstance(args, str): - import json - try: - args = json.loads(args) - except: - args = {"input": args} - - tool = tools_by_name.get(tool_name) - if not tool: - return ToolMessage( - content=f"Error: Tool '{tool_name}' not found", - tool_call_id=tool_id, - name=tool_name - ) - - try: - # Execute tool asynchronously - if asyncio.iscoroutinefunction(tool.coroutine): - result = await tool.coroutine(**args) - elif hasattr(tool, 'ainvoke'): - result = await tool.ainvoke(args) - else: - result = tool.invoke(args) - - return ToolMessage( - content=str(result), - tool_call_id=tool_id, - name=tool_name - ) - except Exception as e: - logger.error(f"Tool {tool_name} failed: {e}") - return ToolMessage( - content=f"Error executing {tool_name}: {str(e)}", - tool_call_id=tool_id, - name=tool_name - ) - - # Execute ALL tool calls in parallel - logger.info(f"Executing {len(tool_calls)} tool calls in parallel") - tool_messages = await asyncio.gather( - *[execute_single_tool(tc) for tc in tool_calls], - return_exceptions=True - ) - - # Convert exceptions to error messages - result_messages = [] - for i, msg in enumerate(tool_messages): - if isinstance(msg, Exception): - tool_call = tool_calls[i] - tool_name = tool_call.get("name", "unknown") - tool_id = tool_call.get("id", f"call_{i}") - result_messages.append(ToolMessage( - content=f"Error: {str(msg)}", - tool_call_id=tool_id, - name=tool_name - )) - else: - result_messages.append(msg) - - result = {"messages": result_messages} - - # Check if we got no results and should retry - no_results_keywords = [ - "returned 0 rows", - "no data", - "keine artikel gefunden", - "keine ergebnisse" - ] - - # Check tool results for no data - for msg in result.get("messages", []): - content = getattr(msg, "content", "") - if isinstance(content, str): - content_lower = content.lower() - if any(keyword in content_lower for keyword in no_results_keywords): - # Check if we haven't retried yet (avoid infinite loops) - retry_count = sum(1 for m in state.messages if "retry" in str(getattr(m, "content", "")).lower()) - if retry_count < 2: # Allow max 2 retries - logger.info("No results found in tool output, adding retry instruction") - retry_message = HumanMessage( - content="WICHTIG: Die vorherige Suche hat keine Ergebnisse gefunden. " - "Bitte versuche eine alternative Suchstrategie:\n" - "1. Wenn die Frage im Format 'X von Y' war (z.B. 'Lampen von Eaton'), " - "verwende IMMER eine Kombination aus Lieferanten-Filter (WHERE a.\"Lieferant\" LIKE '%Y%') " - "UND Produkttyp-Filter (WHERE a.\"Artikelbezeichnung\" LIKE '%X%' OR ...)\n" - "2. Verwende mehrere Synonyme für den Produkttyp (z.B. bei 'Lampen': Lampe, LED, Beleuchtung, Licht, Leuchte, Strahler)\n" - "3. Führe zuerst eine COUNT-Abfrage durch, dann die Detail-Abfrage mit Lagerbeständen\n" - "4. Verwende LIKE '%Lieferant%' für den Lieferanten-Filter, um auch Varianten zu finden" - ) - result["messages"].append(retry_message) - break - - return result - - # Compose the workflow: planner -> route -> agent_* -> tools (Tavily only) or END - workflow = StateGraph(ChatState) - workflow.add_node("planner", planner_node) - workflow.add_node("agent_sql_plan", agent_sql_plan_node) - workflow.add_node("parse_execute_sql", parse_execute_sql_node) - workflow.add_node("agent_formulate", agent_formulate_node) - workflow.add_node("agent_tavily", agent_tavily_node) - workflow.add_node("agent_answer", agent_answer_node) - workflow.add_node("tools", tools_with_retry) - workflow.add_edge(START, "planner") - workflow.add_conditional_edges("planner", route_by_plan) - # SQL path: agent_sql_plan -> parse_execute_sql -> agent_formulate -> END (no tools, /api/analyze compatible) - workflow.add_edge("agent_sql_plan", "parse_execute_sql") - workflow.add_edge("parse_execute_sql", "agent_formulate") - workflow.add_edge("agent_formulate", END) - workflow.add_conditional_edges("agent_tavily", should_continue_tavily) - workflow.add_edge("agent_answer", END) - workflow.add_conditional_edges("tools", route_back) - return workflow.compile(checkpointer=memory) - - async def chat(self, message: str, chat_id: str = "default") -> List[BaseMessage]: - """Processes a chat message by calling the LLM and tools and returns the chat history. - - Args: - message: The user message to process. - chat_id: The chat thread ID. - - Returns: - The list of messages in the chat history. - """ - config = {"configurable": {"thread_id": chat_id}} - tools_by_name = {t.name: t for t in self._tools} - graph_ctx = ChatbotGraphContext( - model=self.model, - planner_model=self.planner_model or self.model, - tools=self._tools, - tools_by_name=tools_by_name, - sql_tool=tools_by_name.get("sqlite_query"), - tavily_tool=tools_by_name.get("tavily_search"), - streaming_tool=tools_by_name.get("send_streaming_message"), - prompt_sections=_split_system_prompt(self.system_prompt), - system_prompt=self.system_prompt, - ) - ctx_token = _set_graph_context(graph_ctx) - cp_token = set_checkpointer(self.memory) - try: - result = await self.app.ainvoke( - {"messages": [HumanMessage(content=message)]}, config=config - ) - return result["messages"] - finally: - _reset_graph_context(ctx_token) - reset_checkpointer(cp_token) - - async def stream_events( - self, *, message: str, chat_id: str = "default" - ) -> AsyncIterator[dict]: - """Stream UI-focused events using astream_events v2. - - Args: - message: The user message to process. - chat_id: Logical thread identifier; forwarded in the runnable config so - memory and tools are scoped per thread. - - Yields: - dict: One of: - - ``{"type": "status", "label": str}`` for short progress updates. - - ``{"type": "final", "response": {"thread": str, "chat_history": list[dict]}}`` - where ``chat_history`` only includes ``user``/``assistant`` roles. - - ``{"type": "error", "message": str}`` if an exception occurs. - """ - # Thread-aware config for LangGraph/LangChain - config = {"configurable": {"thread_id": chat_id}} - - def _is_root(ev: dict) -> bool: - """Return True if the event is from the root run (v2: empty parent_ids).""" - return not ev.get("parent_ids") - - # Build tool lookup for cached graph context - tools_by_name = {t.name: t for t in self._tools} - graph_ctx = ChatbotGraphContext( - model=self.model, - planner_model=self.planner_model or self.model, - tools=self._tools, - tools_by_name=tools_by_name, - sql_tool=tools_by_name.get("sqlite_query"), - tavily_tool=tools_by_name.get("tavily_search"), - streaming_tool=tools_by_name.get("send_streaming_message"), - prompt_sections=_split_system_prompt(self.system_prompt), - system_prompt=self.system_prompt, - ) - ctx_token = _set_graph_context(graph_ctx) - cp_token = set_checkpointer(self.memory) - try: - async for event in self.app.astream_events( - {"messages": [HumanMessage(content=message)]}, - config=config, - version="v2", - ): - etype = event.get("event") - ename = event.get("name") or "" - edata = event.get("data") or {} - - # Stream LLM tokens for ChatGPT-like incremental display - if etype in ("on_llm_stream", "on_chat_model_stream"): - ch = edata.get("chunk") - if ch is None: - continue - # Chunk can be string, AIMessageChunk (has .content), or dict - content = "" - if isinstance(ch, str): - content = ch - elif hasattr(ch, "content"): - content = ch.content or "" - if isinstance(content, list): - content = "".join(str(x) for x in content) - elif isinstance(ch, dict): - content = ch.get("content", "") or "" - if isinstance(content, str) and content: - yield {"type": "chunk", "content": content} - continue - - # Stream human-readable progress via the special send_streaming_message tool - # Match the legacy implementation exactly (line 267-272 in legacy/chatbot.py) - if etype == "on_tool_start": - # Log all tool starts to debug - logger.debug(f"Tool start event: name='{ename}', event='{etype}'") - if ename == "send_streaming_message": - tool_in = edata.get("input") or {} - msg = tool_in.get("message") - logger.info(f"send_streaming_message tool called with input: {tool_in}") - if isinstance(msg, str) and msg.strip(): - logger.info(f"Status-Update gesendet: {msg.strip()}") - yield {"type": "status", "label": msg.strip()} - continue - - # Emit the final payload when the root run finishes - if etype == "on_chain_end" and _is_root(event): - output_obj = edata.get("output") - - # Extract message list from the graph's final output (ChatStreamingHelper from service center) - helper = self._chat_streaming_helper - if not helper: - raise RuntimeError("ChatStreamingHelper required; pass chat_streaming_helper to Chatbot.create()") - final_msgs = helper.extract_messages_from_output(output_obj=output_obj) - - # Normalize for the frontend (only user/assistant with text content) - # Exclude planner-only and SQL-path intermediate messages from chat display - _planner_only = frozenset(("sql", "tavily", "both", "none")) - chat_history_payload: List[dict] = [] - for m in final_msgs: - if isinstance(m, BaseMessage): - d = helper.message_to_dict(msg=m) - elif isinstance(m, dict): - d = helper.dict_message_to_dict(obj=m) - else: - continue - if d.get("role") not in ("user", "assistant") or not d.get("content"): - continue - content = (d.get("content") or "").strip() - if d.get("role") == "assistant" and content.lower() in _planner_only: - continue # Skip planner routing message - # Skip agent_sql_plan output: ```sql block OR raw SQL (SELECT...FROM/JOIN) - if d.get("role") == "assistant": - cu = content.upper() - if content.startswith("```") or ( - cu.startswith("SELECT") and ("FROM" in cu or "JOIN" in cu) - ): - continue - # Strip leaked chat template tokens (<|im_start|> etc.) from assistant messages - content = _sanitize_llm_response(content) - if not content: - continue - d = {**d, "content": content} - chat_history_payload.append(d) - - yield { - "type": "final", - "response": { - "thread": chat_id, - "chat_history": chat_history_payload, - }, - } - return - - except Exception as exc: - # Emit a single error envelope and end the stream - logger.error(f"Exception in stream_events: {exc}", exc_info=True) - yield {"type": "error", "message": f"Fehler beim Verarbeiten: {exc}"} - finally: - _reset_graph_context(ctx_token) - reset_checkpointer(cp_token) diff --git a/modules/features/chatbot/chatbotConstants.py b/modules/features/chatbot/chatbotConstants.py deleted file mode 100644 index 0366e4b4..00000000 --- a/modules/features/chatbot/chatbotConstants.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Chatbot constants and helper functions. -""" - -import logging -from typing import Optional - -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum - -logger = logging.getLogger(__name__) - - -async def generate_conversation_name( - services, - prompt: str, - user_language: Optional[str] = None -) -> str: - """ - Generate a conversation name from the user's prompt using AI. - Creates a concise, informative summary name in German based on the user input. - - Args: - services: Services object with AI service - prompt: User's input prompt (always in German) - user_language: User's language preference (not used, always German) - - Returns: - A short, informative conversation name in German - """ - if not prompt or not prompt.strip(): - return "Neue Unterhaltung" - - try: - # Check if AI service is available - if not hasattr(services, 'ai') or services.ai is None: - logger.warning("AI service not available, generating name from prompt") - return generate_name_from_prompt(prompt) - - # Ensure AI service is initialized before use - await services.ai.ensureAiObjectsInitialized() - - # Create AI prompt - very explicit that answer must be in German - ai_prompt = f"""Du bist ein deutscher Assistent. Der Benutzer hat folgende Anfrage auf Deutsch gestellt: - -"{prompt.strip()}" - -Erstelle einen kurzen, zusammenfassenden Titel für diese Unterhaltung. Der Titel muss: -- Auf Deutsch sein (KEIN Englisch!) -- Maximal 50 Zeichen lang sein -- Das Hauptthema zusammenfassen -- Informativ sein - -Beispiele für gute deutsche Titel: -- "LED-Artikel Suche" -- "Lagerbestandsabfrage" -- "Produktinformationen" -- "Artikel-Suche" - -Antworte NUR mit dem deutschen Titel, ohne Anführungszeichen oder Erklärungen.""" - - # Create AI request - request = AiCallRequest( - prompt=ai_prompt, - context="", - options=AiCallOptions( - operationType=OperationTypeEnum.DATA_GENERATE, - priority=PriorityEnum.SPEED, - processingMode=ProcessingModeEnum.BASIC, - compressPrompt=False, - compressContext=False, - temperature=0.3 # Lower temperature for more consistent German output - ) - ) - - # Call AI service - logger.info(f"Calling AI to generate conversation name for prompt: {prompt[:50]}...") - response = await services.ai.callAi(request) - - if not response or not hasattr(response, 'content') or not response.content: - logger.warning("AI response invalid, generating name from prompt") - return generate_name_from_prompt(prompt) - - logger.info(f"AI response received: {response.content[:100]}...") - - # Clean up the AI response - name = str(response.content).strip() - name = name.strip('"\'') - - # Remove markdown code blocks if present - if name.startswith('```'): - lines = name.split('\n') - if len(lines) > 1: - name = '\n'.join(lines[1:-1]) if lines[-1].strip() == '```' else '\n'.join(lines[1:]) - - # Remove newlines and extra spaces - name = " ".join(name.split()) - - # Check if name contains English words - if so, generate from prompt instead - name_lower = name.lower() - english_words = ["search", "find", "show", "display", "query", "article", "product", "item", "led articles", "product search"] - if any(word in name_lower for word in english_words): - logger.warning(f"AI generated English name '{name}', generating from prompt instead") - return generate_name_from_prompt(prompt) - - # Limit to 50 characters - if len(name) > 50: - name = name[:47] + "..." - - # If we got a valid name, return it - if name and len(name) >= 3: - logger.info(f"Successfully generated conversation name via AI: '{name}'") - return name - else: - logger.warning(f"Generated name is too short: '{name}', generating from prompt") - return generate_name_from_prompt(prompt) - - except Exception as e: - logger.error(f"Error generating conversation name with AI: {e}", exc_info=True) - return generate_name_from_prompt(prompt) - - -def generate_name_from_prompt(prompt: str) -> str: - """ - Generate a conversation name directly from the German prompt. - Creates a concise title by extracting key words and formatting them. - - Args: - prompt: User's input prompt in German - - Returns: - A short conversation name in German - """ - if not prompt or not prompt.strip(): - return "Neue Unterhaltung" - - # Clean up the prompt - name = prompt.strip() - - # Remove newlines and extra spaces - name = " ".join(name.split()) - - # Remove common question words and phrases - question_words = ["wie", "was", "wo", "wann", "wer", "welche", "welcher", "welches"] - words = name.split() - filtered_words = [w for w in words if w.lower() not in question_words] - - if filtered_words: - name = " ".join(filtered_words) - - # Capitalize first letter - if name: - name = name[0].upper() + name[1:] if len(name) > 1 else name.upper() - - # Limit to 50 characters - if len(name) > 50: - # Try to cut at word boundary - truncated = name[:47] - last_space = truncated.rfind(' ') - if last_space > 20: # Only cut at word boundary if reasonable - name = truncated[:last_space] + "..." - else: - name = truncated + "..." - - # If name is empty or too short, use default - if not name or len(name) < 3: - return "Neue Unterhaltung" - - return name diff --git a/modules/features/chatbot/config.py b/modules/features/chatbot/config.py deleted file mode 100644 index d10ce57e..00000000 --- a/modules/features/chatbot/config.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Configuration system for chatbot instances. -Loads configuration from the database (FeatureInstance.config JSONB field). -""" - -import logging -from dataclasses import dataclass, field -from typing import Optional, Dict, Any, List, TYPE_CHECKING - -if TYPE_CHECKING: - from modules.datamodels.datamodelFeatures import FeatureInstance - -logger = logging.getLogger(__name__) - -# Cache for loaded configs (by instance ID for database configs) -_config_cache: Dict[str, 'ChatbotConfig'] = {} - -# Default system prompt when none is configured -DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant. You have access to SQL query tools and web search tools. Use them to help answer user questions." - - -@dataclass -class DatabaseConfig: - """Database configuration for a chatbot instance.""" - schema: Dict[str, Any] = field(default_factory=dict) - connector: str = "preprocessor" - - def is_sql_enabled(self) -> bool: - """Check if SQL queries are possible (has connector).""" - return bool(self.connector) - - -@dataclass -class ToolConfig: - """Tool configuration for a chatbot instance.""" - sql: Dict[str, Any] = field(default_factory=lambda: {"enabled": True}) - tavily: Optional[Dict[str, Any]] = None - streaming: Dict[str, Any] = field(default_factory=lambda: {"enabled": True}) - - def is_sql_enabled(self) -> bool: - """Check if SQL tool is enabled.""" - if self.sql is None: - return True # Default enabled - return self.sql.get("enabled", True) - - def is_tavily_enabled(self) -> bool: - """Check if Tavily web search tool is enabled.""" - if self.tavily is None: - return False # Default disabled - return self.tavily.get("enabled", False) - - def is_streaming_enabled(self) -> bool: - """Check if streaming status tool is enabled.""" - if self.streaming is None: - return True # Default enabled - return self.streaming.get("enabled", True) - - -@dataclass -class ModelConfig: - """Model configuration for a chatbot instance.""" - operationType: str = "DATA_ANALYSE" - processingMode: str = "BASIC" # Changed from DETAILED for faster responses - allowedProviders: List[str] = field(default_factory=list) # Empty = all providers allowed - - -@dataclass -class ChatbotConfig: - """Configuration for a chatbot instance.""" - id: str - name: str - systemPrompt: str - database: DatabaseConfig - tools: ToolConfig - model: ModelConfig - - @classmethod - def from_dict(cls, data: Dict[str, Any], config_id: str = "default") -> 'ChatbotConfig': - """ - Create ChatbotConfig from dictionary. - - Supports two config formats: - 1. New format (file-based): systemPrompt, database, tools, model - 2. Legacy frontend format: connector, prompts, behavior - - Args: - data: Configuration dictionary (from JSON file or FeatureInstance.config) - config_id: Identifier for this config (instance ID or file name) - - Returns: - ChatbotConfig instance with validated values - """ - # Detect config format and normalize - if "prompts" in data or "connector" in data or "behavior" in data: - # Legacy frontend format - convert to new format - data = cls._convert_legacy_config(data) - - # Get system prompt - required field, use default if not provided - system_prompt = data.get("systemPrompt") - if not system_prompt: - logger.warning(f"Config {config_id}: No systemPrompt provided, using default") - system_prompt = DEFAULT_SYSTEM_PROMPT - - # Parse database config - db_data = data.get("database", {}) - database_config = DatabaseConfig( - schema=db_data.get("schema", {}), - connector=db_data.get("connector", "preprocessor") - ) - - # Parse tools config with defaults - tools_data = data.get("tools", {}) - tools_config = ToolConfig( - sql=tools_data.get("sql", {"enabled": True}), - tavily=tools_data.get("tavily", {"enabled": False}), - streaming=tools_data.get("streaming", {"enabled": True}) - ) - - # Parse model config with defaults - model_data = data.get("model", {}) - allowed_providers = model_data.get("allowedProviders") or data.get("allowedProviders", []) - model_config = ModelConfig( - operationType=model_data.get("operationType", "DATA_ANALYSE"), - processingMode=model_data.get("processingMode", "DETAILED"), - allowedProviders=allowed_providers if isinstance(allowed_providers, list) else [] - ) - - return cls( - id=data.get("id", config_id), - name=data.get("name", "Chatbot"), - systemPrompt=system_prompt, - database=database_config, - tools=tools_config, - model=model_config - ) - - @staticmethod - def _convert_legacy_config(data: Dict[str, Any]) -> Dict[str, Any]: - """ - Convert legacy frontend config format to new format. - - Legacy format (from AdminFeatureAccessPage.tsx): - { - "connector": {"types": [...], "type": "preprocessor"}, - "prompts": {"customAnalysisPrompt": "...", "customFinalAnswerPrompt": "..."}, - "behavior": {"enableWebResearch": true, ...} - } - - New format: - { - "systemPrompt": "...", - "database": {"connector": "preprocessor"}, - "tools": {"sql": {"enabled": true}, "tavily": {"enabled": true}} - } - """ - converted = {} - - # Extract system prompt from prompts section - prompts = data.get("prompts", {}) - system_prompt = prompts.get("customAnalysisPrompt") or prompts.get("customFinalAnswerPrompt") - if system_prompt: - converted["systemPrompt"] = system_prompt - - # Extract connector from connector section - connector_data = data.get("connector", {}) - connector_type = connector_data.get("type") or "preprocessor" - if isinstance(connector_data.get("types"), list) and connector_data["types"]: - connector_type = connector_data["types"][0] # Use first connector as primary - - converted["database"] = { - "connector": connector_type, - "schema": {} - } - - # Extract tool settings from behavior section - behavior = data.get("behavior", {}) - enable_web_research = behavior.get("enableWebResearch", False) - - converted["tools"] = { - "sql": {"enabled": True}, # SQL always enabled if connector is set - "tavily": {"enabled": enable_web_research}, - "streaming": {"enabled": True} # Streaming always enabled - } - - # Model config defaults - use BASIC for faster responses - converted["model"] = { - "operationType": "DATA_ANALYSE", - "processingMode": "BASIC", - "allowedProviders": data.get("allowedProviders", []) - } - - # Copy other fields - if "id" in data: - converted["id"] = data["id"] - if "name" in data: - converted["name"] = data["name"] - - logger.debug(f"Converted legacy config format to new format") - return converted - - def to_dict(self) -> Dict[str, Any]: - """Convert config to dictionary for serialization.""" - return { - "id": self.id, - "name": self.name, - "systemPrompt": self.systemPrompt, - "database": { - "schema": self.database.schema, - "connector": self.database.connector - }, - "tools": { - "sql": self.tools.sql, - "tavily": self.tools.tavily, - "streaming": self.tools.streaming - }, - "model": { - "operationType": self.model.operationType, - "processingMode": self.model.processingMode, - "allowedProviders": self.model.allowedProviders - } - } - - -def load_chatbot_config_from_instance(instance: 'FeatureInstance') -> ChatbotConfig: - """ - Load chatbot configuration from a FeatureInstance's config field. - - This is the primary method for loading chatbot configuration. - The config is stored in the FeatureInstance.config JSONB field. - - Args: - instance: FeatureInstance object with config field - - Returns: - ChatbotConfig instance - - Raises: - ValueError: If instance has no config and no fallback available - """ - instance_id = instance.id - - # Check cache first (by instance ID) - cache_key = f"instance_{instance_id}" - if cache_key in _config_cache: - logger.debug(f"Returning cached config for instance {instance_id}") - return _config_cache[cache_key] - - # Get config from instance - config_data = instance.config - - if not config_data: - logger.warning(f"Instance {instance_id} has no config, using minimal defaults") - config_data = {} - - logger.debug(f"Instance {instance_id} raw config keys: {list(config_data.keys()) if config_data else []}, allowedProviders: {config_data.get('allowedProviders')}") - - # Create config from dictionary - config = ChatbotConfig.from_dict(config_data, config_id=instance_id) - logger.debug(f"Instance {instance_id} parsed config.model.allowedProviders: {config.model.allowedProviders}") - - # Cache the config - _config_cache[cache_key] = config - logger.info(f"Loaded chatbot config from instance {instance_id}: {config.name}") - - return config - - -def load_chatbot_config_from_dict(config_data: Dict[str, Any], config_id: str = "custom") -> ChatbotConfig: - """ - Load chatbot configuration from a dictionary. - - Useful for testing or when config data is already available. - - Args: - config_data: Configuration dictionary - config_id: Identifier for this config - - Returns: - ChatbotConfig instance - """ - return ChatbotConfig.from_dict(config_data, config_id=config_id) - - -def clear_config_cache(instance_id: Optional[str] = None): - """ - Clear the configuration cache. - - Args: - instance_id: Optional instance ID to clear specific cache entry. - If None, clears entire cache. - """ - global _config_cache - if instance_id: - cache_key = f"instance_{instance_id}" - if cache_key in _config_cache: - del _config_cache[cache_key] - logger.debug(f"Cleared chatbot config cache for instance {instance_id}") - else: - _config_cache.clear() - logger.debug("Cleared all chatbot config cache") diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py deleted file mode 100644 index 68d672a4..00000000 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ /dev/null @@ -1,1618 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Interface to Chatbot database and AI Connectors. -Uses the PostgreSQL connector for data access with user/feature-instance filtering. -Chatbot-specific models in poweron_chatbot (separate from workflow engine). -""" - -import logging -import uuid -import math -from typing import Dict, Any, List, Optional, Union -from enum import Enum -from pydantic import BaseModel, Field - -import asyncio - -from modules.security.rbac import RbacClass -from modules.datamodels.datamodelRbac import AccessRuleContext -from modules.datamodels.datamodelUam import AccessLevel - -from modules.datamodels.datamodelChat import UserInputRequest -from modules.datamodels.datamodelBase import PowerOnModel -from modules.shared.dbRegistry import registerDatabase -from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp - -chatbotDatabase = "poweron_chatbot" -registerDatabase(chatbotDatabase) - -# ============================================================================= -# Chatbot-specific Pydantic models for poweron_chatbot (per-instance isolation) -# ============================================================================= - - -class ChatbotDocument(PowerOnModel): - """Documents attached to chatbot messages.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") - messageId: str = Field(description="Foreign key to message") - fileId: str = Field(description="Foreign key to file") - fileName: str = Field(description="Name of the file") - fileSize: int = Field(description="Size of the file") - mimeType: str = Field(description="MIME type of the file") - roundNumber: Optional[int] = Field(None, description="Round number in workflow") - taskNumber: Optional[int] = Field(None, description="Task number within round") - actionNumber: Optional[int] = Field(None, description="Action number within task") - actionId: Optional[str] = Field(None, description="ID of the action that created this document") - - -class ChatbotMessage(PowerOnModel): - """Messages in chatbot conversations. Must match bridge format in memory.py.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") - conversationId: str = Field(description="Foreign key to conversation") - parentMessageId: Optional[str] = Field(None, description="Parent message ID for threading") - documents: List[ChatbotDocument] = Field(default_factory=list, description="Associated documents") - documentsLabel: Optional[str] = Field(None, description="Label for the set of documents") - message: Optional[str] = Field(None, description="Message content") - role: str = Field(description="Role of the message sender") - status: str = Field(description="Status of the message (first, step, last)") - sequenceNr: Optional[int] = Field(default=0, description="Sequence number of the message") - publishedAt: Optional[float] = Field(default=None, description="When the message was published (UTC timestamp)") - success: Optional[bool] = Field(None, description="Whether the message processing was successful") - actionId: Optional[str] = Field(None, description="ID of the action that produced this message") - actionMethod: Optional[str] = Field(None, description="Method of the action that produced this message") - actionName: Optional[str] = Field(None, description="Name of the action that produced this message") - roundNumber: Optional[int] = Field(None, description="Round number in workflow") - taskNumber: Optional[int] = Field(None, description="Task number within round") - actionNumber: Optional[int] = Field(None, description="Action number within task") - taskProgress: Optional[str] = Field(None, description="Task progress status") - actionProgress: Optional[str] = Field(None, description="Action progress status") - - -class ChatbotLog(PowerOnModel): - """Log entries for chatbot conversations.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") - conversationId: str = Field(description="Foreign key to conversation") - message: str = Field(description="Log message") - type: str = Field(description="Log type (info, warning, error, etc.)") - timestamp: float = Field(default_factory=getUtcTimestamp, description="When the log entry was created (UTC timestamp)") - status: Optional[str] = Field(None, description="Status of the log entry") - progress: Optional[float] = Field(None, description="Progress indicator (0.0 to 1.0)") - performance: Optional[Dict[str, Any]] = Field(None, description="Performance metrics") - parentId: Optional[str] = Field(None, description="Parent operation ID") - operationId: Optional[str] = Field(None, description="Operation ID to group related log entries") - roundNumber: Optional[int] = Field(None, description="Round number in workflow") - taskNumber: Optional[int] = Field(None, description="Task number within round") - actionNumber: Optional[int] = Field(None, description="Action number within task") - - -class ChatbotWorkflowModeEnum(str, Enum): - WORKFLOW_CHATBOT = "Chatbot" - - -class ChatbotConversation(PowerOnModel): - """Chatbot conversation container. Per feature-instance isolation via featureInstanceId.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") - featureInstanceId: str = Field(description="Feature instance ID for per-instance isolation") - name: Optional[str] = Field(None, description="Name of the conversation") - status: str = Field(default="running", description="Current status") - currentRound: int = Field(default=0, description="Current round number") - lastActivity: float = Field(default_factory=getUtcTimestamp, description="Timestamp of last activity") - startedAt: float = Field(default_factory=getUtcTimestamp, description="When the conversation started") - workflowMode: ChatbotWorkflowModeEnum = Field(default=ChatbotWorkflowModeEnum.WORKFLOW_CHATBOT, description="Workflow mode") - maxSteps: int = Field(default=10, description="Maximum number of iterations") - # Hydrated from child tables (not stored in DB) - logs: List[ChatbotLog] = Field(default_factory=list, description="Conversation logs") - messages: List[ChatbotMessage] = Field(default_factory=list, description="Conversation messages") - - -import json -from modules.datamodels.datamodelUam import User - -# DYNAMIC PART: Connectors to the Interface -from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC - -# Basic Configurations -from modules.shared.configuration import APP_CONFIG -logger = logging.getLogger(__name__) - -# Singleton factory for Chat instances -_chatInterfaces = {} - - -def storeDebugMessageAndDocuments(message, currentUser, mandateId=None, featureInstanceId=None) -> None: - """ - Store message and documents (metadata and file bytes) for debugging purposes. - Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ - - message.json, message_text.txt - - document_###_metadata.json - - document_###_ (actual file bytes) - - Args: - message: ChatbotMessage or ChatMessage-like object to store - currentUser: Current user for component interface access - mandateId: Mandate ID for RBAC context (avoids overwriting singleton state) - featureInstanceId: Feature instance ID for RBAC context - """ - try: - import os - from datetime import datetime, UTC - from modules.shared.debugLogger import getBaseDebugDir, ensureDir - from modules.interfaces.interfaceDbManagement import getInterface - - # Create base debug directory (use base debug dir, not prompts subdirectory) - baseDebugDir = getBaseDebugDir() - debug_root = os.path.join(baseDebugDir, 'messages') - ensureDir(debug_root) - - # Generate timestamp - timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] - - # Create message folder name: m_round_task_action_timestamp - # Use actual values from message, not defaults - round_str = str(message.roundNumber) if message.roundNumber is not None else "0" - task_str = str(message.taskNumber) if message.taskNumber is not None else "0" - action_str = str(message.actionNumber) if message.actionNumber is not None else "0" - message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}" - - message_path = os.path.join(debug_root, message_folder) - os.makedirs(message_path, exist_ok=True) - - # Store message data - use dict() instead of model_dump() for compatibility - message_file = os.path.join(message_path, "message.json") - with open(message_file, "w", encoding="utf-8") as f: - # Convert message to dict manually to avoid model_dump() issues - message_dict = { - "id": message.id, - "workflowId": getattr(message, "conversationId", None) or getattr(message, "workflowId", ""), - "parentMessageId": message.parentMessageId, - "message": message.message, - "role": message.role, - "status": message.status, - "sequenceNr": message.sequenceNr, - "publishedAt": message.publishedAt, - "roundNumber": message.roundNumber, - "taskNumber": message.taskNumber, - "actionNumber": message.actionNumber, - "documentsLabel": message.documentsLabel, - "actionId": message.actionId, - "actionMethod": message.actionMethod, - "actionName": message.actionName, - "success": message.success, - "documents": [] - } - json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str) - - # Store message content as text - if message.message: - message_text_file = os.path.join(message_path, "message_text.txt") - with open(message_text_file, "w", encoding="utf-8") as f: - f.write(str(message.message)) - - # Store documents if provided - if message.documents and len(message.documents) > 0: - # Group documents by documentsLabel - documents_by_label = {} - for doc in message.documents: - label = message.documentsLabel or 'default' - if label not in documents_by_label: - documents_by_label[label] = [] - documents_by_label[label].append(doc) - - # Create subfolder for each document label - for label, docs in documents_by_label.items(): - # Sanitize label for filesystem - safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip() - safe_label = safe_label.replace(' ', '_') - if not safe_label: - safe_label = "default" - - label_folder = os.path.join(message_path, safe_label) - ensureDir(label_folder) - - # Store each document - for i, doc in enumerate(docs): - # Create document metadata file - doc_meta = { - "id": doc.id, - "messageId": doc.messageId, - "fileId": doc.fileId, - "fileName": doc.fileName, - "fileSize": doc.fileSize, - "mimeType": doc.mimeType, - "roundNumber": doc.roundNumber, - "taskNumber": doc.taskNumber, - "actionNumber": doc.actionNumber, - "actionId": doc.actionId - } - - doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json") - with open(doc_meta_file, "w", encoding="utf-8") as f: - json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str) - - # Also store the actual file bytes next to metadata for debugging - try: - componentInterface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - file_bytes = componentInterface.getFileData(doc.fileId) - if file_bytes: - # Build a safe filename preserving original name - safe_name = doc.fileName or f"document_{i+1:03d}" - # Avoid path traversal - safe_name = os.path.basename(safe_name) - doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name) - with open(doc_file_path, "wb") as df: - df.write(file_bytes) - else: - pass - except Exception as e: - pass - - except Exception as e: - # Silent fail - don't break main flow - pass - -class ChatObjects: - """ - Interface to Chat database and AI Connectors. - Uses the JSON connector for data access with added language support. - """ - - def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - """Initializes the Chat Interface. - - Args: - currentUser: The authenticated user - mandateId: The mandate ID from RequestContext (X-Mandate-Id header) - featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) - """ - # Initialize variables - self.currentUser = currentUser # Store User object directly - self.userId = currentUser.id if currentUser else None - # Use mandateId from parameter (Request-Context), not from user object - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - self.featureCode = "chatbot" # For RBAC buildDataObjectKey - self.rbac = None # RBAC interface - - # Initialize services - self._initializeServices() - - # Initialize database - self._initializeDatabase() - - # Set user context if provided - if currentUser: - self.setUserContext(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - - # ===== Generic Utility Methods ===== - - def _isObjectField(self, fieldType) -> bool: - """Check if a field type represents a complex object (not a simple type).""" - # Simple scalar types - if fieldType in (str, int, float, bool, type(None)): - return False - - # Everything else is an object - return True - - def _separateObjectFields(self, model_class, data: Dict[str, Any]) -> tuple[Dict[str, Any], Dict[str, Any]]: - """Separate simple fields from object fields based on Pydantic model structure.""" - simpleFields = {} - objectFields = {} - - # Get field information from the Pydantic model - modelFields = model_class.model_fields - - for fieldName, value in data.items(): - # Check if this field should be stored as JSONB in the database - if fieldName in modelFields: - fieldInfo = modelFields[fieldName] - # Pydantic v2 only - fieldType = fieldInfo.annotation - - # Always route relational/object fields to object_fields for separate handling - # These fields are stored in separate normalized tables, not as JSONB - if fieldName in ['documents', 'stats', 'logs', 'messages']: - objectFields[fieldName] = value - continue - - # Check if this is a JSONB field (Dict, List, or complex types) - # Purely type-based detection - no hardcoded field names - if (fieldType == dict or - fieldType == list or - (hasattr(fieldType, '__origin__') and fieldType.__origin__ in (dict, list))): - # Store as JSONB - include in simple_fields for database storage - simpleFields[fieldName] = value - elif isinstance(value, (str, int, float, bool, type(None))): - # Simple scalar types - simpleFields[fieldName] = value - else: - # Complex objects that should be filtered out - objectFields[fieldName] = value - else: - # Field not in model - treat as scalar if simple, otherwise filter out - # Underscore-prefixed keys (e.g. UI meta) pass through; sys* live on PowerOnModel subclasses - if fieldName.startswith("_"): - simpleFields[fieldName] = value - elif isinstance(value, (str, int, float, bool, type(None))): - simpleFields[fieldName] = value - else: - objectFields[fieldName] = value - - return simpleFields, objectFields - - def _initializeServices(self): - pass - - def setUserContext(self, currentUser: User, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - """Sets the user context for the interface. - - Args: - currentUser: The authenticated user - mandateId: The mandate ID from RequestContext (X-Mandate-Id header) - featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header) - """ - self.currentUser = currentUser # Store User object directly - self.userId = currentUser.id - # Use mandateId from parameter (Request-Context), not from user object - self.mandateId = mandateId - self.featureInstanceId = featureInstanceId - - if not self.userId: - raise ValueError("Invalid user context: id is required") - - # Note: mandateId is ALWAYS optional here - it comes from Request-Context, not from User. - # Users are NOT assigned to mandates by design - they get mandate context from the request. - # sysAdmin users can additionally perform cross-mandate operations. - # Without mandateId, operations will be filtered to accessible mandates via RBAC. - - # Add language settings - self.userLanguage = currentUser.language # Default user language - - # Initialize RBAC interface - if not self.currentUser: - raise ValueError("User context is required for RBAC") - # Get DbApp connection for RBAC AccessRule queries - from modules.security.rootAccess import getRootDbAppConnector - dbApp = getRootDbAppConnector() - self.rbac = RbacClass(self.db, dbApp=dbApp) - - # Update database context - self.db.updateContext(self.userId) - - def __del__(self): - """Cleanup method to close database connection.""" - if hasattr(self, 'db') and self.db is not None: - try: - self.db.close() - except Exception as e: - logger.error(f"Error closing database connection: {e}") - - - def _initializeDatabase(self): - """Initializes the database connection directly.""" - try: - # Get configuration values with defaults - dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data") - dbDatabase = chatbotDatabase - dbUser = APP_CONFIG.get("DB_USER") - dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) - - from modules.connectors.connectorDbPostgre import getCachedConnector - self.db = getCachedConnector( - dbHost=dbHost, - dbDatabase=dbDatabase, - dbUser=dbUser, - dbPassword=dbPassword, - dbPort=dbPort, - userId=self.userId - ) - - logger.info("Database initialized successfully") - except Exception as e: - logger.error(f"Failed to initialize database: {str(e)}") - raise - - def _initRecords(self): - """Initializes standard records in the database if they don't exist.""" - pass - - - def checkRbacPermission( - self, - modelClass: type, - operation: str, - recordId: Optional[str] = None - ) -> bool: - """ - Check RBAC permission for a specific operation on a table. - - Args: - modelClass: Pydantic model class for the table - operation: Operation to check ('create', 'update', 'delete', 'read') - recordId: Optional record ID for specific record check - - Returns: - Boolean indicating permission - """ - if not self.rbac or not self.currentUser: - return False - - tableName = modelClass.__name__ - from modules.interfaces.interfaceRbac import buildDataObjectKey - objectKey = buildDataObjectKey(tableName, featureCode=getattr(self, 'featureCode', 'chatbot')) - permissions = self.rbac.getUserPermissions( - self.currentUser, - AccessRuleContext.DATA, - objectKey, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId - ) - - if operation == "create": - return permissions.create != AccessLevel.NONE - elif operation == "update": - return permissions.update != AccessLevel.NONE - elif operation == "delete": - return permissions.delete != AccessLevel.NONE - elif operation == "read": - return permissions.read != AccessLevel.NONE - else: - return False - - def _applyFilters(self, records: List[Dict[str, Any]], filters: Dict[str, Any]) -> List[Dict[str, Any]]: - """ - Apply filter criteria to records. - - Supports: - - General search: {"search": "text"} - searches across all text fields - - Field-specific filters: - - Simple: {"status": "running"} - equals match - - With operator: {"status": {"operator": "equals", "value": "running"}} - - Operators: equals, contains, gt, gte, lt, lte, in, notIn, startsWith, endsWith - - Args: - records: List of record dictionaries to filter - filters: Filter criteria dictionary - - Returns: - Filtered list of records - """ - if not filters or not records: - return records - - filtered = [] - - for record in records: - matches = True - - # Handle general search across text fields - if "search" in filters: - search_term = str(filters["search"]).lower() - if search_term: - # Search in all string fields - found = False - for key, value in record.items(): - if isinstance(value, str) and search_term in value.lower(): - found = True - break - elif isinstance(value, (int, float)) and search_term in str(value): - found = True - break - if not found: - matches = False - - # Handle field-specific filters - for field_name, filter_value in filters.items(): - if field_name == "search": - continue # Already handled above - - if field_name not in record: - matches = False - break - - record_value = record.get(field_name) - - # Handle simple value (equals operator) - if not isinstance(filter_value, dict): - if str(record_value).lower() != str(filter_value).lower(): - matches = False - break - continue - - # Handle filter with operator - operator = filter_value.get("operator", "equals") - filter_val = filter_value.get("value") - - if operator in ["equals", "eq"]: - if str(record_value).lower() != str(filter_val).lower(): - matches = False - break - - elif operator == "contains": - record_str = str(record_value).lower() if record_value is not None else "" - filter_str = str(filter_val).lower() if filter_val is not None else "" - if filter_str not in record_str: - matches = False - break - - elif operator == "startsWith": - record_str = str(record_value).lower() if record_value is not None else "" - filter_str = str(filter_val).lower() if filter_val is not None else "" - if not record_str.startswith(filter_str): - matches = False - break - - elif operator == "endsWith": - record_str = str(record_value).lower() if record_value is not None else "" - filter_str = str(filter_val).lower() if filter_val is not None else "" - if not record_str.endswith(filter_str): - matches = False - break - - elif operator == "gt": - try: - record_num = float(record_value) if record_value is not None else float('-inf') - filter_num = float(filter_val) if filter_val is not None else float('-inf') - if record_num <= filter_num: - matches = False - break - except (ValueError, TypeError): - matches = False - break - - elif operator == "gte": - try: - record_num = float(record_value) if record_value is not None else float('-inf') - filter_num = float(filter_val) if filter_val is not None else float('-inf') - if record_num < filter_num: - matches = False - break - except (ValueError, TypeError): - matches = False - break - - elif operator == "lt": - try: - record_num = float(record_value) if record_value is not None else float('inf') - filter_num = float(filter_val) if filter_val is not None else float('inf') - if record_num >= filter_num: - matches = False - break - except (ValueError, TypeError): - matches = False - break - - elif operator == "lte": - try: - record_num = float(record_value) if record_value is not None else float('inf') - filter_num = float(filter_val) if filter_val is not None else float('inf') - if record_num > filter_num: - matches = False - break - except (ValueError, TypeError): - matches = False - break - - elif operator == "in": - if not isinstance(filter_val, list): - filter_val = [filter_val] - if record_value not in filter_val: - matches = False - break - - elif operator == "notIn": - if not isinstance(filter_val, list): - filter_val = [filter_val] - if record_value in filter_val: - matches = False - break - - else: - # Unknown operator - default to equals - if str(record_value).lower() != str(filter_val).lower(): - matches = False - break - - if matches: - filtered.append(record) - - return filtered - - def _applySorting(self, records: List[Dict[str, Any]], sortFields: List[Any]) -> List[Dict[str, Any]]: - """Apply multi-level sorting to records using stable sort (sorts from least to most significant field).""" - if not sortFields: - return records - - # Start with a copy to avoid modifying original - sortedRecords = list(records) - - # Sort from least significant to most significant field (reverse order) - # Python's sort is stable, so this creates proper multi-level sorting - for sortField in reversed(sortFields): - # Handle both dict and object formats - if isinstance(sortField, dict): - fieldName = sortField.get("field") - direction = sortField.get("direction", "asc") - else: - fieldName = getattr(sortField, "field", None) - direction = getattr(sortField, "direction", "asc") - - if not fieldName: - continue - - isDesc = (direction == "desc") - - def sortKey(record): - value = record.get(fieldName) - # Handle None values - place them at the end for both directions - if value is None: - # Use a special value that sorts last - return (1, "") # (is_none_flag, empty_value) - sorts after (0, ...) - else: - # Return tuple with type indicator for proper comparison - if isinstance(value, (int, float)): - return (0, value) - elif isinstance(value, str): - return (0, value) - elif isinstance(value, bool): - return (0, value) - else: - return (0, str(value)) - - # Sort with reverse parameter for descending - sortedRecords.sort(key=sortKey, reverse=isDesc) - - return sortedRecords - - # Utilities - - def getInitialId(self, model_class: type) -> Optional[str]: - """Returns the initial ID for a table.""" - return self.db.getInitialId(model_class) - - - - # Conversation methods (chatbot-specific, poweron_chatbot) - - def getConversations(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: - """ - Returns conversations for current feature instance based on user access. - Supports optional pagination, sorting, and filtering. - - Args: - pagination: Optional pagination parameters. If None, returns all items. - - Returns: - If pagination is None: List[Dict[str, Any]] - If pagination is provided: PaginatedResult with items and metadata - """ - # Use RBAC filtering with featureInstanceId for instance-level isolation - filteredConversations = getRecordsetWithRBAC(self.db, - ChatbotConversation, - self.currentUser, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot" - ) - - # If no pagination requested, return all items (no sorting - frontend handles it) - if pagination is None: - return filteredConversations - - # Apply filtering (if filters provided) - if pagination.filters: - filteredConversations = self._applyFilters(filteredConversations, pagination.filters) - - # Apply sorting (in order of sortFields) - only if provided by frontend - if pagination.sort: - filteredConversations = self._applySorting(filteredConversations, pagination.sort) - - # Count total items after filters - totalItems = len(filteredConversations) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedConversations = filteredConversations[startIdx:endIdx] - - return PaginatedResult( - items=pagedConversations, - totalItems=totalItems, - totalPages=totalPages - ) - - def getWorkflows(self, pagination: Optional[PaginationParams] = None) -> Union[List[Dict[str, Any]], PaginatedResult]: - """Backward-compat alias for getConversations.""" - return self.getConversations(pagination) - - def getConversation(self, conversationId: str) -> Optional[ChatbotConversation]: - """Returns a conversation by ID if user has access.""" - # Use RBAC filtering with featureInstanceId for instance-level isolation - conversations = getRecordsetWithRBAC(self.db, - ChatbotConversation, - self.currentUser, - recordFilter={"id": conversationId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot" - ) - - if not conversations: - return None - - conv = conversations[0] - try: - # Load related data from normalized tables - logs = self.getLogs(conversationId) - messages = self.getMessages(conversationId) - - # Build ChatbotConversation with hydrated logs/messages - return ChatbotConversation( - id=conv["id"], - featureInstanceId=conv.get("featureInstanceId") or self.featureInstanceId or "", - name=conv.get("name"), - status=conv.get("status", "running"), - currentRound=conv.get("currentRound", 0) or 0, - lastActivity=conv.get("lastActivity", getUtcTimestamp()), - startedAt=conv.get("startedAt", getUtcTimestamp()), - workflowMode=ChatbotWorkflowModeEnum(conv.get("workflowMode", "Chatbot")), - maxSteps=conv.get("maxSteps") if conv.get("maxSteps") is not None else 10, - logs=logs, - messages=messages - ) - except Exception as e: - logger.error(f"Error validating conversation data: {str(e)}") - return None - - def getWorkflow(self, workflowId: str) -> Optional[ChatbotConversation]: - """Backward-compat alias: workflowId maps to conversationId.""" - return self.getConversation(workflowId) - - def getWorkflowMinimal(self, workflowId: str) -> Optional[ChatbotConversation]: - """Lightweight fetch: conversation record only, no logs/messages. For resume path.""" - conversations = getRecordsetWithRBAC( - self.db, - ChatbotConversation, - self.currentUser, - recordFilter={"id": workflowId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot", - ) - if not conversations: - return None - conv = conversations[0] - max_steps = conv.get("maxSteps") - return ChatbotConversation( - id=conv["id"], - featureInstanceId=conv.get("featureInstanceId") or self.featureInstanceId or "", - name=conv.get("name"), - status=conv.get("status", "running"), - currentRound=conv.get("currentRound", 0) or 0, - lastActivity=conv.get("lastActivity", getUtcTimestamp()), - startedAt=conv.get("startedAt", getUtcTimestamp()), - workflowMode=ChatbotWorkflowModeEnum(conv.get("workflowMode", "Chatbot")), - maxSteps=max_steps if max_steps is not None else 10, - logs=[], - messages=[], - ) - - def getMessageCount(self, conversationId: str) -> int: - """Returns message count for a conversation (single query, no document fetch).""" - messages = getRecordsetWithRBAC( - self.db, - ChatbotMessage, - self.currentUser, - recordFilter={"conversationId": conversationId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot", - ) - return len(messages) if messages else 0 - - def updateWorkflowMinimal( - self, workflowId: str, workflowData: Dict[str, Any] - ) -> ChatbotConversation: - """Lightweight update: no logs/messages fetch. For resume when caller has minimal workflow.""" - if not self.checkRbacPermission(ChatbotConversation, "update", workflowId): - raise PermissionError(f"No permission to update conversation {workflowId}") - simpleFields, _ = self._separateObjectFields(ChatbotConversation, workflowData) - simpleFields["lastActivity"] = getUtcTimestamp() - updated = self.db.recordModify(ChatbotConversation, workflowId, simpleFields) - max_steps = updated.get("maxSteps") - return ChatbotConversation( - id=updated["id"], - featureInstanceId=updated.get("featureInstanceId") or self.featureInstanceId or "", - name=updated.get("name"), - status=updated.get("status", "running"), - currentRound=updated.get("currentRound", 0) or 0, - lastActivity=updated.get("lastActivity", getUtcTimestamp()), - startedAt=updated.get("startedAt", getUtcTimestamp()), - workflowMode=ChatbotWorkflowModeEnum(updated.get("workflowMode", "Chatbot")), - maxSteps=max_steps if max_steps is not None else 10, - logs=[], - messages=[], - ) - - def createConversation(self, conversationData: Dict[str, Any]) -> ChatbotConversation: - """Creates a new conversation if user has permission.""" - if not self.checkRbacPermission(ChatbotConversation, "create"): - raise PermissionError("No permission to create conversations") - - # Set timestamp if not present - currentTime = getUtcTimestamp() - if "startedAt" not in conversationData: - conversationData["startedAt"] = currentTime - if "lastActivity" not in conversationData: - conversationData["lastActivity"] = currentTime - - # Set featureInstanceId from context (no mandateId in DB) - if "featureInstanceId" not in conversationData or not conversationData["featureInstanceId"]: - conversationData["featureInstanceId"] = self.featureInstanceId or "" - if not conversationData.get("featureInstanceId"): - conversationData["featureInstanceId"] = self.featureInstanceId or "" - - # Use generic field separation - logs/messages go to objectFields, not stored - simpleFields, objectFields = self._separateObjectFields(ChatbotConversation, conversationData) - - # Create conversation in database - created = self.db.recordCreate(ChatbotConversation, simpleFields) - - return ChatbotConversation( - id=created["id"], - featureInstanceId=created.get("featureInstanceId") or self.featureInstanceId or "", - name=created.get("name"), - status=created.get("status", "running"), - currentRound=created.get("currentRound", 0) or 0, - lastActivity=created.get("lastActivity", currentTime), - startedAt=created.get("startedAt", currentTime), - workflowMode=ChatbotWorkflowModeEnum(created.get("workflowMode", "Chatbot")), - maxSteps=created.get("maxSteps", 10), - logs=[], - messages=[] - ) - - def createWorkflow(self, workflowData: Dict[str, Any]) -> ChatbotConversation: - """Backward-compat alias: maps workflowData to conversationData.""" - return self.createConversation(workflowData) - - def updateConversation(self, conversationId: str, conversationData: Dict[str, Any]) -> ChatbotConversation: - """Updates a conversation if user has access.""" - conv = self.getConversation(conversationId) - if not conv: - return None - - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - raise PermissionError(f"No permission to update conversation {conversationId}") - - simpleFields, objectFields = self._separateObjectFields(ChatbotConversation, conversationData) - simpleFields["lastActivity"] = getUtcTimestamp() - - updated = self.db.recordModify(ChatbotConversation, conversationId, simpleFields) - - # Reuse logs/messages from conv — update only touches simple fields, not related data - return ChatbotConversation( - id=updated["id"], - featureInstanceId=updated.get("featureInstanceId") or conv.featureInstanceId or self.featureInstanceId or "", - name=updated.get("name", conv.name), - status=updated.get("status", conv.status), - currentRound=updated.get("currentRound", conv.currentRound), - lastActivity=updated.get("lastActivity", conv.lastActivity), - startedAt=updated.get("startedAt", conv.startedAt), - workflowMode=ChatbotWorkflowModeEnum(updated.get("workflowMode", conv.workflowMode.value)), - maxSteps=updated.get("maxSteps") if updated.get("maxSteps") is not None else conv.maxSteps, - logs=conv.logs, - messages=conv.messages - ) - - def updateWorkflow(self, workflowId: str, workflowData: Dict[str, Any]) -> ChatbotConversation: - """Backward-compat alias.""" - return self.updateConversation(workflowId, workflowData) - - def deleteConversation(self, conversationId: str) -> bool: - """Deletes a conversation and all related data if user has access.""" - try: - conv = self.getConversation(conversationId) - if not conv: - return False - - if not self.checkRbacPermission(ChatbotConversation, "delete", conversationId): - raise PermissionError(f"No permission to delete conversation {conversationId}") - - # CASCADE DELETE: Delete all related data first - - # 1. Delete all messages and their documents - messages = self.getMessages(conversationId) - for message in messages: - messageId = message.id - if messageId: - existing_docs = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") - for doc in existing_docs: - self.db.recordDelete(ChatbotDocument, doc["id"]) - self.db.recordDelete(ChatbotMessage, messageId) - - # 2. Delete conversation logs - existing_logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, featureCode="chatbot") - for log in existing_logs: - self.db.recordDelete(ChatbotLog, log["id"]) - - # 3. Delete the conversation - success = self.db.recordDelete(ChatbotConversation, conversationId) - return success - - except Exception as e: - logger.error(f"Error deleting conversation {conversationId}: {str(e)}") - return False - - def deleteWorkflow(self, workflowId: str) -> bool: - """Backward-compat alias.""" - return self.deleteConversation(workflowId) - - - # Message methods - - def getMessages(self, conversationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatbotMessage], PaginatedResult]: - """ - Returns messages for a conversation if user has access. - Supports optional pagination, sorting, and filtering. - - Args: - conversationId: The conversation ID (workflowId for backward compat) - pagination: Optional pagination parameters. If None, returns all items. - - Returns: - If pagination is None: List[ChatbotMessage] - If pagination is provided: PaginatedResult with items and metadata - """ - # Check conversation access first - conversations = getRecordsetWithRBAC(self.db, - ChatbotConversation, - self.currentUser, - recordFilter={"id": conversationId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot" - ) - - if not conversations: - if pagination is None: - return [] - return PaginatedResult(items=[], totalItems=0, totalPages=0) - - # Get messages for this conversation - messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - - # Convert raw messages to dict format for sorting/filtering - messageDicts = [] - for msg in messages: - messageDicts.append({ - "id": msg.get("id"), - "conversationId": msg.get("conversationId"), - "parentMessageId": msg.get("parentMessageId"), - "documentsLabel": msg.get("documentsLabel"), - "message": msg.get("message"), - "role": msg.get("role", "assistant"), - "status": msg.get("status", "step"), - "sequenceNr": msg.get("sequenceNr", 0), - "publishedAt": msg.get("publishedAt", msg.get("timestamp", getUtcTimestamp())), - "success": msg.get("success"), - "actionId": msg.get("actionId"), - "actionMethod": msg.get("actionMethod"), - "actionName": msg.get("actionName"), - "roundNumber": msg.get("roundNumber"), - "taskNumber": msg.get("taskNumber"), - "actionNumber": msg.get("actionNumber"), - "taskProgress": msg.get("taskProgress"), - "actionProgress": msg.get("actionProgress") - }) - - # Apply default sorting by publishedAt if no sort specified - if pagination is None or not pagination.sort: - messageDicts.sort(key=lambda x: x.get("publishedAt", getUtcTimestamp())) - - # Apply filtering (if filters provided) - if pagination and pagination.filters: - messageDicts = self._applyFilters(messageDicts, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination and pagination.sort: - messageDicts = self._applySorting(messageDicts, pagination.sort) - - # If no pagination requested, return all items (batch-fetch documents to avoid N+1) - if pagination is None: - msg_ids = [m["id"] for m in messageDicts] - docs_by_message = self.getDocumentsForMessages(msg_ids) if msg_ids else {} - chat_messages = [] - for msg in messageDicts: - documents = docs_by_message.get(msg["id"], []) - chat_message = ChatbotMessage( - id=msg["id"], - conversationId=msg["conversationId"], - parentMessageId=msg.get("parentMessageId"), - documents=documents, - documentsLabel=msg.get("documentsLabel"), - message=msg.get("message"), - role=msg.get("role", "assistant"), - status=msg.get("status", "step"), - sequenceNr=msg.get("sequenceNr", 0), - publishedAt=msg.get("publishedAt", getUtcTimestamp()), - success=msg.get("success"), - actionId=msg.get("actionId"), - actionMethod=msg.get("actionMethod"), - actionName=msg.get("actionName"), - roundNumber=msg.get("roundNumber"), - taskNumber=msg.get("taskNumber"), - actionNumber=msg.get("actionNumber"), - taskProgress=msg.get("taskProgress"), - actionProgress=msg.get("actionProgress") - ) - - chat_messages.append(chat_message) - - return chat_messages - - # Count total items after filters - totalItems = len(messageDicts) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedMessageDicts = messageDicts[startIdx:endIdx] - paged_msg_ids = [m["id"] for m in pagedMessageDicts] - docs_by_message = self.getDocumentsForMessages(paged_msg_ids) if paged_msg_ids else {} - chat_messages = [] - for msg in pagedMessageDicts: - documents = docs_by_message.get(msg["id"], []) - chat_message = ChatbotMessage( - id=msg["id"], - conversationId=msg["conversationId"], - parentMessageId=msg.get("parentMessageId"), - documents=documents, - documentsLabel=msg.get("documentsLabel"), - message=msg.get("message"), - role=msg.get("role", "assistant"), - status=msg.get("status", "step"), - sequenceNr=msg.get("sequenceNr", 0), - publishedAt=msg.get("publishedAt", getUtcTimestamp()), - success=msg.get("success"), - actionId=msg.get("actionId"), - actionMethod=msg.get("actionMethod"), - actionName=msg.get("actionName"), - roundNumber=msg.get("roundNumber"), - taskNumber=msg.get("taskNumber"), - actionNumber=msg.get("actionNumber"), - taskProgress=msg.get("taskProgress"), - actionProgress=msg.get("actionProgress") - ) - chat_messages.append(chat_message) - - return PaginatedResult( - items=chat_messages, - totalItems=totalItems, - totalPages=totalPages - ) - - def createMessage(self, messageData: Dict[str, Any], event_manager=None) -> ChatbotMessage: - """Creates a message for a conversation if user has access. Accepts workflowId (from bridge) or conversationId.""" - try: - if "id" not in messageData or not messageData["id"]: - messageData["id"] = f"msg_{uuid.uuid4()}" - # Map workflowId to conversationId (bridge compatibility) - if "workflowId" in messageData and "conversationId" not in messageData: - messageData["conversationId"] = messageData["workflowId"] - requiredFields = ["id", "conversationId"] - for field in requiredFields: - if field not in messageData: - logger.error(f"Required field '{field}' missing in messageData") - raise ValueError(f"Required field '{field}' missing in message data") - - conversationId = messageData["conversationId"] - conv = self.getConversation(conversationId) - if not conv: - raise PermissionError(f"No access to conversation {conversationId}") - - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - raise PermissionError(f"No permission to modify conversation {conversationId}") - - # Validate that ID is not None - if messageData["id"] is None: - messageData["id"] = f"msg_{uuid.uuid4()}" - logger.warning(f"Automatically generated ID for workflow message: {messageData['id']}") - - # Set status if not present - if "status" not in messageData: - messageData["status"] = "step" # Default status for intermediate messages - - # Ensure role and agentName are present - if "role" not in messageData: - messageData["role"] = "assistant" if messageData.get("agentName") else "user" - - if "agentName" not in messageData: - messageData["agentName"] = "" - - # Set roundNumber, taskNumber, actionNumber if not provided - if "roundNumber" not in messageData: - messageData["roundNumber"] = conv.currentRound - if "taskNumber" not in messageData: - messageData["taskNumber"] = 0 - if "actionNumber" not in messageData: - messageData["actionNumber"] = 0 - - # Use generic field separation - no mandateId/featureInstanceId in message - simpleFields, objectFields = self._separateObjectFields(ChatbotMessage, messageData) - - # Handle documents separately - they will be stored in normalized documents table - documents_to_create = objectFields.get("documents", []) - - createdMessage = self.db.recordCreate(ChatbotMessage, simpleFields) - - created_documents = [] - logger.debug(f"Creating {len(documents_to_create)} document(s) for message {createdMessage['id']}") - for idx, doc_data in enumerate(documents_to_create): - try: - if isinstance(doc_data, ChatbotDocument): - doc_dict = doc_data.model_dump() - elif isinstance(doc_data, dict): - doc_dict = dict(doc_data) - else: - try: - doc_dict = ChatbotDocument(**doc_data).model_dump() - except Exception as e: - logger.error(f"Invalid document data type for message creation (document {idx + 1}/{len(documents_to_create)}): {e}") - continue - - # Ensure messageId is set - doc_dict["messageId"] = createdMessage["id"] - logger.debug(f"Creating document {idx + 1}/{len(documents_to_create)}: fileName={doc_dict.get('fileName', 'unknown')}, fileId={doc_dict.get('fileId', 'unknown')}, messageId={doc_dict.get('messageId', 'unknown')}") - - created_doc = self.createDocument(doc_dict) - if created_doc: - created_documents.append(created_doc) - logger.debug(f"Successfully created document {idx + 1}/{len(documents_to_create)}: {created_doc.fileName} (id: {created_doc.id})") - else: - logger.error(f"Failed to create document {idx + 1}/{len(documents_to_create)}: createDocument returned None for fileName={doc_dict.get('fileName', 'unknown')}") - except Exception as e: - logger.error(f"Error processing document {idx + 1}/{len(documents_to_create)}: {e}", exc_info=True) - - logger.info(f"Created {len(created_documents)}/{len(documents_to_create)} document(s) for message {createdMessage['id']}") - - chat_message = ChatbotMessage( - id=createdMessage["id"], - conversationId=createdMessage["conversationId"], - parentMessageId=createdMessage.get("parentMessageId"), - documents=created_documents, - documentsLabel=createdMessage.get("documentsLabel"), - message=createdMessage.get("message"), - role=createdMessage.get("role", "assistant"), - status=createdMessage.get("status", "step"), - sequenceNr=len(conv.messages) + 1, - publishedAt=createdMessage.get("publishedAt", getUtcTimestamp()), - roundNumber=createdMessage.get("roundNumber"), - taskNumber=createdMessage.get("taskNumber"), - actionNumber=createdMessage.get("actionNumber"), - success=createdMessage.get("success"), - actionId=createdMessage.get("actionId"), - actionMethod=createdMessage.get("actionMethod"), - actionName=createdMessage.get("actionName") - ) - - if event_manager: - try: - message_timestamp = parseTimestamp(chat_message.publishedAt, default=getUtcTimestamp()) - asyncio.create_task(event_manager.emit_event( - context_id=conversationId, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": chat_message.model_dump() - }, - event_category="chat" - )) - except Exception as e: - logger.debug(f"Could not emit message event: {e}") - - # Debug: Store message and documents for debugging - only if debug enabled - storeDebugMessageAndDocuments(chat_message, self.currentUser, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId) - - return chat_message - - except Exception as e: - logger.error(f"Error creating workflow message: {str(e)}") - return None - - def updateMessage(self, messageId: str, messageData: Dict[str, Any]) -> Optional[ChatbotMessage]: - """Updates a conversation message if user has access.""" - try: - if not messageId: - logger.error("No messageId provided for updateMessage") - raise ValueError("messageId cannot be empty") - - if "workflowId" in messageData and "conversationId" not in messageData: - messageData["conversationId"] = messageData["workflowId"] - - messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"id": messageId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - if not messages: - logger.warning(f"Message with ID {messageId} does not exist in database") - if "conversationId" in messageData: - conv = self.getConversation(messageData["conversationId"]) - if not conv: - raise PermissionError(f"No access to conversation {messageData['conversationId']}") - if not self.checkRbacPermission(ChatbotConversation, "update", messageData["conversationId"]): - raise PermissionError(f"No permission to modify conversation") - logger.info(f"Creating new message with ID {messageId} for conversation {messageData['conversationId']}") - created = self.db.recordCreate(ChatbotMessage, messageData) - return ChatbotMessage(**created) if created else None - logger.error("Conversation ID missing for new message") - return None - - existingMessage = messages[0] - conversationId = existingMessage.get("conversationId") - conv = self.getConversation(conversationId) - if not conv: - raise PermissionError(f"No access to conversation {conversationId}") - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - raise PermissionError(f"No permission to modify conversation {conversationId}") - - simpleFields, objectFields = self._separateObjectFields(ChatbotMessage, messageData) - - if "role" not in simpleFields and "role" not in existingMessage: - simpleFields["role"] = "assistant" - if 'id' not in simpleFields: - simpleFields['id'] = messageId - - if "createdAt" in simpleFields and "startedAt" not in simpleFields: - simpleFields["startedAt"] = simpleFields["createdAt"] - del simpleFields["createdAt"] - - updatedMessage = self.db.recordModify(ChatbotMessage, messageId, simpleFields) - - if 'documents' in objectFields: - for doc_data in objectFields['documents']: - try: - if isinstance(doc_data, ChatbotDocument): - doc_dict = doc_data.model_dump() - elif isinstance(doc_data, dict): - doc_dict = dict(doc_data) - else: - doc_dict = ChatbotDocument(**doc_data).model_dump() - doc_dict["messageId"] = messageId - self.createDocument(doc_dict) - except Exception as e: - logger.error(f"Error updating message documents: {e}") - if not updatedMessage: - logger.warning(f"Failed to update message {messageId}") - return None - - return ChatbotMessage(**updatedMessage) - except Exception as e: - logger.error(f"Error updating message {messageId}: {str(e)}", exc_info=True) - raise ValueError(f"Error updating message {messageId}: {str(e)}") - - def deleteMessage(self, conversationId: str, messageId: str) -> bool: - """Deletes a conversation message and related data if user has access.""" - try: - conv = self.getConversation(conversationId) - if not conv: - logger.warning(f"No access to conversation {conversationId}") - return False - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - raise PermissionError(f"No permission to modify conversation {conversationId}") - - messages = self.getMessages(conversationId) - message = next((m for m in messages if m.id == messageId), None) - if not message: - logger.warning(f"Message {messageId} for conversation {conversationId} not found") - return False - - existing_docs = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") - for doc in existing_docs: - self.db.recordDelete(ChatbotDocument, doc["id"]) - success = self.db.recordDelete(ChatbotMessage, messageId) - - return success - - except Exception as e: - logger.error(f"Error deleting message {messageId}: {str(e)}") - return False - - def deleteFileFromMessage(self, conversationId: str, messageId: str, fileId: str) -> bool: - """Removes a file reference from a message if user has access.""" - try: - conv = self.getConversation(conversationId) - if not conv: - logger.warning(f"No access to conversation {conversationId}") - return False - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - raise PermissionError(f"No permission to modify conversation {conversationId}") - - documents = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, featureCode="chatbot") - - if not documents: - logger.warning(f"No documents found for message {messageId}") - return False - - # Find and delete the specific document - removed = False - for doc in documents: - docId = doc.get("id") - fileIdValue = doc.get("fileId") - - # Flexible matching approach - shouldRemove = ( - (docId == fileId) or - (fileIdValue == fileId) or - (isinstance(docId, str) and str(fileId) in docId) or - (isinstance(fileIdValue, str) and str(fileId) in fileIdValue) - ) - - if shouldRemove: - # Delete the document from normalized table - success = self.db.recordDelete(ChatbotDocument, docId) - if success: - removed = True - else: - logger.warning(f"Failed to delete document {docId}") - - if not removed: - logger.warning(f"No matching file {fileId} found in message {messageId}") - return False - - except Exception as e: - logger.error(f"Error removing file {fileId} from message {messageId}: {str(e)}") - return False - - # Document methods - - def getDocuments(self, messageId: str) -> List[ChatbotDocument]: - """Returns documents for a message from normalized table.""" - try: - documents = getRecordsetWithRBAC(self.db, ChatbotDocument, self.currentUser, recordFilter={"messageId": messageId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - return [ChatbotDocument(**doc) for doc in documents] - except Exception as e: - logger.error(f"Error getting message documents: {str(e)}") - return [] - - def getDocumentsForMessages(self, messageIds: List[str]) -> Dict[str, List[ChatbotDocument]]: - """Returns documents for multiple messages in one query. Returns {messageId: [doc, ...]}.""" - if not messageIds: - return {} - try: - documents = getRecordsetWithRBAC( - self.db, - ChatbotDocument, - self.currentUser, - recordFilter={"messageId": messageIds}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot", - ) - result: Dict[str, List[ChatbotDocument]] = {mid: [] for mid in messageIds} - for doc in documents: - mid = doc.get("messageId") - if mid in result: - result[mid].append(ChatbotDocument(**doc)) - return result - except Exception as e: - logger.error(f"Error getting documents for messages: {e}") - return {mid: [] for mid in messageIds} - - def createDocument(self, documentData: Dict[str, Any]) -> ChatbotDocument: - """Creates a document for a message in normalized table.""" - try: - document = ChatbotDocument(**documentData) - logger.debug(f"Creating document: fileName={document.fileName}, fileId={document.fileId}, messageId={document.messageId}") - created = self.db.recordCreate(ChatbotDocument, document.model_dump()) - if created: - return ChatbotDocument(**created) - logger.error(f"Failed to create document for fileName={document.fileName}") - return None - except Exception as e: - logger.error(f"Error creating message document: {str(e)}", exc_info=True) - return None - - - # Log methods - - def getLogs(self, conversationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[ChatbotLog], PaginatedResult]: - """ - Returns logs for a conversation if user has access. - Supports optional pagination, sorting, and filtering. - """ - conversations = getRecordsetWithRBAC(self.db, - ChatbotConversation, - self.currentUser, - recordFilter={"id": conversationId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot" - ) - if not conversations: - if pagination is None: - return [] - return PaginatedResult(items=[], totalItems=0, totalPages=0) - - logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - - logDicts = [] - for log in logs: - logDicts.append({ - "id": log.get("id"), - "conversationId": log.get("conversationId"), - "message": log.get("message"), - "type": log.get("type"), - "timestamp": log.get("timestamp", getUtcTimestamp()), - "status": log.get("status"), - "progress": log.get("progress") - }) - - # Apply default sorting by timestamp if no sort specified - if pagination is None or not pagination.sort: - logDicts.sort(key=lambda x: parseTimestamp(x.get("timestamp"), default=0)) - - # Apply filtering (if filters provided) - if pagination and pagination.filters: - logDicts = self._applyFilters(logDicts, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination and pagination.sort: - logDicts = self._applySorting(logDicts, pagination.sort) - - if pagination is None: - return [ChatbotLog(**log) for log in logDicts] - - totalItems = len(logDicts) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedLogDicts = logDicts[startIdx:endIdx] - items = [ChatbotLog(**log) for log in pagedLogDicts] - - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) - - def createLog(self, logData: Dict[str, Any], event_manager=None) -> Optional[ChatbotLog]: - """Creates a log entry for a conversation if user has access. Accepts workflowId for backward compat.""" - conversationId = logData.get("conversationId") or logData.get("workflowId") - if not conversationId: - logger.error("No conversationId/workflowId provided for createLog") - return None - - conv = self.getConversation(conversationId) - if not conv: - logger.warning(f"No access to conversation {conversationId}") - return None - if not self.checkRbacPermission(ChatbotConversation, "update", conversationId): - logger.warning(f"No permission to modify conversation {conversationId}") - return None - - if "timestamp" not in logData: - logData["timestamp"] = getUtcTimestamp() - logData["conversationId"] = conversationId - if "status" not in logData and "type" in logData: - logData["status"] = "error" if logData["type"] == "error" else "running" - if "progress" not in logData: - logData["progress"] = 1.0 if logData.get("type") == "error" else 0.5 - - try: - log_model = ChatbotLog(**logData) - except Exception as e: - logger.error(f"Invalid log data: {e}") - return None - - createdLog = self.db.recordCreate(ChatbotLog, log_model.model_dump()) - if not createdLog: - return None - - if event_manager: - try: - log_timestamp = parseTimestamp(createdLog.get("timestamp"), default=getUtcTimestamp()) - asyncio.create_task(event_manager.emit_event( - context_id=conversationId, - event_type="chatdata", - data={"type": "log", "createdAt": log_timestamp, "item": ChatbotLog(**createdLog).model_dump()}, - event_category="log", - message="New log" - )) - except Exception as e: - logger.debug(f"Could not emit log event: {e}") - - return ChatbotLog(**createdLog) - - def getUnifiedChatData(self, conversationId: str, afterTimestamp: Optional[float] = None) -> Dict[str, Any]: - """ - Returns unified chat data (messages, logs) for a conversation in chronological order. - Uses timestamp-based selective data transfer for efficient polling. - """ - conversations = getRecordsetWithRBAC(self.db, - ChatbotConversation, - self.currentUser, - recordFilter={"id": conversationId}, - mandateId=self.mandateId, - featureInstanceId=self.featureInstanceId, - featureCode="chatbot" - ) - if not conversations: - return {"items": []} - - items = [] - messages = getRecordsetWithRBAC(self.db, ChatbotMessage, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - # Batch-fetch documents for all messages (avoids N+1) - message_ids = [m["id"] for m in messages if afterTimestamp is None or parseTimestamp(m.get("publishedAt"), default=getUtcTimestamp()) > afterTimestamp] - docs_by_message = self.getDocumentsForMessages(message_ids) if message_ids else {} - for msg in messages: - msgTimestamp = parseTimestamp(msg.get("publishedAt"), default=getUtcTimestamp()) - if afterTimestamp is not None and msgTimestamp <= afterTimestamp: - continue - documents = docs_by_message.get(msg["id"], []) - chatMessage = ChatbotMessage( - id=msg["id"], - conversationId=msg["conversationId"], - parentMessageId=msg.get("parentMessageId"), - documents=documents, - documentsLabel=msg.get("documentsLabel"), - message=msg.get("message"), - role=msg.get("role", "assistant"), - status=msg.get("status", "step"), - sequenceNr=msg.get("sequenceNr", 0), - publishedAt=msg.get("publishedAt", getUtcTimestamp()), - success=msg.get("success"), - actionId=msg.get("actionId"), - actionMethod=msg.get("actionMethod"), - actionName=msg.get("actionName"), - roundNumber=msg.get("roundNumber"), - taskNumber=msg.get("taskNumber"), - actionNumber=msg.get("actionNumber"), - taskProgress=msg.get("taskProgress"), - actionProgress=msg.get("actionProgress") - ) - items.append({"type": "message", "createdAt": msgTimestamp, "item": chatMessage}) - - logs = getRecordsetWithRBAC(self.db, ChatbotLog, self.currentUser, recordFilter={"conversationId": conversationId}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode="chatbot") - for log in logs: - logTimestamp = parseTimestamp(log.get("timestamp"), default=getUtcTimestamp()) - if afterTimestamp is not None and logTimestamp <= afterTimestamp: - continue - chatLog = ChatbotLog(**log) - items.append({"type": "log", "createdAt": logTimestamp, "item": chatLog}) - - items.sort(key=lambda x: parseTimestamp(x.get("createdAt"), default=0)) - return {"items": items} - - -def getInterface(currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None) -> 'ChatObjects': - """ - Returns a ChatObjects instance for the current user. - Handles initialization of database and records. - - Args: - currentUser: The authenticated user - mandateId: The mandate ID from RequestContext (X-Mandate-Id header). Required. - featureInstanceId: The feature instance ID from RequestContext (X-Feature-Instance-Id header). - """ - if not currentUser: - raise ValueError("Invalid user context: user is required") - - effectiveMandateId = str(mandateId) if mandateId else None - effectiveFeatureInstanceId = str(featureInstanceId) if featureInstanceId else None - - # Create context key including featureInstanceId for proper isolation - contextKey = f"{effectiveMandateId}_{effectiveFeatureInstanceId}_{currentUser.id}" - - # Create new instance if not exists - if contextKey not in _chatInterfaces: - _chatInterfaces[contextKey] = ChatObjects(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) - else: - # Update user context if needed - _chatInterfaces[contextKey].setUserContext(currentUser, mandateId=effectiveMandateId, featureInstanceId=effectiveFeatureInstanceId) - - return _chatInterfaces[contextKey] diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py deleted file mode 100644 index 7b3b6e0c..00000000 --- a/modules/features/chatbot/mainChatbot.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Chatbot Feature Container - Main Module. -Handles feature initialization and RBAC catalog registration. -""" - -import logging -from typing import Dict, List, Any, Optional - -from modules.shared.i18nRegistry import t - -logger = logging.getLogger(__name__) - -# Feature metadata -FEATURE_CODE = "chatbot" -FEATURE_LABEL = t("Chatbot", context="UI") -FEATURE_ICON = "mdi-robot" - -# UI Objects for RBAC catalog -UI_OBJECTS = [ - { - "objectKey": "ui.feature.chatbot.conversations", - "label": t("Konversationen", context="UI"), - "meta": {"area": "conversations"} - } -] - -# Resource Objects for RBAC catalog -RESOURCE_OBJECTS = [ - { - "objectKey": "resource.feature.chatbot.startStream", - "label": t("Chat starten (Stream)", context="UI"), - "meta": {"endpoint": "/api/chatbot/{instanceId}/start/stream", "method": "POST"} - }, - { - "objectKey": "resource.feature.chatbot.stop", - "label": t("Chat stoppen", context="UI"), - "meta": {"endpoint": "/api/chatbot/{instanceId}/stop/{workflowId}", "method": "POST"} - }, - { - "objectKey": "resource.feature.chatbot.threads", - "label": t("Threads abrufen", context="UI"), - "meta": {"endpoint": "/api/chatbot/{instanceId}/threads", "method": "GET"} - }, - { - "objectKey": "resource.feature.chatbot.delete", - "label": t("Chat löschen", context="UI"), - "meta": {"endpoint": "/api/chatbot/{instanceId}/{workflowId}", "method": "DELETE"} - }, -] - -# Service requirements - services this feature needs from the service center -# Format: [{serviceKey, meta}]. Used by getChatbotServices() to resolve only needed services. -REQUIRED_SERVICES = [ - { - "serviceKey": "chat", - "meta": {"usage": "File info, document handling"} - }, - { - "serviceKey": "ai", - "meta": {"usage": "AI calls, conversation name generation"} - }, - { - "serviceKey": "billing", - "meta": {"usage": "Usage tracking, balance checks"} - }, - { - "serviceKey": "streaming", - "meta": {"usage": "Event manager, ChatStreamingHelper"} - }, -] - -# Template roles for this feature -# Role names MUST follow convention: {featureCode}-{roleName} -TEMPLATE_ROLES = [ - { - "roleLabel": "chatbot-viewer", - "description": "Chatbot Betrachter - Chat-Threads ansehen (nur lesen)", - "accessRules": [ - # UI: only threads view, NO active chat - {"context": "UI", "item": "ui.feature.chatbot.threads", "view": True}, - # RESOURCE: can list threads only - {"context": "RESOURCE", "item": "resource.feature.chatbot.threads", "view": True}, - # DATA access (own records, read-only) - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "n", "update": "n", "delete": "n"}, - ] - }, - { - "roleLabel": "chatbot-user", - "description": "Chatbot Benutzer - Chatbot nutzen und eigene Threads verwalten", - "accessRules": [ - # UI: full access to all views - {"context": "UI", "item": "ui.feature.chatbot.conversations", "view": True}, - {"context": "UI", "item": "ui.feature.chatbot.threads", "view": True}, - # Resource access: can start/stop chats, view threads, delete own - {"context": "RESOURCE", "item": "resource.feature.chatbot.startStream", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.chatbot.stop", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.chatbot.threads", "view": True}, - {"context": "RESOURCE", "item": "resource.feature.chatbot.delete", "view": True}, - # DATA access (own records) - {"context": "DATA", "item": None, "view": True, "read": "m", "create": "m", "update": "m", "delete": "m"}, - ] - }, - { - "roleLabel": "chatbot-admin", - "description": "Chatbot Admin - Vollzugriff auf alle Chatbot-Funktionen", - "accessRules": [ - # Full UI access - {"context": "UI", "item": None, "view": True}, - # Full resource access - {"context": "RESOURCE", "item": None, "view": True}, - # Full DATA access - {"context": "DATA", "item": None, "view": True, "read": "a", "create": "a", "update": "a", "delete": "a"}, - ] - }, -] - - -def getFeatureDefinition() -> Dict[str, Any]: - """Return the feature definition for registration. - - The chatbot feature is currently soft-disabled via ``enabled=False``: its - catalog objects, template roles and routes stay loaded so already-running - instances keep working, but it is filtered out of the Store and the - Admin Feature-Instances "Neue Instanz" selection list. - """ - return { - "code": FEATURE_CODE, - "label": FEATURE_LABEL, - "icon": FEATURE_ICON, - "enabled": False, - } - - -def getRequiredServiceKeys() -> List[str]: - """Return list of service keys this feature requires.""" - return [s["serviceKey"] for s in REQUIRED_SERVICES] - - -def getChatbotServices( - user, - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - workflow=None, -) -> Any: - """ - Get a service hub for the chatbot feature using the service center. - Resolves only the services declared in REQUIRED_SERVICES. - - Returns a hub-like object with: chat, ai, billing, streaming, - plus interfaceDbComponent, user, mandateId, featureInstanceId. - """ - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface - - # Provide workflow or placeholder so billing/etc get featureCode - _workflow = workflow - if _workflow is None: - _workflow = type("_Placeholder", (), {"featureCode": FEATURE_CODE})() - ctx = ServiceCenterContext( - user=user, - mandate_id=mandateId, - feature_instance_id=featureInstanceId, - workflow=_workflow, - ) - - hub = _ChatbotServiceHub() - hub.user = user - hub.mandateId = mandateId - hub.featureInstanceId = featureInstanceId - hub.workflow = workflow - hub.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) - - for spec in REQUIRED_SERVICES: - key = spec["serviceKey"] - try: - svc = getService(key, ctx) - setattr(hub, key, svc) - except Exception as e: - logger.warning(f"Could not resolve service '{key}' for chatbot: {e}") - setattr(hub, key, None) - - return hub - - -def getChatStreamingHelper(): - """ - Get ChatStreamingHelper utility class (used by chatbot for message normalization). - Resolves via service center streaming service. - """ - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - # Minimal context - streaming service only needs it for resolver - ctx = ServiceCenterContext(user=__get_placeholder_user(), mandate_id=None, feature_instance_id=None) - streaming = getService("streaming", ctx) - return streaming.getChatStreamingHelper() if streaming else None - - -def __get_placeholder_user(): - """Placeholder user for contexts that only need service resolution (e.g. ChatStreamingHelper).""" - from modules.interfaces.interfaceDbApp import getRootInterface - return getRootInterface().currentUser - - -def getEventManager(user, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): - """ - Get the global event manager for SSE streaming (used by chatbot routes). - """ - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - - ctx = ServiceCenterContext( - user=user, - mandate_id=mandateId, - feature_instance_id=featureInstanceId, - ) - streaming = getService("streaming", ctx) - return streaming.getEventManager() - - -class _ChatbotServiceHub: - """Lightweight hub exposing only services required by the chatbot feature.""" - user = None - mandateId = None - featureInstanceId = None - workflow = None - interfaceDbComponent = None - chat = None - ai = None - billing = None - streaming = None - featureCode = "chatbot" - allowedProviders = None - - -def getUiObjects() -> List[Dict[str, Any]]: - """Return UI objects for RBAC catalog registration.""" - return UI_OBJECTS - - -def getResourceObjects() -> List[Dict[str, Any]]: - """Return resource objects for RBAC catalog registration.""" - return RESOURCE_OBJECTS - - -def getTemplateRoles() -> List[Dict[str, Any]]: - """Return template roles for this feature.""" - return TEMPLATE_ROLES - - -def registerFeature(catalogService) -> bool: - """ - Register this feature's RBAC objects in the catalog. - - Args: - catalogService: The RBAC catalog service instance - - Returns: - True if registration was successful - """ - try: - # Register UI objects - for uiObj in UI_OBJECTS: - catalogService.registerUiObject( - featureCode=FEATURE_CODE, - objectKey=uiObj["objectKey"], - label=uiObj["label"], - meta=uiObj.get("meta") - ) - - # Register Resource objects - for resObj in RESOURCE_OBJECTS: - catalogService.registerResourceObject( - featureCode=FEATURE_CODE, - objectKey=resObj["objectKey"], - label=resObj["label"], - meta=resObj.get("meta") - ) - - # Sync template roles to database - _syncTemplateRolesToDb() - - logger.info(f"Feature '{FEATURE_CODE}' registered {len(UI_OBJECTS)} UI objects and {len(RESOURCE_OBJECTS)} resource objects") - return True - - except Exception as e: - logger.error(f"Failed to register feature '{FEATURE_CODE}': {e}") - return False - - -def getChatbotServices( - user, - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - workflow=None, -) -> "_ChatbotServiceHub": - """ - Get lightweight service hub for chatbot (chat, ai, streaming) without loading - the full legacy Services hub. Avoids ~90 ms from _loadFeatureInterfaces + - _loadFeatureServices; only instantiates required services. - Uses interfaceFeatureChatbot (ChatObjects) for interfaceDbChat to avoid - duplicate DB init - chatProcess reuses hub.interfaceDbChat. - """ - from modules.serviceHub import PublicService - from modules.interfaces.interfaceDbApp import getInterface as getAppInterface - from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface - from modules.serviceCenter.services.serviceChat.mainServiceChat import ChatService - from modules.serviceCenter.services.serviceAi.mainServiceAi import AiService - from modules.serviceCenter.core.serviceStreaming.mainServiceStreaming import StreamingService - - hub = _ChatbotServiceHub() - hub.user = user - hub.mandateId = mandateId - hub.featureInstanceId = featureInstanceId - hub.workflow = workflow - hub.featureCode = "chatbot" - hub.allowedProviders = None - - hub.interfaceDbApp = getAppInterface(user, mandateId=mandateId) - # interfaceDbComponent: lazy-loaded on first access (saves ~100–300 ms when no file uploads) - hub._interfaceDbComponent_val = None - # Use ChatObjects (interfaceFeatureChatbot) - same as chatProcess, avoids extra interfaceDbChat init - hub.interfaceDbChat = getChatbotInterface( - user, mandateId=mandateId, featureInstanceId=featureInstanceId - ) - - hub.chat = PublicService(ChatService(hub)) - hub.ai = PublicService(AiService(hub), functionsOnly=False) - hub.streaming = PublicService(StreamingService(hub)) - - # Resolve billing from service center (required for _preflight_billing_check and billing callback) - try: - from modules.serviceCenter import getService - from modules.serviceCenter.context import ServiceCenterContext - _workflow = workflow or type("_Placeholder", (), {"featureCode": FEATURE_CODE})() - ctx = ServiceCenterContext( - user=user, - mandate_id=mandateId, - feature_instance_id=featureInstanceId, - workflow=_workflow, - ) - hub.billing = getService("billing", ctx) - except Exception as e: - logger.warning(f"Could not resolve billing service for chatbot: {e}") - hub.billing = None - - return hub - - -class _ChatbotServiceHub: - """Lightweight hub with chat, ai, streaming for chatbot; avoids full Services init.""" - - user = None - mandateId = None - featureInstanceId = None - workflow = None - interfaceDbApp = None - _interfaceDbComponent_val = None - interfaceDbChat = None - - @property - def interfaceDbComponent(self): - """Lazy-load interfaceDbComponent on first access (saves ~100–300 ms when no files).""" - if self._interfaceDbComponent_val is None: - from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface - self._interfaceDbComponent_val = getComponentInterface( - self.user, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId - ) - return self._interfaceDbComponent_val - chat = None - ai = None - billing = None - streaming = None - featureCode = "chatbot" - allowedProviders = None - - -def _syncTemplateRolesToDb() -> int: - """ - Sync template roles and their AccessRules to the database. - Creates global template roles (mandateId=None) if they don't exist. - - Returns: - Number of roles created/updated - """ - try: - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext - from modules.datamodels.datamodelUtils import coerce_text_multilingual - - rootInterface = getRootInterface() - - # Get existing template roles for this feature (Pydantic models) - existingRoles = rootInterface.getRolesByFeatureCode(FEATURE_CODE) - # Filter to template roles (mandateId is None) - templateRoles = [r for r in existingRoles if r.mandateId is None] - existingRoleLabels = {r.roleLabel: str(r.id) for r in templateRoles} - - createdCount = 0 - for roleTemplate in TEMPLATE_ROLES: - roleLabel = roleTemplate["roleLabel"] - - if roleLabel in existingRoleLabels: - roleId = existingRoleLabels[roleLabel] - # Ensure AccessRules exist for this role - _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) - else: - # Create new template role - newRole = Role( - roleLabel=roleLabel, - description=coerce_text_multilingual(roleTemplate.get("description", {})), - featureCode=FEATURE_CODE, - mandateId=None, # Global template - featureInstanceId=None, - isSystemRole=False - ) - createdRole = rootInterface.db.recordCreate(Role, newRole.model_dump()) - roleId = createdRole.get("id") - - # Create AccessRules for this role - _ensureAccessRulesForRole(rootInterface, roleId, roleTemplate.get("accessRules", [])) - - logger.info(f"Created template role '{roleLabel}' with ID {roleId}") - createdCount += 1 - - if createdCount > 0: - logger.info(f"Feature '{FEATURE_CODE}': Created {createdCount} template roles") - - return createdCount - - except Exception as e: - logger.error(f"Error syncing template roles for feature '{FEATURE_CODE}': {e}") - return 0 - - -def _ensureAccessRulesForRole(rootInterface, roleId: str, ruleTemplates: List[Dict[str, Any]]) -> int: - """ - Ensure AccessRules exist for a role based on templates. - - Args: - rootInterface: Root interface instance - roleId: Role ID - ruleTemplates: List of rule templates - - Returns: - Number of rules created - """ - from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext - - # Get existing rules for this role (Pydantic models) - existingRules = rootInterface.getAccessRulesByRole(roleId) - - # Create a set of existing rule signatures to avoid duplicates - # IMPORTANT: Use .value for enum comparison, not str() which gives "AccessRuleContext.DATA" in Python 3.11+ - existingSignatures = set() - for rule in existingRules: - sig = (rule.context.value if rule.context else None, rule.item) - existingSignatures.add(sig) - - createdCount = 0 - for template in ruleTemplates: - context = template.get("context", "UI") - item = template.get("item") - sig = (context, item) - - if sig in existingSignatures: - continue - - # Map context string to enum - if context == "UI": - contextEnum = AccessRuleContext.UI - elif context == "DATA": - contextEnum = AccessRuleContext.DATA - elif context == "RESOURCE": - contextEnum = AccessRuleContext.RESOURCE - else: - contextEnum = context - - newRule = AccessRule( - roleId=roleId, - context=contextEnum, - item=item, - view=template.get("view", False), - read=template.get("read"), - create=template.get("create"), - update=template.get("update"), - delete=template.get("delete"), - ) - rootInterface.db.recordCreate(AccessRule, newRule.model_dump()) - createdCount += 1 - - if createdCount > 0: - logger.debug(f"Created {createdCount} AccessRules for role {roleId}") - - return createdCount diff --git a/modules/features/chatbot/routeFeatureChatbot.py b/modules/features/chatbot/routeFeatureChatbot.py deleted file mode 100644 index 6b79287b..00000000 --- a/modules/features/chatbot/routeFeatureChatbot.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Chatbot routes for the backend API. -Implements chatbot endpoints using LangGraph-based conversation workflows. -""" - -import logging -import json -import asyncio -import math -import uuid -from typing import Optional, Any, Dict, Union -from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Request, status -from fastapi.responses import StreamingResponse -from modules.shared.timeUtils import parseTimestamp, getUtcTimestamp - -# Import auth modules -from modules.auth import limiter, getRequestContext, RequestContext - -# Import interfaces -from . import interfaceFeatureChatbot as interfaceDbChat -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC -from modules.interfaces.interfaceDbApp import getRootInterface -from modules.interfaces.interfaceFeatures import getFeatureInterface - -# Import models -from modules.datamodels.datamodelChat import UserInputRequest -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata -from modules.features.chatbot.interfaceFeatureChatbot import ChatbotConversation - -# Import chatbot feature -from modules.features.chatbot import chatProcess -from modules.features.chatbot.mainChatbot import getEventManager -from modules.shared.i18nRegistry import apiRouteContext -routeApiMsg = apiRouteContext("routeFeatureChatbot") - -# Configure logger -logger = logging.getLogger(__name__) - -# Create router for chatbot endpoints -router = APIRouter( - prefix="/api/chatbot", - tags=["Chatbot"], - responses={404: {"description": "Not found"}} -) - -def _getServiceChat(context: RequestContext, instanceId: Optional[str] = None, mandateId: Optional[str] = None): - """Get chatbot interface with instance context. - Pass mandateId when available (e.g. from _validateInstanceAccess) to ensure cache hit with getChatbotServices. - """ - effective_mandate = mandateId if mandateId is not None else (str(context.mandateId) if context.mandateId else None) - return interfaceDbChat.getInterface( - context.user, - mandateId=effective_mandate, - featureInstanceId=instanceId - ) - - -def _validateInstanceAccess(instanceId: str, context: RequestContext) -> str: - """ - Validate that the user has access to the feature instance. - Returns the mandateId for the instance. - - Args: - instanceId: The FeatureInstance ID from URL - context: The request context with user info - - Returns: - mandateId of the instance - - Raises: - HTTPException 404 if instance not found - HTTPException 403 if user doesn't have access - """ - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - - instance = featureInterface.getFeatureInstance(instanceId) - if not instance: - raise HTTPException( - status_code=404, - detail=f"Feature instance '{instanceId}' not found" - ) - - # Verify it's a chatbot instance - if instance.featureCode != "chatbot": - raise HTTPException( - status_code=400, - detail=f"Instance '{instanceId}' is not a chatbot instance" - ) - - # Verify user has access to this instance - if not context.isPlatformAdmin: - # Check if user has FeatureAccess for this instance - featureAccesses = rootInterface.getFeatureAccessesForUser(str(context.user.id)) - hasAccess = any( - str(fa.featureInstanceId) == instanceId and fa.enabled - for fa in featureAccesses - ) - if not hasAccess: - raise HTTPException( - status_code=403, - detail=f"Access denied to feature instance '{instanceId}'" - ) - - return str(instance.mandateId) - - -# ============================================================================= -# List threads - MUST be first to avoid /{instanceId}/{workflowId} matching -# GET /api/chatbot/{instanceId}/threads before DELETE /api/chatbot/{instanceId}/{workflowId} -# ============================================================================= -@router.get("/{instanceId}/threads") -@limiter.limit("120/minute") -def get_chatbot_threads( - request: Request, - instanceId: str = Path(..., description="Feature Instance ID"), - workflowId: Optional[str] = Query(None, description="Optional workflow ID to get details and chat data for a specific thread"), - pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object (only used when workflowId is not provided)"), - context: RequestContext = Depends(getRequestContext) -) -> Union[PaginatedResponse[ChatbotConversation], Dict[str, Any]]: - """ - List all chatbot workflows (threads) for the current user, or get details and chat data for a specific thread. - - - If workflowId is provided: Returns the workflow details and all chat data (messages, logs, stats) - - If workflowId is not provided: Returns a paginated list of all workflows - """ - mandateId = _validateInstanceAccess(instanceId, context) - - try: - interfaceDbChat = _getServiceChat(context, instanceId, mandateId=mandateId) - - if workflowId: - workflow = interfaceDbChat.getWorkflow(workflowId) - if not workflow: - raise HTTPException( - status_code=404, - detail=f"Workflow with ID {workflowId} not found" - ) - - if hasattr(workflow, 'model_dump'): - workflow_dict = workflow.model_dump() - elif isinstance(workflow, dict): - workflow_dict = dict(workflow) - else: - workflow_dict = workflow - - if workflow_dict.get("maxSteps") is None: - workflow_dict["maxSteps"] = 10 - - chatData = interfaceDbChat.getUnifiedChatData(workflowId, None) - - return { - "workflow": workflow_dict, - "chatData": chatData - } - - paginationParams = None - if pagination: - try: - paginationDict = json.loads(pagination) - paginationParams = PaginationParams(**paginationDict) if paginationDict else None - except (json.JSONDecodeError, ValueError) as e: - raise HTTPException( - status_code=400, - detail=f"Invalid pagination parameter: {str(e)}" - ) - - all_workflows = interfaceDbChat.getWorkflows(pagination=None) - chatbot_workflows_data = [ - wf for wf in all_workflows - if (wf.get("workflowMode") or getattr(wf, "workflowMode", None)) == "Chatbot" - ] - - if paginationParams: - if paginationParams.sort: - chatbot_workflows_data = interfaceDbChat._applySorting(chatbot_workflows_data, paginationParams.sort) - totalItems = len(chatbot_workflows_data) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - workflows = chatbot_workflows_data[startIdx:endIdx] - else: - workflows = chatbot_workflows_data - totalItems = len(chatbot_workflows_data) - totalPages = 1 - - normalized_workflows = [] - for wf in workflows: - normalized_wf = dict(wf) - if normalized_wf.get("maxSteps") is None: - normalized_wf["maxSteps"] = 10 - normalized_workflows.append(normalized_wf) - - from modules.routes.routeHelpers import enrichRowsWithFkLabels - enriched = enrichRowsWithFkLabels(normalized_workflows, ChatbotConversation) - - return { - "items": enriched, - "pagination": PaginationMetadata( - currentPage=paginationParams.page if paginationParams else 1, - pageSize=paginationParams.pageSize if paginationParams else len(workflows), - totalItems=totalItems, - totalPages=totalPages, - sort=paginationParams.sort if paginationParams else [], - filters=paginationParams.filters if paginationParams else None - ).model_dump(), - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error getting chatbot threads: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=f"Error getting chatbot threads: {str(e)}" - ) - - -# Chatbot streaming endpoint (SSE) -@router.post("/{instanceId}/start/stream") -@limiter.limit("120/minute") -async def stream_chatbot_start( - request: Request, - instanceId: str = Path(..., description="Feature Instance ID"), - workflowId: Optional[str] = Query(None, description="Optional ID of the workflow to continue (can also be in request body)"), - userInput: UserInputRequest = Body(...), - context: RequestContext = Depends(getRequestContext) -) -> StreamingResponse: - """ - Starts a new chatbot workflow or continues an existing one with SSE streaming. - Streams progress updates in real-time via Server-Sent Events. - - workflowId can be provided either: - - As a query parameter: /api/chatbot/{instanceId}/start/stream?workflowId=xxx - - In the request body as part of UserInputRequest - - Query parameter takes precedence if both are provided - """ - # Validate instance access - mandateId = _validateInstanceAccess(instanceId, context) - - event_manager = getEventManager(context.user, mandateId=mandateId, featureInstanceId=instanceId) - - try: - # Use workflowId from query parameter if provided, otherwise from request body - final_workflow_id = workflowId or userInput.workflowId - - # Start background processing (this will create the workflow and event queue) - # Pass featureInstanceId and event_manager to chatProcess - workflow = await chatProcess( - context.user, mandateId, userInput, final_workflow_id, - featureInstanceId=instanceId, event_manager=event_manager - ) - - # Check if workflow was created successfully - if not workflow: - raise HTTPException( - status_code=500, - detail=routeApiMsg("Failed to create or load workflow") - ) - - # Get event queue for the workflow - queue = event_manager.get_queue(workflow.id) - if not queue: - # Create queue if it doesn't exist - queue = event_manager.create_queue(workflow.id) - - async def event_stream(): - """Async generator for SSE events - pure event-driven streaming (no polling).""" - try: - # Yield keepalive immediately so client gets 200 + first byte fast (normal chatbot feel) - yield ": keepalive\n\n" - - # Use same mandateId as chatProcess so we hit interface cache (avoid duplicate DB init) - interfaceDbChat = _getServiceChat(context, instanceId, mandateId=mandateId) - - # Use workflow from chatProcess (no refetch) - current_round = workflow.currentRound if workflow else None - is_resuming = final_workflow_id is not None and current_round and current_round > 1 - - # Send initial chat data (exact format as chatData endpoint) - only once at start - try: - chatData = interfaceDbChat.getUnifiedChatData(workflow.id, None) - if chatData.get("items"): - # Filter items by round number if resuming - filtered_items = [] - for item in chatData["items"]: - if is_resuming and current_round: - # Get round number from item - item_round = None - item_data = item.get("item") - if item_data: - # Handle both dict and object access - if isinstance(item_data, dict): - item_round = item_data.get("roundNumber") - elif hasattr(item_data, "roundNumber"): - item_round = item_data.roundNumber - - # When resuming, only include items from current round onwards - # Exclude items without roundNumber (they're from old rounds before roundNumber was added) - # Exclude items with roundNumber < current_round (from previous rounds) - if item_round is None or item_round < current_round: - continue # Skip items from previous rounds or without round info - - filtered_items.append(item) - - # Emit filtered items - for item in filtered_items: - _inner = item.get("item") - serializable_item = { - "type": item.get("type"), - "createdAt": item.get("createdAt"), - "item": _inner.model_dump() if _inner is not None and hasattr(_inner, "model_dump") else _inner, - } - # Emit item directly in exact chatData format: {type, createdAt, item} - yield f"data: {json.dumps(serializable_item)}\n\n" - except Exception as e: - logger.warning(f"Error fetching initial chat data: {e}") - - # Keepalive interval (30 seconds) - keepalive_interval = 30.0 - last_keepalive = asyncio.get_event_loop().time() - - # Status check interval (check workflow status every 5 seconds - less frequent since we're event-driven) - status_check_interval = 5.0 - last_status_check = asyncio.get_event_loop().time() - - # Stream events until completion or timeout - pure event-driven (no polling) - timeout = 300.0 # 5 minutes max - start_time = asyncio.get_event_loop().time() - - while True: - # Check timeout - elapsed = asyncio.get_event_loop().time() - start_time - if elapsed > timeout: - logger.info(f"Stream timeout for workflow {workflow.id}") - break - - # Check for client disconnection - if await request.is_disconnected(): - logger.info(f"Client disconnected for workflow {workflow.id}") - break - - current_time = asyncio.get_event_loop().time() - - # Periodically check workflow status (less frequent since we're event-driven) - if current_time - last_status_check >= status_check_interval: - try: - current_workflow = interfaceDbChat.getWorkflow(workflow.id) - if current_workflow and current_workflow.status == "stopped": - logger.info(f"Workflow {workflow.id} was stopped, closing stream") - break - except Exception as e: - logger.warning(f"Error checking workflow status: {e}") - last_status_check = current_time - - # Get event from queue (pure event-driven - no polling database) - try: - event = await asyncio.wait_for(queue.get(), timeout=1.0) - - # Handle different event types - event_type = event.get("type") - event_data = event.get("data", {}) - - # Emit chatdata events (messages, logs, stats, status, chunk) in exact chatData format - if event_type == "chatdata" and event_data: - # Handle status events (transient UI feedback) - if event_data.get("type") == "status": - # Status events have simple structure: {type: "status", label: "..."} - status_item = { - "type": "status", - "label": event_data.get("label", "") - } - yield f"data: {json.dumps(status_item)}\n\n" - elif event_data.get("type") == "chunk": - # Token chunks for ChatGPT-like streaming - chunk_item = { - "type": "chunk", - "content": event_data.get("content", "") - } - yield f"data: {json.dumps(chunk_item)}\n\n" - else: - # Emit other chatdata items (messages, logs, stats) in exact chatData format - chatdata_item = event_data - # Ensure item field is serializable (convert Pydantic models to dicts) - if isinstance(chatdata_item, dict) and "item" in chatdata_item: - item_obj = chatdata_item.get("item") - if hasattr(item_obj, "model_dump"): - chatdata_item = chatdata_item.copy() - chatdata_item["item"] = item_obj.model_dump() - yield f"data: {json.dumps(chatdata_item)}\n\n" - - # Handle completion/stopped events to close stream - elif event_type == "complete": - logger.info(f"Workflow {workflow.id} completed, closing stream") - break - elif event_type == "stopped": - logger.info(f"Workflow {workflow.id} stopped, closing stream") - break - elif event_type == "error" and event.get("step") == "error": - logger.warning(f"Workflow {workflow.id} error, closing stream") - break - - last_keepalive = current_time - except asyncio.TimeoutError: - # Send keepalive if needed (no events received, but keep connection alive) - current_time = asyncio.get_event_loop().time() - if current_time - last_keepalive >= keepalive_interval: - yield f": keepalive\n\n" - last_keepalive = current_time - continue - except Exception as e: - logger.error(f"Error in event stream: {e}") - break - - except Exception as e: - logger.error(f"Error in event stream generator: {e}", exc_info=True) - finally: - # Stream ends - cleanup handled by event manager - pass - - return StreamingResponse( - event_stream(), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Accel-Buffering": "no" # Disable buffering for nginx - } - ) - - except Exception as e: - logger.error(f"Error in stream_chatbot_start: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=str(e) - ) - - -# Workflow stop endpoint -@router.post("/{instanceId}/stop/{workflowId}", response_model=ChatbotConversation) -@limiter.limit("120/minute") -async def stop_chatbot( - request: Request, - instanceId: str = Path(..., description="Feature Instance ID"), - workflowId: str = Path(..., description="ID of the workflow to stop"), - context: RequestContext = Depends(getRequestContext) -) -> ChatbotConversation: - """Stops a running chatbot workflow.""" - # Validate instance access - mandateId = _validateInstanceAccess(instanceId, context) - - try: - # Get chatbot interface with instance context - interfaceDbChat = _getServiceChat(context, instanceId, mandateId=mandateId) - - # Get workflow to verify it exists and belongs to this instance - workflow = interfaceDbChat.getWorkflow(workflowId) - if not workflow: - raise HTTPException( - status_code=404, - detail=f"Workflow {workflowId} not found" - ) - - # Verify workflow belongs to this instance - if workflow.featureInstanceId and workflow.featureInstanceId != instanceId: - raise HTTPException( - status_code=403, - detail=f"Workflow {workflowId} does not belong to instance {instanceId}" - ) - - # Update workflow status to stopped - interfaceDbChat.updateWorkflow(workflowId, { - "status": "stopped", - "lastActivity": getUtcTimestamp() - }) - - event_manager = getEventManager(context.user, mandateId=mandateId, featureInstanceId=instanceId) - # Store log entry (createLog emits when event_manager is provided) - interfaceDbChat.createLog({ - "id": f"log_{uuid.uuid4()}", - "workflowId": workflowId, - "message": "Workflow stopped by user", - "type": "warning", - "status": "stopped", - "timestamp": getUtcTimestamp(), - "roundNumber": workflow.currentRound if workflow else 1 - }, event_manager=event_manager) - - # Reload workflow to return updated version - workflow = interfaceDbChat.getWorkflow(workflowId) - - # Emit stopped event to active streams - await event_manager.emit_event( - context_id=workflowId, - event_type="stopped", - data={"workflowId": workflowId}, - event_category="workflow", - message="Workflow stopped by user", - step="stopped" - ) - logger.info(f"Stopped workflow {workflowId} and emitted stopped event") - - return workflow - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error in stop_chatbot: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=str(e) - ) - -# Delete chatbot workflow endpoint -# NOTE: This catch-all route MUST be defined AFTER more specific routes like /threads -@router.delete("/{instanceId}/{workflowId}", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -def delete_chatbot( - request: Request, - instanceId: str = Path(..., description="Feature Instance ID"), - workflowId: str = Path(..., description="ID of the workflow to delete"), - context: RequestContext = Depends(getRequestContext) -) -> Dict[str, Any]: - """Deletes a chatbot workflow and its associated data.""" - # Validate instance access - if user has access to instance, they can delete their workflows - mandateId = _validateInstanceAccess(instanceId, context) - - try: - interfaceDbChat = _getServiceChat(context, instanceId, mandateId=mandateId) - - # Get workflow directly (interface already handles mandate filtering) - workflow = interfaceDbChat.getWorkflow(workflowId) - if not workflow: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Workflow with ID {workflowId} not found" - ) - - # Check if workflow is a chatbot workflow - if (workflow.workflowMode or getattr(workflow, "workflowMode", None)) != "Chatbot": - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Workflow {workflowId} is not a chatbot workflow" - ) - - # User has instance access, allow delete (no complex RBAC checks needed) - logger.info(f"User {context.user.id} deleting workflow {workflowId} from instance {instanceId}") - - # Delete workflow - success = interfaceDbChat.deleteWorkflow(workflowId) - - if not success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=routeApiMsg("Failed to delete workflow") - ) - - return { - "id": workflowId, - "message": "Chatbot workflow and associated data deleted successfully" - } - except HTTPException: - raise - except Exception as e: - logger.error(f"Error in delete_chatbot: {str(e)}", exc_info=True) - raise HTTPException( - status_code=500, - detail=f"Error deleting chatbot workflow: {str(e)}" - ) diff --git a/modules/features/chatbot/service.py b/modules/features/chatbot/service.py deleted file mode 100644 index 5d60cdd7..00000000 --- a/modules/features/chatbot/service.py +++ /dev/null @@ -1,1525 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -""" -Simple chatbot feature - basic implementation. -User input is processed by AI to create list of needed queries. -Those queries get streamed back. -""" - -import logging -import json -import uuid -import asyncio -import re -from typing import Optional, Dict, Any, List - -from modules.datamodels.datamodelChat import UserInputRequest -from modules.features.chatbot.interfaceFeatureChatbot import ChatbotConversation, ChatbotDocument -from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum -from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference -from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp -from modules.features.chatbot.mainChatbot import getChatbotServices, getChatStreamingHelper -from modules.features.chatbot.chatbot import Chatbot -from modules.features.chatbot.bridges.ai import AICenterChatModel, clear_workflow_allowed_providers -from modules.features.chatbot.bridges.memory import DatabaseCheckpointer -from modules.features.chatbot.config import ( - load_chatbot_config_from_instance, - ChatbotConfig -) -from modules.datamodels.datamodelAi import OperationTypeEnum, ProcessingModeEnum -from modules.workflows.methods.methodAi.methodAi import MethodAi -from modules.connectors.connectorPreprocessor import PreprocessorConnector -from modules.features.chatbot.chatbotConstants import generate_conversation_name, generate_name_from_prompt -import base64 - -logger = logging.getLogger(__name__) - - -def _extractJsonFromResponse(content: str) -> Optional[dict]: - """Extract JSON from AI response, handling markdown code blocks.""" - # Try direct JSON parse first - try: - return json.loads(content.strip()) - except json.JSONDecodeError: - pass - - # Try to extract JSON from markdown code blocks - json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL) - if json_match: - try: - return json.loads(json_match.group(1)) - except json.JSONDecodeError: - pass - - # Try to find JSON object in the text - json_match = re.search(r'\{.*\}', content, re.DOTALL) - if json_match: - try: - return json.loads(json_match.group(0)) - except json.JSONDecodeError: - pass - - return None - - -async def chatProcess( - currentUser: User, - mandateId: Optional[str], - userInput: UserInputRequest, - workflowId: Optional[str] = None, - featureInstanceId: Optional[str] = None, - event_manager=None # Required when called from streaming route -) -> ChatbotConversation: - """ - Simple chatbot processing - analyze user input and generate queries. - - Flow: - 1. Create or load workflow - 2. Store user message - 3. AI analyzes user input to create list of needed queries - 4. Stream queries back - - Args: - currentUser: Current user - mandateId: Mandate ID for the workflow - userInput: User input request - workflowId: Optional workflow ID to continue existing conversation - featureInstanceId: Feature instance ID for loading instance-specific config - - Returns: - ChatbotConversation instance - """ - try: - # Get services from service center (only services declared in mainChatbot.REQUIRED_SERVICES) - services = getChatbotServices(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - - # Load instance config and apply allowedProviders for AI calls (conversation name + main chat) - chatbot_config = await _load_chatbot_config(featureInstanceId) - if chatbot_config.model.allowedProviders: - services.allowedProviders = chatbot_config.model.allowedProviders - logger.info(f"Chatbot instance {featureInstanceId}: restricting to providers {chatbot_config.model.allowedProviders}") - - from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface - interfaceDbChat = getChatbotInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - - # Create or load workflow (event_manager passed from route) - if workflowId: - # Lightweight resume: minimal fetch + minimal update (no logs/messages load) - workflow = interfaceDbChat.getWorkflowMinimal(workflowId) - if not workflow: - raise ValueError(f"Workflow {workflowId} not found") - new_round = workflow.currentRound + 1 - workflow = interfaceDbChat.updateWorkflowMinimal(workflowId, { - "status": "running", - "currentRound": new_round, - "lastActivity": getUtcTimestamp() - }) - logger.info(f"Resumed workflow {workflowId}, round incremented to {new_round}") - - # Create event queue if it doesn't exist (for streaming) - if not event_manager.has_queue(workflowId): - event_manager.create_queue(workflowId) - else: - # Use placeholder name immediately - don't block on AI call - prompt_stripped = (userInput.prompt or "").strip() - trivial_prompts = {"test", "hi", "hallo", "hello", "hey"} - is_short = len(prompt_stripped) < 30 or prompt_stripped.lower() in trivial_prompts - - if is_short: - conversation_name = generate_name_from_prompt(userInput.prompt) - run_name_task = False - else: - conversation_name = "Neue Unterhaltung" - run_name_task = True - - # Create new conversation (per feature instance) - conversationData = { - "id": str(uuid.uuid4()), - "featureInstanceId": featureInstanceId, - "status": "running", - "name": conversation_name, - "currentRound": 1, - "workflowMode": "Chatbot", - "startedAt": getUtcTimestamp(), - "lastActivity": getUtcTimestamp() - } - workflow = interfaceDbChat.createConversation(conversationData) - logger.info(f"Created new chatbot workflow: {workflow.id} with name: {conversation_name}") - - # Run AI name generation in background (for longer prompts only) - if run_name_task: - asyncio.create_task(_update_conversation_name_async( - services=services, - currentUser=currentUser, - mandateId=mandateId, - featureInstanceId=featureInstanceId, - workflowId=workflow.id, - prompt=userInput.prompt, - userLanguage=userInput.userLanguage, - interfaceDbChat=interfaceDbChat, - event_manager=event_manager - )) - - # Create event queue for new workflow (for streaming) - event_manager.create_queue(workflow.id) - - # Reload workflow to get current message count - workflow = interfaceDbChat.getWorkflow(workflow.id) - services.workflow = workflow # Required for chat service document resolution - - # Process uploaded files and create ChatbotDocuments - user_documents = [] - if userInput.listFileId and len(userInput.listFileId) > 0: - logger.info(f"Processing {len(userInput.listFileId)} uploaded file(s) for user message") - for fileId in userInput.listFileId: - try: - # Get file info from chat service - fileInfo = services.chat.getFileInfo(fileId) - if not fileInfo: - logger.warning(f"No file info found for file ID {fileId}") - continue - - originalFileName = fileInfo.get("fileName", "unknown") - originalMimeType = fileInfo.get("mimeType", "application/octet-stream") - fileSizeToUse = fileInfo.get("size", 0) - - # Create ChatbotDocument for the file - document = ChatbotDocument( - id=str(uuid.uuid4()), - messageId="", # Will be set when message is created - fileId=fileId, - fileName=originalFileName, - fileSize=fileSizeToUse, - mimeType=originalMimeType, - roundNumber=workflow.currentRound, - taskNumber=0, - actionNumber=0 - ) - user_documents.append(document) - logger.info(f"Created ChatbotDocument for file {fileId} -> {originalFileName}") - except Exception as e: - logger.error(f"Error processing file ID {fileId}: {e}", exc_info=True) - - # Store user message (sequenceNr: for resume use message count, else len+1) - seq_nr = ( - interfaceDbChat.getMessageCount(workflow.id) + 1 - if workflowId - else len(workflow.messages) + 1 - ) - userMessageData: Dict[str, Any] = { - "id": f"msg_{uuid.uuid4()}", - "conversationId": workflow.id, - "message": userInput.prompt, - "role": "user", - "status": "first" if workflowId is None else "step", - "sequenceNr": seq_nr, - "publishedAt": getUtcTimestamp(), - "roundNumber": workflow.currentRound, - "taskNumber": 0, - "actionNumber": 0 - } - if user_documents: - userMessageData["documents"] = [d.model_dump() for d in user_documents] - # Don't pass event_manager: event_stream sends initial chatData from DB (includes user msg). - # Emitting here would duplicate it (initial chatData + queue event). - userMessage = interfaceDbChat.createMessage(userMessageData, event_manager=None) - logger.info(f"Stored user message: {userMessage.id} with {len(user_documents)} document(s)") - - # Update workflow status (minimal update for lastActivity; resume already did this) - if not workflowId: - interfaceDbChat.updateWorkflowMinimal(workflow.id, { - "status": "running", - "lastActivity": getUtcTimestamp() - }) - - # Pre-flight billing check before starting LangGraph (if mandateId present) - if mandateId: - _preflight_billing_check(services, mandateId, featureInstanceId) - - # Process in background using LangGraph (async) - asyncio.create_task(_processChatbotMessageLangGraph( - services, - currentUser, - workflow.id, - userInput, - userMessage.id, - featureInstanceId=featureInstanceId, - config=chatbot_config, - event_manager=event_manager - )) - return workflow - - except Exception as e: - logger.error(f"Error in chatProcess: {str(e)}", exc_info=True) - raise - - -async def _update_conversation_name_async( - services, - currentUser: User, - mandateId: Optional[str], - featureInstanceId: Optional[str], - workflowId: str, - prompt: str, - userLanguage: str, - interfaceDbChat, - event_manager, -) -> None: - """ - Background task: generate conversation name via AI and update workflow. - Runs in parallel to LangGraph so it doesn't block the first response. - """ - try: - new_name = await generate_conversation_name(services, prompt, userLanguage) - if new_name: - interfaceDbChat.updateWorkflow(workflowId, {"name": new_name, "lastActivity": getUtcTimestamp()}) - logger.info(f"Updated workflow {workflowId} name to: {new_name}") - # Emit stat event so frontend can refresh thread list/title - workflow = interfaceDbChat.getWorkflow(workflowId) - if workflow: - wf_dict = workflow.model_dump() - await event_manager.emit_event( - context_id=workflowId, - event_type="chatdata", - data={ - "type": "stat", - "createdAt": getUtcTimestamp(), - "item": wf_dict - }, - event_category="chat", - message="Workflow name updated", - step="workflow_update" - ) - except Exception as e: - logger.warning(f"Background conversation name update failed for workflow {workflowId}: {e}") - - -async def _execute_queries_parallel(queries: List[Dict[str, Any]]) -> Dict[str, Any]: - """ - Execute multiple SQL queries in parallel with shared connector. - - Args: - queries: List of query dictionaries, each containing: - - "query": SQL query string - - "purpose": Description of what the query retrieves - - "table": Primary table name - - Returns: - Dictionary mapping query indices to results: - - "query_1", "query_2", etc.: Success result text - - "query_1_data", "query_2_data", etc.: Raw data arrays - - "query_1_error", "query_2_error", etc.: Error messages if query failed - """ - # Create single connector instance to reuse across all queries - connector = PreprocessorConnector() - try: - async def execute_single_query(idx: int, query_info: Dict[str, Any]): - """Execute a single query using shared connector.""" - try: - query_text = query_info.get("query", "") - result = await connector.executeQuery(query_text, return_json=True) - return idx, result, None - except Exception as e: - return idx, None, str(e) - - # Execute all queries in parallel with shared connector - tasks = [execute_single_query(i, q) for i, q in enumerate(queries)] - results = await asyncio.gather(*tasks, return_exceptions=True) - finally: - # Close connector once after all queries complete - await connector.close() - - # Process results into dictionary - query_results = {} - for result in results: - if isinstance(result, Exception): - # Handle exceptions from gather - logger.error(f"Exception in parallel query execution: {result}") - continue - - idx, result_data, error = result - - if error: - query_results[f"query_{idx+1}_error"] = error - logger.error(f"Query {idx+1} failed: {error}") - else: - if result_data and not result_data.get("text", "").startswith(("Error:", "Query failed:")): - query_results[f"query_{idx+1}"] = result_data.get("text", "") - query_results[f"query_{idx+1}_data"] = result_data.get("data", []) - row_count = len(result_data.get('data', [])) - logger.info(f"Query {idx+1} executed successfully, returned {row_count} rows") - else: - error_text = result_data.get("text", "Query failed") if result_data else "Query failed: No response" - query_results[f"query_{idx+1}_error"] = error_text - logger.error(f"Query {idx+1} failed: {error_text}") - - return query_results - - -async def _emit_log_and_event( - interfaceDbChat, - workflowId: str, - event_manager, - message: str, - log_type: str = "info", - status: str = "running", - round_number: Optional[int] = None -) -> None: - """ - Store log in database and emit event for streaming. - - Args: - interfaceDbChat: Database interface - workflowId: Workflow ID - event_manager: Event manager for streaming - message: Log message - log_type: Log type (info, warning, error) - status: Status string - round_number: Optional round number (will be fetched from workflow if not provided) - """ - try: - # Get round number from workflow if not provided - if round_number is None: - workflow = interfaceDbChat.getWorkflow(workflowId) - if workflow: - round_number = workflow.currentRound - - log_timestamp = getUtcTimestamp() - log_data = { - "id": f"log_{uuid.uuid4()}", - "workflowId": workflowId, - "message": message, - "type": log_type, - "timestamp": log_timestamp, - "status": status, - "roundNumber": round_number - } - # Store log in database (createLog emits when event_manager is provided) - created_log = interfaceDbChat.createLog(log_data, event_manager=event_manager) - except Exception as e: - logger.error(f"Error storing log: {e}", exc_info=True) - - -async def _check_workflow_stopped(interfaceDbChat, workflowId: str) -> bool: - """ - Check if workflow was stopped. - - Args: - interfaceDbChat: Database interface - workflowId: Workflow ID - - Returns: - True if workflow is stopped, False otherwise - """ - try: - workflow = interfaceDbChat.getWorkflow(workflowId) - return workflow and workflow.status == "stopped" - except Exception as e: - logger.warning(f"Error checking workflow status: {e}") - return False - - -def _buildWebResearchQuery(userPrompt: str, workflowMessages: List, queryResults: Optional[Dict[str, Any]] = None) -> str: - """ - Build enriched web research query by extracting product context from conversation history and current prompt. - - Extracts product information from: - 1. Current user prompt (article numbers, product mentions) - 2. Database query results (if available) - 3. Previous assistant messages (conversation history) - - Args: - userPrompt: Current user prompt - workflowMessages: List of workflow messages (conversation history) - queryResults: Optional database query results to extract product info from - - Returns: - Enriched search query string - """ - # Normalize user prompt for detection - prompt_lower = userPrompt.lower().strip() - - # Patterns that indicate a search request - search_patterns = [ - "ja", "yes", "oui", "si", - "such", "suche", "search", "recherche", "recherchier", - "internet", "web", "online", - "datenblatt", "datasheet", "fiche technique", - "mehr informationen", "more information", "plus d'information", - "weitere informationen", "further information", "additional information" - ] - - # Certification patterns that require web research - certification_patterns = [ - "ul", "ce", "tüv", "vde", "iec", "en", "iso", - "zertifiziert", "certified", "certification", "zertifizierung", - "geprüft", "approved", "compliance" - ] - - # Check if current prompt contains search-related keywords - has_search_intent = any(pattern in prompt_lower for pattern in search_patterns) - - # Check if prompt contains certification-related keywords - has_certification_intent = any(pattern in prompt_lower for pattern in certification_patterns) - - # Extract product information - try multiple sources - article_number = None - article_description = None - supplier = None - - # Pattern for article numbers like "6AV2 181-8XP00-0AX0" or "6AV2181-8XP00-0AX0" - article_patterns = [ - r'\b[A-Z0-9]{2,}\s+[0-9]{3,}-[A-Z0-9-]+\b', # With space: "6AV2 181-8XP00-0AX0" - r'\b[A-Z0-9]{4,}[\s-][A-Z0-9-]{6,}\b', # General pattern - r'\b[A-Z]{2,}[0-9]+\s+[0-9]+-[A-Z0-9-]+\b', # Specific Siemens pattern - ] - - # 1. First, try to extract from current user prompt - for pattern in article_patterns: - matches = re.findall(pattern, userPrompt) - if matches: - article_number = matches[0] - logger.info(f"Extracted article number from user prompt: {article_number}") - break - - # 2. Try to extract from database query results if available - # Always check queryResults to enrich with product description and supplier, even if article_number was already found - if queryResults: - # Look for article numbers in query result text (if not already found) - if not article_number: - for key in queryResults.keys(): - if key.startswith("query_") and not key.endswith("_error") and not key.endswith("_data"): - result_text = queryResults.get(key, "") - if isinstance(result_text, str): - for pattern in article_patterns: - matches = re.findall(pattern, result_text) - if matches: - article_number = matches[0] - logger.info(f"Extracted article number from query results: {article_number}") - break - if article_number: - break - - # Always check data arrays for product description and supplier (even if article_number already found) - for key in queryResults.keys(): - if key.startswith("query_") and not key.endswith("_error") and not key.endswith("_data"): - data_key = f"{key}_data" - if data_key in queryResults: - data_array = queryResults[data_key] - if isinstance(data_array, list) and len(data_array) > 0: - # Look for article number in first row (if not already found) - first_row = data_array[0] - if isinstance(first_row, dict): - # Check common article number fields (if not already found) - if not article_number: - for field in ["Artikelnummer", "Artikelkürzel", "article_number", "articleNumber"]: - if field in first_row and first_row[field]: - article_number = str(first_row[field]) - logger.info(f"Extracted article number from query data: {article_number}") - break - - # Always check article description (can enrich even if article_number already found) - if not article_description: - for field in ["Artikelbezeichnung", "Bezeichnung", "article_description", "description"]: - if field in first_row and first_row[field]: - article_description = str(first_row[field]) - logger.info(f"Extracted article description from query data: {article_description}") - break - - # Always check supplier (can enrich even if article_number already found) - if not supplier: - for field in ["Lieferant", "Supplier", "supplier"]: - if field in first_row and first_row[field]: - supplier = str(first_row[field]) - logger.info(f"Extracted supplier from query data: {supplier}") - break - - # If we found all needed info, we can stop - if article_number and article_description and supplier: - break - - # Check if current prompt is an explicit search request that should NOT use context - # If user explicitly asks to search for something, prioritize that over previous messages - explicit_search_patterns = [ - r"recherchier\s+(?:im\s+internet\s+)?nach\s+(.+)", - r"suche\s+(?:im\s+internet\s+)?nach\s+(.+)", - r"search\s+(?:the\s+internet\s+)?for\s+(.+)", - r"find\s+(?:information\s+)?(?:about\s+)?(.+)", - r"recherche\s+(?:sur\s+internet\s+)?(.+)" - ] - - explicit_search_term = None - for pattern in explicit_search_patterns: - match = re.search(pattern, userPrompt, re.IGNORECASE) - if match: - explicit_search_term = match.group(1).strip() - logger.info(f"Found explicit search term in prompt: '{explicit_search_term}'") - break - - # 3. Extract from previous assistant messages (conversation history) - # ONLY if there's no explicit search term (to avoid using old context for new searches) - if not explicit_search_term and (not article_number or not article_description): - for msg in reversed(workflowMessages[-10:]): - if msg.role == "assistant": - message_text = msg.message - - # Extract article number if not found yet - if not article_number: - for pattern in article_patterns: - matches = re.findall(pattern, message_text) - if matches: - article_number = matches[0] - break - - # Extract article description if not found yet - if not article_description: - description_patterns = [ - r'Es handelt sich um\s+([^\.]+)', - r'It is a\s+([^\.]+)', - r'C\'est\s+([^\.]+)', - r'Bezeichnung:\s*([^\n]+)', - r'Description:\s*([^\n]+)', - r'Artikelbezeichnung:\s*([^\n]+)', - r'Artikelbezeichnung:\s*([^\n]+)' - ] - for pattern in description_patterns: - match = re.search(pattern, message_text, re.IGNORECASE) - if match: - article_description = match.group(1).strip() - break - - # Extract supplier if not found yet - if not supplier: - supplier_patterns = [ - r'von\s+([A-Z][A-Za-z\s]+(?:AG|GmbH|Ltd|Inc|Corp)?)', - r'from\s+([A-Z][A-Za-z\s]+(?:AG|GmbH|Ltd|Inc|Corp)?)', - r'Lieferant:\s*([^\n]+)', - r'Supplier:\s*([^\n]+)' - ] - for pattern in supplier_patterns: - match = re.search(pattern, message_text, re.IGNORECASE) - if match: - supplier = match.group(1).strip() - break - - # Stop if we found everything - if article_number and article_description and supplier: - break - - # Build enriched search query - query_parts = [] - - # If we have an explicit search term, use it as the primary query - if explicit_search_term: - query_parts.append(explicit_search_term) - logger.info(f"Using explicit search term as primary query: '{explicit_search_term}'") - # If we have search intent but no product info, try to use the user prompt intelligently - elif has_search_intent and not article_number and not article_description: - # Try to extract meaningful parts from the prompt - # Remove common search phrases and keep the product-related parts - cleaned_prompt = userPrompt - for phrase in ["recherchier", "recherche", "suche nach", "search for", "find", "informationen zu", "information about", "weitere informationen", "further information", "im internet", "the internet", "sur internet"]: - cleaned_prompt = re.sub(phrase, "", cleaned_prompt, flags=re.IGNORECASE) - cleaned_prompt = cleaned_prompt.strip() - - # Use cleaned prompt if it has meaningful content - if cleaned_prompt and len(cleaned_prompt) > 2: - query_parts.append(cleaned_prompt) - - # Add article description if found (but NOT if we have an explicit search term) - if article_description and not explicit_search_term: - query_parts.append(article_description) - - # Add article number if found (but NOT if we have an explicit search term) - if article_number and not explicit_search_term: - query_parts.append(article_number) - - # Add supplier if found (but NOT if we have an explicit search term) - if supplier and not explicit_search_term: - query_parts.append(supplier) - - # Extract certification information from prompt if present - certification_terms = [] - if has_certification_intent: - # Extract specific certification mentions - cert_keywords = { - "ul": "UL certification", - "ce": "CE certification", - "tüv": "TÜV certification", - "vde": "VDE certification", - "iec": "IEC certification", - "iso": "ISO certification" - } - for cert_key, cert_term in cert_keywords.items(): - if cert_key in prompt_lower: - certification_terms.append(cert_term) - - # If no specific certification found but certification intent detected, add generic term - if not certification_terms: - certification_terms.append("certification") - - # Add certification terms to query if found - if certification_terms: - query_parts.extend(certification_terms) - - # Add "Datenblatt" or "datasheet" if user requested it or if we have product info - # But NOT if we have an explicit search term (user wants to search for something specific) - if not explicit_search_term: - if "datenblatt" in prompt_lower or "datasheet" in prompt_lower or "fiche technique" in prompt_lower: - query_parts.append("Datenblatt") - elif query_parts and (article_number or article_description): - # If we have product info but no explicit request for datasheet, add it anyway - query_parts.append("Datenblatt") - - # If we found product information or built a meaningful query, use it - if query_parts: - enriched_query = " ".join(query_parts) - logger.info(f"Built enriched search query: '{enriched_query}' from context (original: '{userPrompt}')") - return enriched_query - else: - # Fall back to original prompt, but try to clean it up - logger.info(f"No product context found, using original prompt: '{userPrompt}'") - return userPrompt - - -async def _convert_file_ids_to_document_references( - services, - file_ids: List[str] -) -> DocumentReferenceList: - """ - Convert file IDs to DocumentReferenceList for use with ai.process. - - Args: - services: Services instance - file_ids: List of file IDs to convert - - Returns: - DocumentReferenceList with docItem references - """ - references = [] - - # Get workflow to search for ChatbotDocuments - workflow = services.workflow - if not workflow: - logger.error("Cannot convert file IDs to document references: workflow not set in services") - return DocumentReferenceList(references=[]) - - for file_id in file_ids: - try: - # Get file info to verify it exists - file_info = services.chat.getFileInfo(file_id) - if not file_info: - logger.warning(f"File {file_id} not found, skipping") - continue - - # Find ChatbotDocument that has this fileId - document_id = None - if workflow.messages: - for message in workflow.messages: - if hasattr(message, 'documents') and message.documents: - for doc in message.documents: - if getattr(doc, 'fileId', None) == file_id: - document_id = getattr(doc, 'id', None) - break - if document_id: - break - - # Search chatbot database if not found in messages - if not document_id: - try: - from modules.interfaces.interfaceRbac import getRecordsetWithRBAC - chatbotInterface = services.interfaceDbChat - documents = getRecordsetWithRBAC( - chatbotInterface.db, - ChatbotDocument, - services.user, - recordFilter={"fileId": file_id} - ) - if documents: - workflow_message_ids = {msg.id for msg in workflow.messages} if workflow.messages else set() - for doc in documents: - if doc.get("messageId") in workflow_message_ids: - document_id = doc.get("id") - break - except Exception: - pass # Fallback to fileId - - # Use ChatbotDocument ID if found, otherwise use fileId as fallback - ref = DocumentItemReference(documentId=document_id if document_id else file_id) - references.append(ref) - except Exception as e: - logger.error(f"Error converting fileId {file_id}: {e}", exc_info=True) - - logger.info(f"Converted {len(references)} file IDs to document references") - return DocumentReferenceList(references=references) - - -def _format_query_results_as_lookup(query_data: Dict[str, List[Dict]]) -> str: - """ - Format database query results as JSON lookup table for Excel matching. - Converts query result data into structured JSON format: {Artikelnummer: {columns...}} - - Args: - query_data: Dict with query_key -> list of row dicts (from connector with return_json=True) - - Returns: - JSON string formatted as lookup table - """ - lookup_table = {} - - for query_key, rows in query_data.items(): - if query_key == "error" or not rows: - logger.warning(f"Skipping query key '{query_key}' - no rows or error") - continue - - logger.info(f"Processing {len(rows)} rows from query '{query_key}'") - - for row in rows: - if not isinstance(row, dict): - logger.warning(f"Skipping non-dict row: {type(row)}") - continue - - # Find Artikelnummer field (case-insensitive) - artikelnummer = None - for key in row.keys(): - if key.lower() in ['artikelnummer', 'artikel_nummer', 'art_nr', 'part_number']: - artikelnummer = str(row[key]) - break - - if artikelnummer: - lookup_table[artikelnummer] = row - else: - logger.warning(f"No Artikelnummer found in row with keys: {list(row.keys())}") - - logger.info(f"Generated lookup table with {len(lookup_table)} entries") - if lookup_table: - sample_keys = list(lookup_table.keys())[:3] - logger.info(f"Sample Artikelnummern: {sample_keys}") - if sample_keys: - sample_entry = lookup_table[sample_keys[0]] - logger.info(f"Sample entry keys: {list(sample_entry.keys())}") - - return json.dumps(lookup_table, ensure_ascii=False, indent=2) - - -async def _create_chat_document_from_action_document( - services, - action_document, - message_id: str, - workflow_id: str, - round_number: int -) -> ChatbotDocument: - """ - Create a ChatbotDocument from an ActionDocument by storing the file data. - - Args: - services: Services instance - action_document: ActionDocument from ai.process result - message_id: ID of the message to attach to - workflow_id: Workflow ID - round_number: Round number - - Returns: - ChatbotDocument instance - """ - try: - # Get file data (could be bytes or string) - document_data = action_document.documentData - - # Convert to bytes if needed - if isinstance(document_data, str): - # Check if it's base64 encoded - try: - # Try to decode as base64 first - file_bytes = base64.b64decode(document_data) - except Exception: - # Not base64, encode as UTF-8 - file_bytes = document_data.encode('utf-8') - elif isinstance(document_data, bytes): - file_bytes = document_data - else: - # Try to convert to bytes - try: - file_bytes = bytes(document_data) - except Exception: - # Last resort: convert to string then encode - file_bytes = str(document_data).encode('utf-8') - - # Get MIME type (default to Excel) - mime_type = action_document.mimeType or "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - - # Get file name - file_name = action_document.documentName or "data_export.xlsx" - # Ensure it has .xlsx extension - if not file_name.lower().endswith('.xlsx'): - # Remove any existing extension and add .xlsx - file_name = file_name.rsplit('.', 1)[0] + '.xlsx' - - # Store file using component interface - file_item = services.interfaceDbComponent.createFile( - name=file_name, - mimeType=mime_type, - content=file_bytes - ) - - # Store file data - success = services.interfaceDbComponent.createFileData(file_item.id, file_bytes) - if not success: - logger.warning(f"Failed to store file data for {file_item.id}, but continuing...") - - # Create ChatbotDocument - chat_document = ChatbotDocument( - id=str(uuid.uuid4()), - messageId=message_id, - fileId=file_item.id, - fileName=file_name, - fileSize=len(file_bytes), - mimeType=mime_type, - roundNumber=round_number, - taskNumber=0, - actionNumber=0 - ) - - logger.info(f"Created ChatbotDocument {chat_document.id} from ActionDocument {file_name} (size: {len(file_bytes)} bytes)") - return chat_document - - except Exception as e: - logger.error(f"Error creating ChatbotDocument from ActionDocument: {e}", exc_info=True) - raise - - -async def _bridge_chatbot_events( - event_stream, - event_manager, - workflow_id: str, - interface_db_chat -): - """ - Bridge legacy chatbot events to current event manager format. - - Args: - event_stream: Async iterator from chatbot.stream_events() - event_manager: Event manager instance - workflow_id: Workflow ID - interface_db_chat: Database interface for storing messages - """ - try: - final_message_stored = False - - async for event in event_stream: - event_type = event.get("type") - - # Handle status updates - emit immediately for real-time UI feedback - # Note: Status updates are transient UI feedback, no need to persist them - if event_type == "status": - label = event.get("label", "") - if label: - # Emit the status event directly for real-time UI feedback - await event_manager.emit_event( - context_id=workflow_id, - event_type="chatdata", - data={ - "type": "status", - "label": label.strip() - }, - event_category="chat", - message="Status update", - step="status" - ) - continue - - # Handle token chunks for ChatGPT-like streaming (append to message as it's generated) - if event_type == "chunk": - content = event.get("content", "") - if content: - await event_manager.emit_event( - context_id=workflow_id, - event_type="chatdata", - data={"type": "chunk", "content": content}, - event_category="chat", - message="Token chunk", - step="chunk" - ) - continue - - # Handle final response - if event_type == "final": - response_data = event.get("response", {}) - chat_history = response_data.get("chat_history", []) - - # The final message should already be stored by the memory/checkpointer - # We just need to emit the event, not store it again - # Check if the message was already stored by checking the workflow - workflow = interface_db_chat.getWorkflow(workflow_id) - if workflow and workflow.messages: - # Find the last assistant message in the workflow (already stored by memory) - last_message = workflow.messages[-1] - if last_message.role == "assistant": - final_message_stored = True - - # Emit message event for the already-stored message - message_timestamp = parseTimestamp(last_message.publishedAt, default=getUtcTimestamp()) - await event_manager.emit_event( - context_id=workflow_id, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": last_message.model_dump() - }, - event_category="chat" - ) - else: - # If no assistant message found, try to store from chat_history - assistant_message = None - for msg in reversed(chat_history): - if msg.get("role") == "assistant" and msg.get("content"): - assistant_message = msg - break - - if assistant_message: - message_data = { - "id": f"msg_{workflow_id}_{getUtcTimestamp()}", - "workflowId": workflow_id, - "message": assistant_message.get("content", ""), - "role": "assistant", - "status": "last", - "sequenceNr": len(workflow.messages) + 1 if workflow.messages else 1, - "publishedAt": getUtcTimestamp(), - "roundNumber": workflow.currentRound if workflow else 1, - "taskNumber": 0, - "actionNumber": 0, - "success": True - } - - try: - assistant_msg = interface_db_chat.createMessage(message_data, event_manager=event_manager) - final_message_stored = True - - # Emit message event - message_timestamp = parseTimestamp(assistant_msg.publishedAt, default=getUtcTimestamp()) - await event_manager.emit_event( - context_id=workflow_id, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": assistant_msg.model_dump() - }, - event_category="chat" - ) - except Exception as e: - logger.error(f"Error storing assistant message: {e}", exc_info=True) - - # Emit completion event - await event_manager.emit_event( - context_id=workflow_id, - event_type="complete", - data={"workflowId": workflow_id}, - event_category="workflow", - message="Chatbot-Verarbeitung abgeschlossen", - step="complete" - ) - clear_workflow_allowed_providers(workflow_id) - - # Update workflow status - try: - interface_db_chat.updateWorkflow(workflow_id, { - "status": "completed", - "lastActivity": getUtcTimestamp() - }) - except Exception as e: - logger.error(f"Error updating workflow status: {e}", exc_info=True) - - return - - # Handle errors - if event_type == "error": - error_msg = event.get("message", "Unknown error") - - await event_manager.emit_event( - context_id=workflow_id, - event_type="error", - data={"error": error_msg}, - event_category="workflow", - message=f"Fehler beim Verarbeiten: {error_msg}", - step="error" - ) - clear_workflow_allowed_providers(workflow_id) - - # Update workflow status - try: - interface_db_chat.updateWorkflow(workflow_id, { - "status": "error", - "lastActivity": getUtcTimestamp() - }) - except Exception as e: - logger.error(f"Error updating workflow status: {e}", exc_info=True) - - return - - # If stream ended without final message, store error message - if not final_message_stored: - logger.warning(f"Stream ended for workflow {workflow_id} without a final message") - try: - workflow = interface_db_chat.getWorkflow(workflow_id) - if workflow: - error_message_data = { - "id": f"msg_{workflow_id}_{getUtcTimestamp()}", - "workflowId": workflow_id, - "message": "Entschuldigung, ich konnte keine vollständige Antwort generieren. Bitte versuchen Sie es erneut.", - "role": "assistant", - "status": "last", - "sequenceNr": len(workflow.messages) + 1 if workflow.messages else 1, - "publishedAt": getUtcTimestamp(), - "roundNumber": workflow.currentRound if workflow else 1, - "taskNumber": 0, - "actionNumber": 0, - "success": False - } - - error_msg = interface_db_chat.createMessage(error_message_data, event_manager=event_manager) - - # Emit message event - message_timestamp = parseTimestamp(error_msg.publishedAt, default=getUtcTimestamp()) - await event_manager.emit_event( - context_id=workflow_id, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": error_msg.model_dump() - }, - event_category="chat" - ) - except Exception as e: - logger.error(f"Error storing error message: {e}", exc_info=True) - - # Emit completion event - await event_manager.emit_event( - context_id=workflow_id, - event_type="complete", - data={"workflowId": workflow_id}, - event_category="workflow", - message="Chatbot-Verarbeitung abgeschlossen", - step="complete" - ) - clear_workflow_allowed_providers(workflow_id) - - # Update workflow status - try: - interface_db_chat.updateWorkflow(workflow_id, { - "status": "completed", - "lastActivity": getUtcTimestamp() - }) - except Exception as e: - logger.error(f"Error updating workflow status: {e}", exc_info=True) - - except Exception as e: - logger.error(f"Error in bridge_chatbot_events: {e}", exc_info=True) - - # Emit error event - await event_manager.emit_event( - context_id=workflow_id, - event_type="error", - data={"error": str(e)}, - event_category="workflow", - message=f"Fehler beim Verarbeiten: {str(e)}", - step="error" - ) - clear_workflow_allowed_providers(workflow_id) - - -def _load_chatbot_config_sync(featureInstanceId: Optional[str]) -> ChatbotConfig: - """ - Load chatbot configuration from FeatureInstance (database). Sync version for use in executor. - - Args: - featureInstanceId: Feature instance ID to load config from - - Returns: - ChatbotConfig instance - """ - if not featureInstanceId: - raise ValueError("featureInstanceId is required to load chatbot config") - - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.interfaces.interfaceFeatures import getFeatureInterface - - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - instance = featureInterface.getFeatureInstance(featureInstanceId) - - if not instance: - raise ValueError(f"FeatureInstance {featureInstanceId} not found") - - logger.info(f"Loading chatbot config from FeatureInstance {featureInstanceId}") - return load_chatbot_config_from_instance(instance) - - -def _warm_model_registry_cache_sync( - currentUser: "User", - mandateId: Optional[str] = None, - featureInstanceId: Optional[str] = None, -) -> None: - """ - Pre-warm getAvailableModels cache so planner/agent model selection is a cache hit. - Uses mandateId/featureInstanceId for faster RBAC (fewer roles to load). - Runs in executor to avoid blocking event loop. - """ - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.aicore.aicoreModelRegistry import modelRegistry - - root = getRootInterface() - modelRegistry.getAvailableModels( - currentUser=currentUser, - rbacInstance=root.rbac, - mandateId=mandateId, - featureInstanceId=featureInstanceId, - ) - - -async def _load_chatbot_config(featureInstanceId: Optional[str]) -> ChatbotConfig: - """ - Load chatbot configuration from FeatureInstance (database). - Runs in thread pool to avoid blocking event loop on DB I/O. - - Args: - featureInstanceId: Feature instance ID to load config from - - Returns: - ChatbotConfig instance - - Raises: - ValueError: If no featureInstanceId provided or instance not found - """ - if not featureInstanceId: - raise ValueError("featureInstanceId is required to load chatbot config") - - try: - return await asyncio.to_thread(_load_chatbot_config_sync, featureInstanceId) - except ValueError: - raise - except Exception as e: - logger.error(f"Error loading config from FeatureInstance {featureInstanceId}: {e}") - raise - - -def _preflight_billing_check(services, mandateId: str, featureInstanceId: Optional[str]) -> None: - """ - Pre-flight billing check before starting chatbot AI processing. - Raises if mandate has insufficient balance or no providers allowed. - Uses services.billing from service center (REQUIRED_SERVICES). - Exception types from BillingService class (service center billing API). - """ - from modules.serviceCenter.services.serviceBilling import BillingService - - billingService = services.billing - if not billingService: - raise BillingService.BillingContextError("Billing service not available for chatbot") - try: - balanceCheck = billingService.checkBalance(0.01) - if not balanceCheck.allowed: - mid = str(getattr(services, "mandateId", None) or mandateId or "") - from modules.serviceCenter.services.serviceBilling.billingExhaustedNotify import ( - maybeEmailMandatePoolExhausted, - ) - u = getattr(services, "user", None) - ulabel = ( - (getattr(u, "email", None) or getattr(u, "username", None) or str(getattr(u, "id", ""))) - if u is not None else "" - ) - maybeEmailMandatePoolExhausted( - mid, - str(getattr(u, "id", "") if u is not None else ""), - ulabel, - float(balanceCheck.currentBalance or 0.0), - 0.01, - ) - raise BillingService.InsufficientBalanceException.fromBalanceCheck( - balanceCheck, - mid, - 0.01, - ) - rbacAllowedProviders = billingService.getallowedProviders() - if not rbacAllowedProviders: - raise BillingService.ProviderNotAllowedException( - provider="any", - message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator." - ) - except (BillingService.InsufficientBalanceException, BillingService.ProviderNotAllowedException): - raise - except Exception as e: - logger.error(f"Billing pre-flight failed: {e}") - raise BillingService.BillingContextError(f"Billing check failed: {e}") - - -def _create_chatbot_billing_callback(services, workflow_id: str): - """ - Create billing callback for AICenterChatModel. Records each AI call to poweron_billing. - Uses services.billing from service center (REQUIRED_SERVICES). - """ - from modules.datamodels.datamodelAi import AiCallResponse - - billingService = services.billing - if not billingService: - return lambda _: None # No-op callback if billing unavailable - - def _billing_callback(response: AiCallResponse) -> None: - if not response or getattr(response, "errorCount", 0) > 0: - return - priceCHF = getattr(response, "priceCHF", 0.0) - if not priceCHF or priceCHF <= 0: - return - provider = getattr(response, "provider", None) or "unknown" - modelName = getattr(response, "modelName", None) or "unknown" - try: - billingService.recordUsage( - priceCHF=priceCHF, - workflowId=workflow_id, - aicoreProvider=provider, - aicoreModel=modelName, - description=f"AI: {modelName}" - ) - logger.debug(f"Chatbot billed: {priceCHF:.4f} CHF, provider={provider}, model={modelName}") - except Exception as e: - logger.error(f"Chatbot billing failed: {e}") - - return _billing_callback - - -async def _processChatbotMessageLangGraph( - services, - currentUser: User, - workflowId: str, - userInput: UserInputRequest, - userMessageId: str, - featureInstanceId: Optional[str] = None, - config: Optional[ChatbotConfig] = None, - event_manager=None -): - """ - Process chatbot message using LangGraph. - Uses LangGraph workflow with AI center models and tools. - - Args: - services: Service container - currentUser: Current user - workflowId: Workflow ID - userInput: User input request - userMessageId: User message ID - featureInstanceId: Optional feature instance ID for loading instance-specific config - event_manager: Event manager for streaming (passed from chatProcess) - """ - try: - # Start config + model cache warm in parallel (planner/agent need cache hit to avoid 2–3 s per call) - config_task = asyncio.create_task(_load_chatbot_config(featureInstanceId)) if config is None else None - warm_task = asyncio.create_task(asyncio.to_thread( - _warm_model_registry_cache_sync, currentUser, services.mandateId, featureInstanceId - )) - - # Emit first status immediately so stream feels responsive - await event_manager.emit_event( - context_id=workflowId, - event_type="chatdata", - data={"type": "status", "label": "Starte..."}, - event_category="chat", - message="Status update", - step="status", - ) - - # Reuse interfaceDbChat from services (ChatObjects) - avoids duplicate DB init - interfaceDbChat = services.interfaceDbChat - - # Reload workflow to get current messages - workflow = interfaceDbChat.getWorkflow(workflowId) - if not workflow: - logger.error(f"Workflow {workflowId} not found during processing") - await event_manager.emit_event( - context_id=workflowId, - event_type="error", - data={"error": f"Workflow {workflowId} nicht gefunden"}, - event_category="workflow", - message=f"Workflow {workflowId} nicht gefunden", - step="error" - ) - return - - # Check if workflow was stopped before starting - if await _check_workflow_stopped(interfaceDbChat, workflowId): - logger.info(f"Workflow {workflowId} was stopped, aborting processing") - return - - # Await config and model cache warm (planner gets cache hit, saves ~2–3 s) - if config_task is not None: - config = await config_task - await warm_task - - # Replace {{DATE}} placeholder in system prompt - from datetime import datetime - system_prompt = config.systemPrompt.replace( - "{{DATE}}", - datetime.now().strftime("%d.%m.%Y") - ) - - # Create AI center model - operation_type = OperationTypeEnum[config.model.operationType] - processing_mode = ProcessingModeEnum[config.model.processingMode] - - billing_callback = None - if services.mandateId: - billing_callback = _create_chatbot_billing_callback(services, workflowId) - - allowed_providers = config.model.allowedProviders or None - if allowed_providers: - logger.info(f"Chatbot AICenterChatModel: restricting to providers {allowed_providers}") - model = AICenterChatModel( - user=currentUser, - operation_type=operation_type, - processing_mode=processing_mode, - billing_callback=billing_callback, - workflow_id=workflowId, - allowed_providers=allowed_providers, - mandate_id=services.mandateId, - feature_instance_id=featureInstanceId, - ) - # Fast planner model (gpt-4o-mini etc.) for routing - saves ~1-2 s on first response - planner_model = AICenterChatModel( - user=currentUser, - operation_type=operation_type, - processing_mode=processing_mode, - billing_callback=billing_callback, - workflow_id=workflowId, - allowed_providers=allowed_providers, - prefer_fast_model=True, - mandate_id=services.mandateId, - feature_instance_id=featureInstanceId, - ) - - # Create memory/checkpointer (reuse interface to avoid extra DB init) - memory = DatabaseCheckpointer( - user=currentUser, - workflow_id=workflowId, - mandateId=services.mandateId, - featureInstanceId=featureInstanceId, - interface=interfaceDbChat, - ) - - # Create chatbot instance with config for dynamic tool configuration - # Use mainChatbot.getChatStreamingHelper() - resolves from service center (legacy hub.streaming has no getChatStreamingHelper) - from modules.features.chatbot.mainChatbot import getChatStreamingHelper - chat_streaming_helper = getChatStreamingHelper() - if not chat_streaming_helper: - logger.warning("ChatStreamingHelper not available from streaming service; message normalization may fail") - chatbot = await Chatbot.create( - model=model, - memory=memory, - system_prompt=system_prompt, - workflow_id=workflowId, - config=config, - event_manager=event_manager, - planner_model=planner_model, - chat_streaming_helper=chat_streaming_helper, - ) - - # Emit synthetic status for real-time UI feedback - await event_manager.emit_event( - context_id=workflowId, - event_type="chatdata", - data={"type": "status", "label": "Analysiere Anfrage..."}, - event_category="chat", - message="Status update", - step="status" - ) - - # Stream events using chatbot - event_stream = chatbot.stream_events( - message=userInput.prompt, - chat_id=workflowId - ) - - # Bridge chatbot events to event manager - await _bridge_chatbot_events( - event_stream=event_stream, - event_manager=event_manager, - workflow_id=workflowId, - interface_db_chat=interfaceDbChat - ) - - # Schedule cleanup - await event_manager.cleanup(workflowId, delay=300.0) # 5 minutes delay - - except Exception as e: - logger.error(f"Error processing chatbot message with LangGraph: {str(e)}", exc_info=True) - - # Check if workflow was stopped - if so, don't store error message - if await _check_workflow_stopped(interfaceDbChat, workflowId): - logger.info(f"Workflow {workflowId} was stopped, not storing error message") - return - - # Store error message - try: - workflow = interfaceDbChat.getWorkflow(workflowId) - - if workflow and workflow.status == "stopped": - logger.info(f"Workflow {workflowId} was stopped, not storing error message") - return - - errorMessageData = { - "id": f"msg_{uuid.uuid4()}", - "workflowId": workflowId, - "parentMessageId": userMessageId, - "message": f"Sorry, I encountered an error: {str(e)}", - "role": "assistant", - "status": "last", - "sequenceNr": len(workflow.messages) + 1 if workflow else 1, - "publishedAt": getUtcTimestamp(), - "success": False, - "roundNumber": workflow.currentRound if workflow else 1, - "taskNumber": 0, - "actionNumber": 0 - } - errorMessage = interfaceDbChat.createMessage(errorMessageData, event_manager=event_manager) - - # Emit message event - message_timestamp = parseTimestamp(errorMessage.publishedAt, default=getUtcTimestamp()) - await event_manager.emit_event( - context_id=workflowId, - event_type="chatdata", - data={ - "type": "message", - "createdAt": message_timestamp, - "item": errorMessage.model_dump() - }, - event_category="chat" - ) - - # Update workflow status - if not await _check_workflow_stopped(interfaceDbChat, workflowId): - interfaceDbChat.updateWorkflow(workflowId, { - "status": "error", - "lastActivity": getUtcTimestamp() - }) - - # Schedule cleanup - await event_manager.cleanup(workflowId) - except Exception as storeError: - logger.error(f"Error storing error message: {storeError}") - - -async def _processChatbotMessage( - services, - workflowId: str, - userInput: UserInputRequest, - userMessageId: str -): - """ - DEPRECATED: Old chatbot processing implementation. - Kept for backward compatibility but redirects to LangGraph implementation. - """ - logger.warning("_processChatbotMessage is deprecated, using LangGraph implementation") - # Note: currentUser should be passed, but this function signature doesn't have it - # This is a deprecated function, so we'll need to get user from workflow or services - # For now, raise an error to indicate this needs to be fixed - raise NotImplementedError("_processChatbotMessage is deprecated and requires currentUser parameter") diff --git a/modules/features/commcoach/CONCEPT.md b/modules/features/commcoach/CONCEPT.md index 5444c5f3..7a8fcd00 100644 --- a/modules/features/commcoach/CONCEPT.md +++ b/modules/features/commcoach/CONCEPT.md @@ -33,7 +33,7 @@ Feature Registration → mainCommcoach.py | DatabaseConnector | `connectorDbPostgre.py` | New DB `poweron_commcoach` | | VoiceObjects (STT/TTS) | `interfaceVoiceObjects.py` | Voice pipeline | | MessagingInterface | `interfaceMessaging.py` | Email summaries | -| SSE Pattern | chatbot `routeFeatureChatbot.py` | Chat streaming | +| SSE Pattern | workspace `routeFeatureWorkspace.py` | Chat streaming | | PDF Renderer | `rendererPdf.py` | Dossier export (Iteration 2) | | EventManagement | `eventManagement.py` | Scheduled reminders | diff --git a/modules/features/realEstate/bzoExtraction.py b/modules/features/realEstate/bzoExtraction.py index 9bec66fe..f56405ed 100644 --- a/modules/features/realEstate/bzoExtraction.py +++ b/modules/features/realEstate/bzoExtraction.py @@ -1,12 +1,15 @@ """ -LangGraph-based pipeline for extracting structured content from BZO PDFs. +Pipeline for extracting structured content from BZO PDFs. + +The extraction runs as a plain sequential pipeline of step functions; each step +takes the shared state dict, mutates/returns it, and the steps are chained +directly (no external workflow-orchestration framework). """ import logging import re from typing import TypedDict, List, Dict, Any, Optional from dataclasses import dataclass -from langgraph.graph import StateGraph, START, END from modules.features.realEstate.bzoPdfExtractor import BZOPdfExtractor, TextBlock from modules.features.realEstate.bzoRuleTaxonomy import RULE_TAXONOMY @@ -14,7 +17,7 @@ from modules.features.realEstate.bzoRuleTaxonomy import RULE_TAXONOMY logger = logging.getLogger(__name__) -# ===== BZO Params Extraction State (LangGraph with LLM) ===== +# ===== BZO Params Extraction State (LLM step) ===== class BZOParamsExtractionState(TypedDict): """State for BZO params extraction via LLM.""" @@ -947,7 +950,7 @@ def extract_wohnzone_params( } -# ===== LangGraph: LLM-based BZO Params Extraction ===== +# ===== LLM-based BZO Params Extraction ===== def _build_bauzone_context_for_llm(state: BZOParamsExtractionState) -> str: """Build context string for LLM from extracted BZO content.""" @@ -1089,7 +1092,7 @@ Antwort NUR mit den relevanten Artikelnummern, eine pro Zeile (z.B. "Art. 15", " async def llm_extract_bauzone_params_node(state: BZOParamsExtractionState) -> BZOParamsExtractionState: - """LangGraph node: use LLM to extract BZO parameters for Bauzone as bullet list.""" + """Use LLM to extract BZO parameters for Bauzone as bullet list.""" bauzone = state.get("bauzone", "") gemeinde = state.get("gemeinde", "") ai_service = state.get("ai_service") @@ -1192,15 +1195,6 @@ Antwort NUR mit der Bullet-Liste, sonst nichts:""" } -def create_bzo_params_extraction_graph(): - """Create LangGraph for LLM-based BZO params extraction.""" - workflow = StateGraph(BZOParamsExtractionState) - workflow.add_node("llm_extract", llm_extract_bauzone_params_node) - workflow.set_entry_point("llm_extract") - workflow.add_edge("llm_extract", END) - return workflow.compile() - - def _filter_articles_by_bauzone(articles: List[Dict[str, Any]], bauzone: str) -> List[Dict[str, Any]]: """Filter articles that mention the Bauzone.""" bauzone_upper = (bauzone or "").upper() @@ -1247,7 +1241,7 @@ async def run_bzo_params_extraction( total_area_m2: Optional[float] = None, ) -> Dict[str, Any]: """ - Run LangGraph workflow to extract BZO parameters for a Bauzone via LLM. + Extract BZO parameters for a Bauzone via LLM. Returns fakten (item/value/source), bauzone_params_list (bullet strings), zusatzinformationen. """ rules = relevant_rules if relevant_rules is not None else _bzo_filter_rules_by_bauzone( @@ -1276,8 +1270,7 @@ async def run_bzo_params_extraction( "errors": [], } - graph = create_bzo_params_extraction_graph() - final_state = await graph.ainvoke(state) + final_state = await llm_extract_bauzone_params_node(state) return { "bauzone": bauzone, @@ -1288,19 +1281,14 @@ async def run_bzo_params_extraction( } -# ===== Graph Construction ===== +# ===== Pipeline Execution ===== -def create_bzo_extraction_graph(): - """Create and compile the BZO extraction graph (simplified 4-node pipeline).""" - workflow = StateGraph(BZOExtractionState) - workflow.add_node("classify_and_assemble", classify_and_assemble) - workflow.add_node("extract_zones_and_tables", extract_zones_and_tables) - workflow.add_node("extract_rules", extract_rules) - workflow.set_entry_point("classify_and_assemble") - workflow.add_edge("classify_and_assemble", "extract_zones_and_tables") - workflow.add_edge("extract_zones_and_tables", "extract_rules") - workflow.add_edge("extract_rules", END) - return workflow.compile() +def _run_bzo_extraction_pipeline(state: BZOExtractionState) -> BZOExtractionState: + """Run the BZO extraction steps sequentially on the shared state.""" + state = classify_and_assemble(state) + state = extract_zones_and_tables(state) + state = extract_rules(state) + return state def run_extraction(pdf_bytes: bytes, pdf_id: str = None, dokument_id: str = None) -> Dict[str, Any]: @@ -1357,9 +1345,8 @@ def run_extraction(pdf_bytes: bytes, pdf_id: str = None, dokument_id: str = None for tb in text_blocks_objects ] - # Create and run graph - graph = create_bzo_extraction_graph() - final_state = graph.invoke(state) + # Run the extraction pipeline + final_state = _run_bzo_extraction_pipeline(state) # Sort and structure results articles = sorted( diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py index 061a54c3..d825ec50 100644 --- a/modules/features/realEstate/mainRealEstate.py +++ b/modules/features/realEstate/mainRealEstate.py @@ -278,7 +278,7 @@ from .interfaceFeatureRealEstate import getInterface as getRealEstateInterface from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface from modules.connectors.connectorSwissTopoMapServer import SwissTopoMapServerConnector from modules.features.realEstate.bzoDocumentRetriever import BZODocumentRetriever -from modules.features.realEstate.bzoExtractionLangGraph import run_extraction, run_bzo_params_extraction +from modules.features.realEstate.bzoExtraction import run_extraction, run_bzo_params_extraction from modules.features.realEstate.parcelSelectionService import compute_selection_summary from modules.features.realEstate.realEstateGemeindeService import ( ensure_single_gemeinde, @@ -2331,7 +2331,7 @@ async def extract_bzo_information( Extract BZO information from PDF documents for a specific Bauzone in a Gemeinde. Retrieves BZO documents for the specified Gemeinde, extracts content using - langgraph workflow, filters by Bauzone, and uses AI to find relevant information. + the BZO extraction pipeline, filters by Bauzone, and uses AI to find relevant information. When total_area_m2 or parcels are provided, runs Machbarkeitsstudie for structured output. Args: @@ -2489,7 +2489,7 @@ async def extract_bzo_information( ) continue - # Run extraction using langgraph workflow + # Run extraction using the BZO extraction pipeline extraction_result = run_extraction( pdf_bytes=pdf_bytes, pdf_id=dokument.dokumentReferenz or f"dok_{dokument.id}", @@ -2543,7 +2543,7 @@ async def extract_bzo_information( selection_summary = compute_selection_summary(parcels) _total_area_m2 = selection_summary.get("total_area_m2") or 0.0 - # Extract BZO parameters for Wohnzone via LangGraph + LLM (bullet list with sources) + # Extract BZO parameters for Wohnzone via LLM (bullet list with sources) bzo_params_result = None try: services = getServices( diff --git a/modules/features/realEstate/routeFeatureRealEstate.py b/modules/features/realEstate/routeFeatureRealEstate.py index 05f029e7..230c5d80 100644 --- a/modules/features/realEstate/routeFeatureRealEstate.py +++ b/modules/features/realEstate/routeFeatureRealEstate.py @@ -706,7 +706,7 @@ async def get_parcel_documents( """ Ensure BZO document exists for Gemeinde, return documents for parcel info display. Creates Gemeinde (Swiss Topo) and BZO (Tavily) if not in DB. - Returns documents for preview - does NOT run LangGraph. + Returns documents for preview - does NOT run the BZO extraction pipeline. """ mandateId = _validateInstanceAccess(instanceId, context) interface = getRealEstateInterface( @@ -787,7 +787,7 @@ async def get_instance_bzo_information( parcel_ids: Optional[str] = Query(None, description="Comma-separated parcel IDs; total area computed from parcels"), context: RequestContext = Depends(getRequestContext), ) -> Dict[str, Any]: - """Extract BZO information for a Bauzone in a Gemeinde. Runs LangGraph workflow. With total_area_m2 or parcel_ids, includes Machbarkeitsstudie.""" + """Extract BZO information for a Bauzone in a Gemeinde. Runs the BZO extraction pipeline. With total_area_m2 or parcel_ids, includes Machbarkeitsstudie.""" mandateId = _validateInstanceAccess(instanceId, context) parcels = None if parcel_ids: diff --git a/modules/features/workspace/routeFeatureWorkspace.py b/modules/features/workspace/routeFeatureWorkspace.py index e5ba470a..6b701432 100644 --- a/modules/features/workspace/routeFeatureWorkspace.py +++ b/modules/features/workspace/routeFeatureWorkspace.py @@ -203,8 +203,9 @@ def buildDataSourceContext(chatService, dataSourceIds: List[str]) -> str: "- Use ONLY browseDataSource, searchDataSource, and downloadFromDataSource to access these sources.", "- Use the dataSourceId (UUID) exactly as shown below.", "- Do NOT use listFiles, externalBrowse, or externalSearch for attached data sources -- those tools are for other purposes.", - "- browseDataSource returns BOTH files and folders at the given path.", - "- When downloading files, ALWAYS provide the human-readable fileName (with extension) from the browse results.", + "- SEARCH FIRST: for any targeted question (specific topic, sender, keyword, date range), use searchDataSource. It queries the source server-side and is far cheaper than listing everything. These sources can be huge (gigabytes); NEVER browse and download an entire source to find a few items.", + "- browseDataSource is for directory listing or getting the newest items of a folder. Download ONLY the items that match the user's request.", + "- When downloading files, ALWAYS provide the human-readable fileName (with extension) from the browse/search results.", "", ] found = False diff --git a/modules/interfaces/interfaceFeatures.py b/modules/interfaces/interfaceFeatures.py index c965edb2..0f7d20e1 100644 --- a/modules/interfaces/interfaceFeatures.py +++ b/modules/interfaces/interfaceFeatures.py @@ -50,7 +50,7 @@ class FeatureInterface: Get a feature by code. Args: - featureCode: Feature code (e.g., "trustee", "chatbot") + featureCode: Feature code (e.g., "trustee", "commcoach") Returns: Feature object or None diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index 948609ef..29182ae2 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -196,11 +196,6 @@ TABLE_NAMESPACE = { "ChatLog": "chat", "ChatDocument": "chat", "Prompt": "chat", - # Chatbot (poweron_chatbot) - per feature-instance isolation - "ChatbotConversation": "chatbot", - "ChatbotMessage": "chatbot", - "ChatbotDocument": "chatbot", - "ChatbotLog": "chatbot", # Files - benutzer-eigen "FileItem": "files", "FileData": "files", @@ -228,7 +223,7 @@ TABLE_NAMESPACE = { # Namespaces ohne Mandantenkontext - GROUP wird auf MY gemappt # NOTE: "files" is NOT in this set – files use scope-based visibility for GROUP -USER_OWNED_NAMESPACES = {"chat", "chatbot", "automation", "knowledge", "datasource"} +USER_OWNED_NAMESPACES = {"chat", "automation", "knowledge", "datasource"} def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str: diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index 511babde..2b1a928e 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -53,7 +53,7 @@ router = APIRouter( class FeatureInstanceCreate(BaseModel): """Request model for creating a feature instance""" - featureCode: str = Field(..., description="Feature code (e.g., 'trustee', 'chatbot')") + featureCode: str = Field(..., description="Feature code (e.g., 'trustee', 'commcoach')") label: str = Field(..., description="Instance label (e.g., 'Buchhaltung 2025')") enabled: bool = Field(True, description="Whether this feature instance is enabled") copyTemplateRoles: bool = Field(True, description="Whether to copy template roles on creation") @@ -778,11 +778,6 @@ def updateFeatureInstance( detail=routeApiMsg("Failed to update feature instance") ) - # Clear chatbot config cache when config was updated for chatbot instances - if "config" in updateData and instance.featureCode == "chatbot": - from modules.features.chatbot.config import clear_config_cache - clear_config_cache(instanceId) - logger.info(f"User {context.user.id} updated feature instance {instanceId}: {updateData}") return updated.model_dump() @@ -1637,7 +1632,7 @@ def get_feature( /instances, /my, /templates, etc. Args: - featureCode: Feature code (e.g., 'trustee', 'chatbot') + featureCode: Feature code (e.g., 'trustee', 'commcoach') """ try: # Features come from the RBAC Catalog (code-defined, not DB-stored) diff --git a/modules/routes/routeAdminSttBenchmark.py b/modules/routes/routeAdminSttBenchmark.py deleted file mode 100644 index ae24e792..00000000 --- a/modules/routes/routeAdminSttBenchmark.py +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""STT Benchmark route — compare Speech-to-Text v1 (latest_long) vs v2 (Chirp 2). - -Sysadmin-only page for evaluating STT model quality and latency. -""" - -import json -import time -import logging -from typing import Any, Dict - -from fastapi import APIRouter, HTTPException, Depends, Request, UploadFile, File, Form -from modules.auth import limiter, getCurrentUser -from modules.datamodels.datamodelUam import User -from modules.shared.configuration import APP_CONFIG - -logger = logging.getLogger(__name__) - -router = APIRouter( - prefix="/api/admin/stt-benchmark", - tags=["Admin STT Benchmark"], - responses={401: {"description": "Unauthorized"}, 403: {"description": "Forbidden"}}, -) - - -def _requireSysAdmin(currentUser: User = Depends(getCurrentUser)) -> User: - if not getattr(currentUser, "isSysAdmin", False) and not getattr(currentUser, "isPlatformAdmin", False): - raise HTTPException(status_code=403, detail="SysAdmin required") - return currentUser - - -def _getCredentials(): - apiKey = APP_CONFIG.get("Connector_GoogleSpeech_API_KEY_SECRET") - if not apiKey or apiKey.startswith("YOUR_"): - raise HTTPException(status_code=500, detail="Google Speech API key not configured") - from google.oauth2 import service_account - return service_account.Credentials.from_service_account_info(json.loads(apiKey)) - - -def _runV1(audioBytes: bytes, language: str, model: str) -> Dict[str, Any]: - """Run Speech-to-Text v1 recognition.""" - from google.cloud import speech - credentials = _getCredentials() - client = speech.SpeechClient(credentials=credentials) - - config = speech.RecognitionConfig( - encoding=speech.RecognitionConfig.AudioEncoding.ENCODING_UNSPECIFIED, - language_code=language, - model=model, - enable_automatic_punctuation=True, - enable_word_time_offsets=True, - enable_word_confidence=True, - max_alternatives=3, - use_enhanced=True, - ) - audio = speech.RecognitionAudio(content=audioBytes) - - t0 = time.perf_counter() - response = client.recognize(config=config, audio=audio) - elapsed = time.perf_counter() - t0 - - results = [] - for r in response.results: - for alt in r.alternatives: - results.append({ - "transcript": alt.transcript, - "confidence": round(alt.confidence, 4), - "words": len(alt.words) if alt.words else 0, - }) - - return { - "api": "v1", - "model": model, - "latencyMs": round(elapsed * 1000, 1), - "results": results, - "resultCount": len(response.results), - } - - -def _runV2(audioBytes: bytes, language: str, model: str, location: str) -> Dict[str, Any]: - """Run Speech-to-Text v2 recognition (Chirp 2).""" - from google.cloud.speech_v2 import SpeechClient - from google.cloud.speech_v2.types import cloud_speech - - credentials = _getCredentials() - credInfo = json.loads(APP_CONFIG.get("Connector_GoogleSpeech_API_KEY_SECRET")) - projectId = credInfo.get("project_id", "") - - client = SpeechClient( - credentials=credentials, - client_options={"api_endpoint": f"{location}-speech.googleapis.com"}, - ) - - config = cloud_speech.RecognitionConfig( - auto_decoding_config=cloud_speech.AutoDetectDecodingConfig(), - language_codes=[language], - model=model, - features=cloud_speech.RecognitionFeatures( - enable_automatic_punctuation=True, - enable_word_time_offsets=True, - enable_word_confidence=True, - ), - ) - - recognizer = f"projects/{projectId}/locations/{location}/recognizers/_" - - request = cloud_speech.RecognizeRequest( - recognizer=recognizer, - config=config, - content=audioBytes, - ) - - t0 = time.perf_counter() - response = client.recognize(request=request) - elapsed = time.perf_counter() - t0 - - results = [] - for r in response.results: - for alt in r.alternatives: - results.append({ - "transcript": alt.transcript, - "confidence": round(alt.confidence, 4), - "words": len(alt.words) if alt.words else 0, - }) - - return { - "api": "v2", - "model": model, - "location": location, - "latencyMs": round(elapsed * 1000, 1), - "results": results, - "resultCount": len(getattr(response, "results", [])), - } - - -@router.post("/run") -@limiter.limit("10/minute") -async def runBenchmark( - request: Request, - file: UploadFile = File(...), - language: str = Form(default="de-DE"), - v1Model: str = Form(default="latest_long"), - v2Model: str = Form(default="chirp_2"), - v2Location: str = Form(default="europe-west4"), - currentUser: User = Depends(_requireSysAdmin), -) -> Dict[str, Any]: - """Upload audio and compare v1 vs v2 STT results.""" - audioBytes = await file.read() - if len(audioBytes) > 10 * 1024 * 1024: - raise HTTPException(status_code=400, detail="Audio file too large (max 10 MB)") - if len(audioBytes) < 100: - raise HTTPException(status_code=400, detail="Audio file too small") - - logger.info("STT benchmark: %s, %d bytes, language=%s, v1=%s, v2=%s@%s", - file.filename, len(audioBytes), language, v1Model, v2Model, v2Location) - - v1Result = None - v1Error = None - try: - v1Result = _runV1(audioBytes, language, v1Model) - except Exception as e: - v1Error = str(e) - logger.warning("STT v1 benchmark failed: %s", e) - - v2Result = None - v2Error = None - try: - v2Result = _runV2(audioBytes, language, v2Model, v2Location) - except Exception as e: - v2Error = str(e) - logger.warning("STT v2 benchmark failed: %s", e) - - return { - "filename": file.filename, - "fileSizeBytes": len(audioBytes), - "language": language, - "v1": v1Result or {"error": v1Error}, - "v2": v2Result or {"error": v2Error}, - } - - -@router.get("/models") -@limiter.limit("30/minute") -async def getAvailableModels( - request: Request, - currentUser: User = Depends(_requireSysAdmin), -) -> Dict[str, Any]: - """Return available STT models for the benchmark UI.""" - return { - "v1Models": [ - {"value": "latest_long", "label": "latest_long (default)"}, - {"value": "latest_short", "label": "latest_short"}, - {"value": "phone_call", "label": "phone_call"}, - {"value": "video", "label": "video"}, - {"value": "command_and_search", "label": "command_and_search"}, - ], - "v2Models": [ - {"value": "chirp_2", "label": "Chirp 2 (recommended)"}, - {"value": "chirp", "label": "Chirp (original)"}, - {"value": "long", "label": "long"}, - {"value": "short", "label": "short"}, - ], - "locations": [ - {"value": "europe-west4", "label": "Europe West (NL)"}, - {"value": "us-central1", "label": "US Central"}, - {"value": "asia-southeast1", "label": "Asia Southeast"}, - ], - "languages": [ - {"value": "de-DE", "label": "Deutsch (DE)"}, - {"value": "de-CH", "label": "Deutsch (CH)"}, - {"value": "en-US", "label": "English (US)"}, - {"value": "en-GB", "label": "English (GB)"}, - {"value": "fr-FR", "label": "Francais (FR)"}, - {"value": "it-IT", "label": "Italiano (IT)"}, - ], - } diff --git a/modules/routes/routeDataSources.py b/modules/routes/routeDataSources.py index eccbf276..e448523b 100644 --- a/modules/routes/routeDataSources.py +++ b/modules/routes/routeDataSources.py @@ -2,7 +2,7 @@ # All rights reserved. """DataSource auxiliary endpoints: settings (ragLimits) and cost estimate. -Flag toggles (neutralize / scope / ragIndexEnabled) have moved to the +Flag toggles (neutralize / ragIndexEnabled) have moved to the generic UDB router (`POST /api/udb/node/{key}/flag/{flag}`); see `modules/routes/routeUdb.py` and the wiki UDB reference page. """ @@ -127,9 +127,8 @@ def _updateDataSourceSettings( Currently supports `ragLimits` only. Unknown top-level keys in the body are rejected to avoid silently storing garbage that no consumer reads. - DataSource: owner-only (or sysadmin). For mandate/feature scopes the - mandateAdmin also passes. FeatureDataSource has no userId/scope; for - those we require a feature-admin role on the FDS's featureInstanceId. + DataSource: owner-only (or sysadmin). FeatureDataSource requires + a feature-admin role on the FDS's featureInstanceId. """ if not isinstance(settings, dict): raise HTTPException(status_code=400, detail="settings must be an object") @@ -148,13 +147,7 @@ def _updateDataSourceSettings( if model is DataSource: ownerId = str(rec.get("userId") or "") if ownerId and ownerId != currentUserId and not context.isSysAdmin: - from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag - connectionId = rec.get("connectionId", "") - allDs = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId}) - scope = str(getEffectiveFlag(rec, "scope", allDs, mode="walk")) - isMandateAdmin = getattr(context, "isMandateAdmin", False) - if scope == "personal" or not isMandateAdmin: - raise HTTPException(status_code=403, detail="Not allowed to modify this DataSource's settings") + raise HTTPException(status_code=403, detail="Not allowed to modify this DataSource's settings") else: from modules.serviceCenter.services.serviceKnowledge.udbNodes import _isFeatureAdmin featureInstanceId = str(rec.get("featureInstanceId") or "") diff --git a/modules/routes/routeRealEstate.py b/modules/routes/routeRealEstate.py index cc571c2f..247d3b8b 100644 --- a/modules/routes/routeRealEstate.py +++ b/modules/routes/routeRealEstate.py @@ -2234,13 +2234,13 @@ async def get_bzo_information( """ Extract BZO information from PDF documents for a specific Bauzone in a Gemeinde. - Uses a langgraph workflow to extract content from BZO PDF documents for the + Uses the BZO extraction pipeline to extract content from BZO PDF documents for the specified Gemeinde, then uses AI to search for relevant information specific to the specified Bauzone. The workflow: 1. Finds BZO documents for the Gemeinde (by name or ID) - 2. Extracts content from PDFs using langgraph workflow + 2. Extracts content from PDFs using the BZO extraction pipeline 3. Filters rules, zones, and articles by Bauzone 4. Uses AI to generate a summary and find relevant information diff --git a/modules/routes/routeSystem.py b/modules/routes/routeSystem.py index 573df000..772e5018 100644 --- a/modules/routes/routeSystem.py +++ b/modules/routes/routeSystem.py @@ -114,9 +114,6 @@ def _getFeatureUiObjects(featureCode: str) -> List[Dict[str, Any]]: elif featureCode == "neutralization": from modules.features.neutralization.mainNeutralization import UI_OBJECTS return UI_OBJECTS - elif featureCode == "chatbot": - from modules.features.chatbot.mainChatbot import UI_OBJECTS - return UI_OBJECTS elif featureCode == "commcoach": from modules.features.commcoach.mainCommcoach import UI_OBJECTS return UI_OBJECTS diff --git a/modules/routes/routeUdb.py b/modules/routes/routeUdb.py index 177778d2..dfec38e9 100644 --- a/modules/routes/routeUdb.py +++ b/modules/routes/routeUdb.py @@ -50,9 +50,6 @@ router = APIRouter( ) -_VALID_SCOPES = {"personal", "featureInstance", "mandate", "global"} - - # --------------------------------------------------------------------------- # POST /api/udb/tree/children # --------------------------------------------------------------------------- @@ -98,7 +95,6 @@ class _UdbFlagBody(BaseModel): the flag (see `_extractFlagValue` for the mapping). `value` is typed as Any because the legal type depends on the flag: - neutralize/ragIndexEnabled : bool | null (null = inherit) - - scope : str | null (one of _VALID_SCOPES, null = inherit) """ value: Any = Field(default=None, description="New flag value or null to reset to inherit.") @@ -108,7 +104,7 @@ class _UdbFlagBody(BaseModel): async def _udbNodeFlag( request: Request, nodeKey: str = Path(..., description="Tree key of the node to modify"), - flag: str = Path(..., description="One of: neutralize | scope | ragIndexEnabled"), + flag: str = Path(..., description="One of: neutralize | ragIndexEnabled"), body: _UdbFlagBody = Body(default_factory=_UdbFlagBody), context: RequestContext = Depends(getRequestContext), ) -> Dict[str, Any]: @@ -122,7 +118,7 @@ async def _udbNodeFlag( RBAC: `node.canEdit(context, rootIf)` decides; the route never re-implements ownership rules. """ - if flag not in ("neutralize", "scope", "ragIndexEnabled"): + if flag not in ("neutralize", "ragIndexEnabled"): raise HTTPException(status_code=400, detail=f"Unknown flag: {flag}") value = _validateFlagValue(flag, body.value, context) @@ -188,15 +184,6 @@ def _validateFlagValue(flag: str, value: Any, context: RequestContext) -> Any: """ if value is None: return None - if flag == "scope": - if not isinstance(value, str) or value not in _VALID_SCOPES: - raise HTTPException( - status_code=400, - detail=f"Invalid scope: {value!r}. Must be one of {sorted(_VALID_SCOPES)}", - ) - if value == "global" and not context.isSysAdmin: - raise HTTPException(status_code=403, detail=routeApiMsg("Only sysadmins can set global scope")) - return value # neutralize / ragIndexEnabled if isinstance(value, bool): return value diff --git a/modules/security/rbacHelpers.py b/modules/security/rbacHelpers.py index c42716e7..a191e5d8 100644 --- a/modules/security/rbacHelpers.py +++ b/modules/security/rbacHelpers.py @@ -53,7 +53,7 @@ def checkUiAccess( Args: RbacInstance: RbacClass instance currentUser: Current user object - uiPath: UI path (e.g., "playground.voice.settings", "chatbot.search") + uiPath: UI path (e.g., "playground.voice.settings", "workspace.search") Returns: True if user has view permission for the UI element, False otherwise diff --git a/modules/serviceCenter/core/serviceStreaming/__init__.py b/modules/serviceCenter/core/serviceStreaming/__init__.py index 7f93d475..ae7f582b 100644 --- a/modules/serviceCenter/core/serviceStreaming/__init__.py +++ b/modules/serviceCenter/core/serviceStreaming/__init__.py @@ -3,7 +3,6 @@ """Streaming core service for SSE event management.""" from .eventManager import EventManager, get_event_manager -from .helpers import ChatStreamingHelper from .mainServiceStreaming import StreamingService -__all__ = ["EventManager", "get_event_manager", "ChatStreamingHelper", "StreamingService"] +__all__ = ["EventManager", "get_event_manager", "StreamingService"] diff --git a/modules/serviceCenter/core/serviceStreaming/helpers.py b/modules/serviceCenter/core/serviceStreaming/helpers.py deleted file mode 100644 index 664130ec..00000000 --- a/modules/serviceCenter/core/serviceStreaming/helpers.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. -"""Streaming helper utilities for chat message processing and normalization.""" - -from __future__ import annotations - -from typing import Any, Dict, List, Literal, Mapping, Optional - -from langchain_core.messages import ( - AIMessage, - BaseMessage, - HumanMessage, - SystemMessage, - ToolMessage, -) - -Role = Literal["user", "assistant", "system", "tool"] - - -class ChatStreamingHelper: - """Pure helper methods for streaming and message normalization. - - This class provides static utility methods for converting between different - message formats, extracting content, and normalizing message structures - for streaming chat applications. - """ - - @staticmethod - def role_from_message(*, msg: BaseMessage) -> Role: - """Extract the role from a BaseMessage instance. - - Args: - msg: The BaseMessage instance to extract the role from. - - Returns: - The role as a string literal: "user", "assistant", "system", or "tool". - Defaults to "assistant" if the message type is not recognized. - - Examples: - >>> from langchain_core.messages import HumanMessage - >>> msg = HumanMessage(content="Hello") - >>> ChatStreamingHelper.role_from_message(msg=msg) - 'user' - """ - if isinstance(msg, HumanMessage): - return "user" - if isinstance(msg, AIMessage): - return "assistant" - if isinstance(msg, SystemMessage): - return "system" - if isinstance(msg, ToolMessage): - return "tool" - return getattr(msg, "role", "assistant") - - @staticmethod - def flatten_content(*, content: Any) -> str: - """Convert complex content structures to plain text. - - This method handles various content formats including strings, lists of - content parts, and dictionaries with text fields. It's designed to - normalize content from different message sources into a consistent - plain text format. - - Args: - content: The content to flatten. Can be: - - str: Returned as-is after stripping whitespace - - list: Each item processed and joined with newlines - - dict: Text extracted from "text" or "content" fields - - None: Returns empty string - - Any other type: Converted to string - - Returns: - The flattened content as a plain text string with whitespace stripped. - - Examples: - >>> content = [{"type": "text", "text": "Hello"}, {"type": "text", "text": "world"}] - >>> ChatStreamingHelper.flatten_content(content=content) - 'Hello\nworld' - - >>> content = {"text": "Simple message"} - >>> ChatStreamingHelper.flatten_content(content=content) - 'Simple message' - """ - if content is None: - return "" - if isinstance(content, str): - return content.strip() - if isinstance(content, list): - parts: List[str] = [] - for part in content: - if isinstance(part, dict): - if "text" in part and isinstance(part["text"], str): - parts.append(part["text"]) - elif part.get("type") == "text" and isinstance( - part.get("text"), str - ): - parts.append(part["text"]) - elif "content" in part and isinstance(part["content"], str): - parts.append(part["content"]) - else: - # Fallback for unknown dictionary structures - val = part.get("value") - if isinstance(val, str): - parts.append(val) - else: - parts.append(str(part)) - return "\n".join(p.strip() for p in parts if p is not None) - if isinstance(content, dict): - if "text" in content and isinstance(content["text"], str): - return content["text"].strip() - if "content" in content and isinstance(content["content"], str): - return content["content"].strip() - return str(content).strip() - - @staticmethod - def message_to_dict(*, msg: BaseMessage) -> Dict[str, Any]: - """Convert a BaseMessage instance to a dictionary for streaming output. - - This method normalizes BaseMessage instances into a consistent dictionary - format suitable for JSON serialization and streaming to clients. - - Args: - msg: The BaseMessage instance to convert. - - Returns: - A dictionary containing: - - "role": The message role (user, assistant, system, tool) - - "content": The flattened message content as plain text - - "tool_calls": Tool calls if present (optional) - - "name": Message name if present (optional) - - Examples: - >>> from langchain_core.messages import HumanMessage - >>> msg = HumanMessage(content="Hello there") - >>> result = ChatStreamingHelper.message_to_dict(msg=msg) - >>> result["role"] - 'user' - >>> result["content"] - 'Hello there' - """ - payload: Dict[str, Any] = { - "role": ChatStreamingHelper.role_from_message(msg=msg), - "content": ChatStreamingHelper.flatten_content( - content=getattr(msg, "content", "") - ), - } - tool_calls = getattr(msg, "tool_calls", None) - if tool_calls: - payload["tool_calls"] = tool_calls - name = getattr(msg, "name", None) - if name: - payload["name"] = name - return payload - - @staticmethod - def dict_message_to_dict(*, obj: Mapping[str, Any]) -> Dict[str, Any]: - """Convert a dictionary-shaped message to a normalized dictionary. - - This method handles messages that come from serialized state and are - represented as dictionaries rather than BaseMessage instances. It - normalizes various dictionary formats into a consistent structure. - - Args: - obj: The dictionary-shaped message to convert. Expected to contain - fields like "role", "type", "content", "text", etc. - - Returns: - A normalized dictionary containing: - - "role": The message role (user, assistant, system, tool) - - "content": The flattened message content as plain text - - "tool_calls": Tool calls if present (optional) - - "name": Message name if present (optional) - - Examples: - >>> obj = {"type": "human", "content": "Hello"} - >>> result = ChatStreamingHelper.dict_message_to_dict(obj=obj) - >>> result["role"] - 'user' - >>> result["content"] - 'Hello' - """ - role: Optional[str] = obj.get("role") - if not role: - # Handle alternative type field mappings - typ = obj.get("type") - if typ in ("human", "user"): - role = "user" - elif typ in ("ai", "assistant"): - role = "assistant" - elif typ in ("system",): - role = "system" - elif typ in ("tool", "function"): - role = "tool" - - content = obj.get("content") - if content is None and "text" in obj: - content = obj["text"] - - out: Dict[str, Any] = { - "role": role or "assistant", - "content": ChatStreamingHelper.flatten_content(content=content), - } - if "tool_calls" in obj: - out["tool_calls"] = obj["tool_calls"] - if obj.get("name"): - out["name"] = obj["name"] - return out - - @staticmethod - def extract_messages_from_output(*, output_obj: Any) -> List[Any]: - """Extract messages from LangGraph output objects. - - This method handles various output formats from LangGraph execution, - extracting the messages list from different possible structures. - - Args: - output_obj: The output object from LangGraph execution. Can be: - - An object with a "messages" attribute - - A dictionary with a "messages" key - - Any other object (returns empty list) - - Returns: - A list of extracted messages, or an empty list if no messages - are found or if the output object is None. - - Examples: - >>> output = {"messages": [{"role": "user", "content": "Hello"}]} - >>> messages = ChatStreamingHelper.extract_messages_from_output(output_obj=output) - >>> len(messages) - 1 - """ - if output_obj is None: - return [] - - # Try to parse dicts first - if isinstance(output_obj, dict): - msgs = output_obj.get("messages") - return msgs if isinstance(msgs, list) else [] - - # Then try to get messages attribute - msgs = getattr(output_obj, "messages", None) - return msgs if isinstance(msgs, list) else [] diff --git a/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py b/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py index 8594c0e6..ee534706 100644 --- a/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py +++ b/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py @@ -9,7 +9,6 @@ import logging from typing import Any, Callable from modules.serviceCenter.core.serviceStreaming.eventManager import EventManager, get_event_manager -from modules.serviceCenter.core.serviceStreaming.helpers import ChatStreamingHelper logger = logging.getLogger(__name__) @@ -25,7 +24,3 @@ class StreamingService: def getEventManager(self) -> EventManager: """Get the global event manager instance for SSE streaming.""" return get_event_manager() - - def getChatStreamingHelper(self): - """Get ChatStreamingHelper utility for message normalization (no legacy import at call site).""" - return ChatStreamingHelper diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py index 5ad4310d..a853301f 100644 --- a/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_dataSourceTools.py @@ -21,6 +21,90 @@ from modules.serviceCenter.services.serviceAgent.coreTools._helpers import ( logger = logging.getLogger(__name__) +def _formatMailLine(entry) -> str: + """Format a mail ExternalEntry with sender + date so the agent can decide + which messages to download without fetching every body.""" + meta = entry.metadata or {} + sender = meta.get("from") or "" + received = (meta.get("receivedDateTime") or meta.get("date") or "")[:24] + parts = [] + if received: + parts.append(received) + if sender: + parts.append(f"from: {sender}") + metaStr = f" [{', '.join(parts)}]" if parts else "" + return f"- ✉️ {entry.name}{metaStr} path: {entry.path}" + + +def _formatContactLine(entry) -> str: + """Format a contact ExternalEntry with email/phone/company inline so the + agent does not have to download a .vcf for every contact. Handles the + differing metadata keys across MSFT, Google and Infomaniak adapters.""" + meta = entry.metadata or {} + # email: MSFT 'emailAddresses' (list), Google 'emails' (list), Infomaniak 'email' (str) + emails = meta.get("emailAddresses") or meta.get("emails") or [] + if isinstance(emails, str): + emails = [emails] + email = next((e for e in emails if e), None) or meta.get("email") or "" + # phone: MSFT 'businessPhones'/'mobilePhone', Google 'phones', Infomaniak 'phone' + phones = meta.get("phones") or meta.get("businessPhones") or [] + if isinstance(phones, str): + phones = [phones] + phone = next((p for p in phones if p), None) or meta.get("mobilePhone") or meta.get("phone") or "" + company = meta.get("companyName") or meta.get("organization") or "" + parts = [] + if email: + parts.append(email) + if phone: + parts.append(phone) + if company: + parts.append(company) + metaStr = f" [{', '.join(parts)}]" if parts else "" + return f"- 👤 {entry.name}{metaStr} path: {entry.path}" + + +def _formatTaskLine(entry) -> str: + """Format a ClickUp task with status/assignee/due-date inline so the agent + can answer task questions without downloading every task JSON.""" + meta = entry.metadata or {} + task = meta.get("task") or {} + parts = [] + status = ((task.get("status") or {}).get("status")) if isinstance(task.get("status"), dict) else task.get("status") + if status: + parts.append(f"status: {status}") + assignees = [a.get("username") or a.get("email") for a in (task.get("assignees") or []) if a] + assignees = [a for a in assignees if a] + if assignees: + parts.append(f"assignee: {', '.join(assignees)}") + dueMs = task.get("due_date") + if dueMs: + try: + from datetime import datetime, timezone + due = datetime.fromtimestamp(int(dueMs) / 1000, tz=timezone.utc).strftime("%Y-%m-%d") + parts.append(f"due: {due}") + except (TypeError, ValueError, OverflowError): + pass + metaStr = f" [{', '.join(parts)}]" if parts else "" + return f"- ☑️ {entry.name}{metaStr} path: {entry.path}" + + +def _buildCountLine(entries, limit) -> str: + """Build a summary count line, including total estimate if available.""" + realCount = sum(1 for e in entries if not (e.path or "").endswith("/_count")) + line = f"\n\n({realCount} entries returned" + if limit is not None: + line += f", limit={limit}" + for e in entries: + if (e.path or "").endswith("/_count"): + meta = e.metadata or {} + total = meta.get("totalEstimate") or meta.get("totalCount") + if total: + line += f", ~{total} total in source" + break + line += ")" + return line + + def _registerDataSourceTools(registry: ToolRegistry, services): """Auto-extracted from registerCoreTools.""" @@ -81,6 +165,8 @@ def _registerDataSourceTools(registry: ToolRegistry, services): _MAIL_SERVICES = {"outlook", "gmail"} _CALENDAR_SERVICES = {"calendar", "calendarFolder"} + _CONTACT_SERVICES = {"contact", "contactFolder"} + _CLICKUP_SERVICES = {"clickup", "clickupList"} async def _browseDataSource(args: Dict[str, Any], context: Dict[str, Any]): dsId = args.get("dataSourceId", "") @@ -118,6 +204,9 @@ def _registerDataSourceTools(registry: ToolRegistry, services): return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data="Empty directory.") lines = [] isCalendar = service in _CALENDAR_SERVICES + isMail = service in _MAIL_SERVICES + isContact = service in _CONTACT_SERVICES + isClickup = service in _CLICKUP_SERVICES for e in entries: prefix = "[DIR]" if e.isFolder else "[FILE]" sizeInfo = f" ({e.size} bytes)" if e.size else "" @@ -127,18 +216,24 @@ def _registerDataSourceTools(registry: ToolRegistry, services): loc = e.metadata.get("location") or "" locStr = f" 📍 {loc}" if loc else "" lines.append(f"- 📅 {start} – {end} {e.name}{locStr}") + elif isMail and not e.isFolder: + lines.append(_formatMailLine(e)) + elif isContact and not e.isFolder: + lines.append(_formatContactLine(e)) + elif isClickup and not e.isFolder and (e.metadata or {}).get("cuType") == "task": + lines.append(_formatTaskLine(e)) else: lines.append(f"- {prefix} {e.name}{sizeInfo} path: {e.path}") result = "\n".join(lines) - countLine = f"\n\n({len(entries)} entries returned" - if limit is not None: - countLine += f", limit={limit}" - countLine += ")" - result += countLine + result += _buildCountLine(entries, limit) if service in _MAIL_SERVICES: result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." if isCalendar and not any(e.isFolder for e in entries): result += "\n\nThese are calendar event summaries with date/time. You do NOT need to download individual events — this listing already contains subject, start, end, and location. Use the filter parameter with a date range (e.g. '2026-06') for specific periods." + if isContact and not any(e.isFolder for e in entries): + result += "\n\nThese are contacts with name, email, phone and company shown inline. You do NOT need to download a vCard for each contact — only download when you need the full record." + if isClickup and any((e.metadata or {}).get("cuType") == "task" for e in entries): + result += "\n\nThese are ClickUp tasks with status, assignee and due-date shown inline. Only download a task (JSON) when you need its full description, comments or custom fields." return ToolResult(toolCallId="", toolName="browseDataSource", success=True, data=result) except Exception as e: return ToolResult(toolCallId="", toolName="browseDataSource", success=False, error=str(e)) @@ -173,6 +268,9 @@ def _registerDataSourceTools(registry: ToolRegistry, services): if not entries: return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data="No results found.") isCalendar = service in _CALENDAR_SERVICES + isMail = service in _MAIL_SERVICES + isContact = service in _CONTACT_SERVICES + isClickup = service in _CLICKUP_SERVICES lines = [] for e in entries: if isCalendar and e.metadata: @@ -181,18 +279,24 @@ def _registerDataSourceTools(registry: ToolRegistry, services): loc = e.metadata.get("location") or "" locStr = f" 📍 {loc}" if loc else "" lines.append(f"- 📅 {start} – {end} {e.name}{locStr}") + elif isMail and not e.isFolder: + lines.append(_formatMailLine(e)) + elif isContact and not e.isFolder: + lines.append(_formatContactLine(e)) + elif isClickup and not e.isFolder and (e.metadata or {}).get("cuType") == "task": + lines.append(_formatTaskLine(e)) else: lines.append(f"- {e.name} (path: {e.path})") result = "\n".join(lines) - countLine = f"\n\n({len(entries)} entries returned" - if limit is not None: - countLine += f", limit={limit}" - countLine += ")" - result += countLine + result += _buildCountLine(entries, limit) if service in _MAIL_SERVICES: result += "\n\nIMPORTANT: These are email subjects only. To read the full email content, use downloadFromDataSource with the path, then readFile on the returned file ID." if isCalendar: result += "\n\nThese are calendar event summaries. You do NOT need to download individual events — subject, start, end, and location are shown above. For date-specific queries, use a date range as query (e.g. '2026-06')." + if isContact: + result += "\n\nThese are contacts with name, email, phone and company shown inline. You do NOT need to download a vCard for each contact — only download when you need the full record." + if isClickup: + result += "\n\nThese are ClickUp tasks with status, assignee and due-date shown inline. Only download a task (JSON) when you need its full description, comments or custom fields." return ToolResult(toolCallId="", toolName="searchDataSource", success=True, data=result) except Exception as e: return ToolResult(toolCallId="", toolName="searchDataSource", success=False, error=str(e)) @@ -295,15 +399,20 @@ def _registerDataSourceTools(registry: ToolRegistry, services): registry.register( "browseDataSource", _browseDataSource, description=( - "Browse files, folders, or emails in a data source. Accepts either:\n" + "List the contents of a folder/directory in a data source. Accepts either:\n" "- dataSourceId (for attached data sources shown in the prompt), OR\n" "- connectionId + service (for direct connection access via listConnections).\n" "\n" - "DEFAULT BEHAVIOUR: omit `limit` to get the connector's full default page. " - "For mail folders (Outlook/Gmail) the default returns up to 100 newest " - "messages -- DO NOT pass a smaller limit just to be safe; users almost " - "always want the full default page or explicitly more. Only set `limit` " - "when the user asks for a specific number (e.g. 'show me the latest 5 mails')." + "WHEN TO USE: Use browse to enumerate a directory's contents, or to get the " + "newest items of a mail/calendar folder. For TARGETED queries (find emails " + "from a person, files about a topic, events in a date range), prefer " + "searchDataSource -- it queries the source server-side and avoids listing/" + "downloading large amounts of irrelevant data.\n" + "\n" + "For calendar folders, pass a date range via `filter` (e.g. '2026-06' or " + "'2026-06-01 2026-06-30') so only that period is fetched. " + "Omit `limit` for the connector default; set it only when the user asks for a " + "specific count or you need MORE after hitting the default." ), parameters={ "type": "object", @@ -334,8 +443,21 @@ def _registerDataSourceTools(registry: ToolRegistry, services): registry.register( "searchDataSource", _searchDataSource, description=( - "Search for files within a data source. Accepts either dataSourceId OR connectionId+service. " - "Use the `limit` parameter to control how many hits are returned." + "PRIMARY tool for finding specific data in a source. The query runs " + "server-side, so this is far more efficient than browsing + downloading -- " + "ALWAYS prefer search when the user is looking for something specific " + "(a topic, a sender, a date range, a keyword). NEVER browse and download an " + "entire large source to find a few items; search first, then download ONLY " + "the matching results.\n" + "\n" + "Per-service query syntax:\n" + "- Outlook (KQL): 'from:alice subject:budget', 'received>=2026-05-01'.\n" + "- Gmail: 'from:alice after:2026/05/01 before:2026/06/01 budget'.\n" + "- SharePoint/OneDrive: free-text, searches file names AND content.\n" + "- Google Drive: searches file names and content.\n" + "- Calendar: pass a date range (e.g. '2026-06' or '2026-06-01 2026-06-30').\n" + "- ClickUp: task name/description keywords.\n" + "Accepts either dataSourceId OR connectionId+service." ), parameters={ "type": "object", @@ -344,7 +466,7 @@ def _registerDataSourceTools(registry: ToolRegistry, services): "connectionId": {"type": "string", "description": "UserConnection ID (alternative to dataSourceId)"}, "service": {"type": "string", "description": "Service name (alternative to dataSourceId)"}, "path": {"type": "string", "description": "Scope path (used with connectionId+service)"}, - "query": {"type": "string", "description": "Search query"}, + "query": {"type": "string", "description": "Search query (use the per-service syntax in the tool description)"}, "limit": { "type": "integer", "description": "Maximum number of search results (default ~100, max 1000).", diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py index 7249e959..12732a4b 100644 --- a/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_featureSubAgentTools.py @@ -157,6 +157,28 @@ def _registerFeatureSubAgentTools(registry: ToolRegistry, services): success=False, error=f"No data tables available for feature '{featureCode}'", ) + # A2: build the per-table type/inheritance-aware neutralization policy. + # tableActive = effective (own or inherited) table-level neutralize flag; + # explicitFields = fields whose neutralize flag is set explicitly. + from modules.serviceCenter.services.serviceKnowledge._inheritFlags import resolveEffectiveForFds + neutralizePolicy: Dict[str, Dict[str, Any]] = {} + for tblObj in selectedTables: + tn = tblObj.get("meta", {}).get("table", "") if isinstance(tblObj, dict) else "" + if not tn: + continue + eff = resolveEffectiveForFds(featureInstanceId, tn, None, _fdsAll, mode="walk") + tableActive = eff.get("effectiveNeutralize") is True + explicitFields = set(neutralizeFieldsPerTable.get(tn, [])) + if tableActive or explicitFields: + neutralizePolicy[tn] = {"tableActive": tableActive, "explicitFields": explicitFields} + + neutralizationService = services.getService("neutralization") if hasattr(services, "getService") else None + if neutralizationService is not None and not getattr(neutralizationService, "interfaceDbComponent", None): + try: + neutralizationService.interfaceDbComponent = services.chat.interfaceDbComponent + except Exception: + pass + cacheKey = f"{featureInstanceId}:{hashlib.md5(question.encode()).hexdigest()}" if cacheKey in _featureQueryCache: cachedAt, cachedResult = _featureQueryCache[cacheKey] @@ -202,7 +224,8 @@ def _registerFeatureSubAgentTools(registry: ToolRegistry, services): instanceLabel=instanceLabel, tableFilters=tableFilters, requestLang=requestLang, - neutralizeFields=neutralizeFieldsPerTable if neutralizeFieldsPerTable else None, + neutralizePolicy=neutralizePolicy if neutralizePolicy else None, + neutralizationService=neutralizationService, maxRounds=parentMaxRounds, maxCostCHF=parentMaxCostCHF, ) diff --git a/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py b/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py index 79e53092..5d377bc7 100644 --- a/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py +++ b/modules/serviceCenter/services/serviceAgent/coreTools/_mediaTools.py @@ -115,6 +115,31 @@ def _registerMediaTools(registry: ToolRegistry, services): logger.warning(f"renderDocument: knowledge service unavailable: {e}") resolvedImages = 0 + # Large-document path: for binary doc formats (pdf/docx) we keep block + # images as fileId references and let the renderer fetch the bytes lazily + # (one image at a time) instead of embedding every image's base64 in the + # JSON. Inline images and other formats keep the eager pre-resolution. + lazyBlockImages = outputFormat.strip().lower() in ("pdf", "docx", "doc") + + def _imageBytesResolver(fileId: str): + """Lazy resolver passed to the renderer: fileId -> raw image bytes.""" + if not fileId: + return None + if knowledgeService: + try: + chunks = knowledgeService._knowledgeDb.getContentChunks(fileId) + imageChunks = [c for c in (chunks or []) if c.get("contentType") == "image"] + if imageChunks and imageChunks[0].get("data"): + import base64 as _b64 + return _b64.b64decode(imageChunks[0]["data"]) + except Exception as e: + logger.warning(f"renderDocument: lazy knowledge image fetch failed for {fileId}: {e}") + try: + return services.chat.getFileData(fileId) + except Exception as e: + logger.warning(f"renderDocument: lazy file image fetch failed for {fileId}: {e}") + return None + def _resolveImageRef(targetObj, fileRefKey="_fileRef", fileIdKey="fileId"): """Resolve a single image reference dict to base64Data in-place.""" nonlocal resolvedImages @@ -153,6 +178,10 @@ def _registerMediaTools(registry: ToolRegistry, services): cType = section.get("content_type") # Block-level image sections if cType == "image": + # For pdf/docx, defer to lazy renderer-side resolution: keep + # the fileId reference, do not embed base64 into the JSON. + if lazyBlockImages: + continue for element in section.get("elements", []): contentObj = element.get("content", {}) _resolveImageRef(contentObj) @@ -195,6 +224,8 @@ def _registerMediaTools(registry: ToolRegistry, services): title=title, userPrompt=content, style=args.get("style"), + documentTheme=args.get("documentTheme"), + imageResolver=_imageBytesResolver if lazyBlockImages else None, ) if not documents: @@ -262,6 +293,10 @@ def _registerMediaTools(registry: ToolRegistry, services): "For long documents: write markdown with writeFile (mode=create then append chunks), then call this tool with " "`sourceFileId` only (tiny JSON — avoids model output truncation). For short docs you may pass `content` inline. " "Images: ![alt text](file:fileId) in the markdown. " + "Layout primitives (PDF/DOCX): a fenced ```cover_page block with `title:`/`subtitle:`/`author:`/`date:`/`logo: file:ID` " + "lines renders a centered title page (e.g. legal filing / report front page); a fenced ```image_grid block with an " + "optional `columns: N` line followed by image refs (`![alt](file:ID)` or `file:ID`, one per line) renders an N-column " + "image arrangement (marketing layouts). " "Each rendered file's result line contains `file id: ` (for embeds / readFile) AND " "`doc ref: docItem:` -- pass the latter inside `documentList` of subsequent " "`ai_process` / `ai_summarizeDocument` / `context_extractContent` calls." @@ -280,6 +315,17 @@ def _registerMediaTools(registry: ToolRegistry, services): "outputFormat": {"type": "string", "description": "Target format: pdf, docx, xlsx, pptx, csv, html, md, json, txt", "default": "pdf"}, "title": {"type": "string", "description": "Document title", "default": "Document"}, "language": {"type": "string", "description": "Document language (ISO 639-1)", "default": "de"}, + "documentTheme": { + "type": "string", + "enum": ["general", "finance", "legal", "technical", "hr", "marketing"], + "description": ( + "Named style preset applied by the renderer (colors, fonts, spacing). " + "Pick the one that matches the document purpose: 'legal' for serif/justified " + "legal filings, 'marketing' for bold image-friendly layouts, 'finance', " + "'technical', 'hr', or 'general' (default). The explicit 'style' object, if " + "provided, overrides individual preset keys." + ), + }, "style": { "type": "object", "description": ( @@ -840,6 +886,88 @@ def _registerMediaTools(registry: ToolRegistry, services): except Exception as e: return ToolResult(toolCallId="", toolName="neutralizeData", success=False, error=str(e)) + async def _revealDocument(args: Dict[str, Any], context: Dict[str, Any]): + """De-neutralize (reveal) placeholder text back to original values for a one-time download. + + PRIVACY: The cleartext is delivered to the user's browser as a transient download + only (SSE side-event). It is NEVER saved as a file, indexed into the knowledge base, + or written back into chat history. The persisted tool result contains only a + confirmation -- not the revealed cleartext. Resolution uses ONLY the private local + placeholder mapping (no external LLM). + """ + import base64 as _b64 + import re as _re + text = args.get("text", "") + fileId = (args.get("fileId") or "").strip() + fileName = (args.get("fileName") or "").strip() + if not isinstance(text, str): + text = str(text) if text is not None else "" + if not text and not fileId: + return ToolResult(toolCallId="", toolName="revealDocument", success=False, + error="text or fileId is required") + try: + neutralizationService = services.getService("neutralization") if hasattr(services, "getService") else None + if not neutralizationService or not hasattr(neutralizationService, "resolveText"): + return ToolResult(toolCallId="", toolName="revealDocument", success=False, + error="Neutralization service not available") + if not getattr(neutralizationService, "interfaceDbComponent", None): + neutralizationService.interfaceDbComponent = services.chat.interfaceDbComponent + + if fileId and not text: + dbMgmt = services.chat.interfaceDbComponent + fileRow = dbMgmt.getFile(fileId) + if not fileRow: + return ToolResult(toolCallId="", toolName="revealDocument", success=False, + error=f"fileId not found: {fileId}") + rawBytes = dbMgmt.getFileData(fileId) + if not rawBytes: + return ToolResult(toolCallId="", toolName="revealDocument", success=False, + error="File data not accessible") + decoded = None + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + decoded = rawBytes.decode(encoding) + break + except (UnicodeDecodeError, ValueError): + continue + if decoded is None: + return ToolResult(toolCallId="", toolName="revealDocument", success=False, + error="File is binary or could not be decoded as text; reveal only supports text content") + text = decoded + if not fileName: + info = fileRow if isinstance(fileRow, dict) else None + fileName = (info.get("fileName") if info else None) or f"{fileId}.txt" + + # Resolve placeholders locally (private mapping, no LLM). Count for the audit message. + placeholderCount = len(_re.findall(r'\[[a-z]+\.[a-f0-9-]{36}\]', text)) + revealed = neutralizationService.resolveText(text) + + if not fileName: + fileName = "revealed.txt" + mimeType = "text/markdown" if fileName.lower().endswith((".md", ".markdown")) else "text/plain" + contentB64 = _b64.b64encode(revealed.encode("utf-8")).decode("ascii") + + return ToolResult( + toolCallId="", toolName="revealDocument", success=True, + data=( + f"Revealed {placeholderCount} placeholder(s) and prepared '{fileName}' for " + f"download in the chat. The cleartext was NOT stored, indexed, or kept in history." + ), + sideEvents=[{ + "type": "revealDownload", + "data": { + "content": contentB64, + "encoding": "base64", + "fileName": fileName, + "mimeType": mimeType, + "placeholderCount": placeholderCount, + }, + }], + ) + except Exception as e: + logger.error(f"revealDocument failed: {e}") + return ToolResult(toolCallId="", toolName="revealDocument", success=False, error=str(e)) + async def _executeCode(args: Dict[str, Any], context: Dict[str, Any]): code = args.get("code", "") language = args.get("language", "python") @@ -899,6 +1027,28 @@ def _registerMediaTools(registry: ToolRegistry, services): readOnly=True ) + registry.register( + "revealDocument", _revealDocument, + description=( + "De-neutralize (reveal) a neutralized text/document by replacing placeholders like " + "[name.] with their original values, using ONLY the private local mapping (no " + "external LLM). The cleartext is delivered to the user as a transient, one-time download " + "in the chat -- it is NEVER saved, indexed, or written to chat history. Use ONLY when the " + "user explicitly asks to download the real/original (de-anonymized) version of a document. " + "Provide either 'fileId' (a stored neutralized text file) or inline 'text'." + ), + parameters={ + "type": "object", + "properties": { + "fileId": {"type": "string", "description": "ID of a stored text file containing placeholders to reveal"}, + "text": {"type": "string", "description": "Inline placeholder text to reveal (alternative to fileId)"}, + "fileName": {"type": "string", "description": "Optional download file name (e.g. 'contract-original.md')"}, + }, + }, + readOnly=True, + displayLabel="preparing de-anonymized download", + ) + from modules.serviceCenter.services.serviceAgent.sandboxExecutor import SANDBOX_ALLOWED_MODULES moduleList = ", ".join(sorted(SANDBOX_ALLOWED_MODULES | {"io"})) registry.register( diff --git a/modules/serviceCenter/services/serviceAgent/datamodelAgent.py b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py index c96265e4..25099bf8 100644 --- a/modules/serviceCenter/services/serviceAgent/datamodelAgent.py +++ b/modules/serviceCenter/services/serviceAgent/datamodelAgent.py @@ -33,6 +33,7 @@ class AgentEventTypeEnum(str, Enum): FILE_EDIT_REJECTED = "fileEditRejected" DATA_SOURCE_ACCESS = "dataSourceAccess" VOICE_RESPONSE = "voiceResponse" + REVEAL_DOWNLOAD = "revealDownload" FINAL = "final" ERROR = "error" diff --git a/modules/serviceCenter/services/serviceAgent/featureDataAgent.py b/modules/serviceCenter/services/serviceAgent/featureDataAgent.py index 51840575..117645dc 100644 --- a/modules/serviceCenter/services/serviceAgent/featureDataAgent.py +++ b/modules/serviceCenter/services/serviceAgent/featureDataAgent.py @@ -57,6 +57,8 @@ async def runFeatureDataAgent( tableFilters: Optional[Dict[str, Dict[str, str]]] = None, requestLang: Optional[str] = None, neutralizeFields: Optional[Dict[str, List[str]]] = None, + neutralizePolicy: Optional[Dict[str, Dict[str, Any]]] = None, + neutralizationService: Optional[Any] = None, maxRounds: Optional[int] = None, maxCostCHF: Optional[float] = None, ) -> str: @@ -74,8 +76,13 @@ async def runFeatureDataAgent( instanceLabel: Human-readable instance name for context. tableFilters: Per-table record filters from FeatureDataSource.recordFilter. requestLang: ISO 639-1 code for resolving multilingual table labels in the schema prompt. - neutralizeFields: Per-table list of field names to mask with placeholders - before returning data to the AI. + neutralizeFields: LEGACY per-table list of field names for whole-value masking. + neutralizePolicy: Per-table type/inheritance-aware neutralization policy + ({"tableActive": bool, "explicitFields": set}) applied via the provider's + finalizeRowsAsync (A2 rules: strings substring-neutralized when effective, + binary dropped, other scalars only when explicit). + neutralizationService: Mandate/instance-scoped NeutralizationService used for + substring neutralization of string cells. maxRounds: Inherited from the parent agent's configured `maxRounds` (workspace user setting `maxAgentRounds` -> `AgentConfig.maxRounds`). Falls back to the legacy 8-round default when not provided so direct @@ -87,7 +94,12 @@ async def runFeatureDataAgent( Plain-text answer produced by the sub-agent. """ - provider = FeatureDataProvider(dbConnector, neutralizeFields=neutralizeFields) + provider = FeatureDataProvider( + dbConnector, + neutralizeFields=neutralizeFields, + neutralizePolicy=neutralizePolicy, + neutralizationService=neutralizationService, + ) validator = _buildValidatorForFeature(featureCode) registry = _buildSubAgentTools(provider, featureInstanceId, mandateId, tableFilters or {}, validator=validator) @@ -207,6 +219,8 @@ def _buildSubAgentTools( offset=offset, extraFilters=_recordFilterToList(tableName), ) + if hasattr(provider, "finalizeRowsAsync") and "rows" in result: + result["rows"] = await provider.finalizeRowsAsync(tableName, result["rows"]) return ToolResult( toolCallId="", toolName="browseTable", success="error" not in result, @@ -237,6 +251,8 @@ def _buildSubAgentTools( offset=offset, extraFilters=_recordFilterToList(tableName), ) + if hasattr(provider, "finalizeRowsAsync") and "rows" in result: + result["rows"] = await provider.finalizeRowsAsync(tableName, result["rows"]) return ToolResult( toolCallId="", toolName="queryTable", success="error" not in result, @@ -271,6 +287,8 @@ def _buildSubAgentTools( groupBy=groupBy, extraFilters=combinedFilters or None, ) + if hasattr(provider, "finalizeRowsAsync") and "rows" in result: + result["rows"] = await provider.finalizeRowsAsync(tableName, result["rows"]) return ToolResult( toolCallId="", toolName="aggregateTable", success="error" not in result, diff --git a/modules/serviceCenter/services/serviceAgent/featureDataProvider.py b/modules/serviceCenter/services/serviceAgent/featureDataProvider.py index 27ec36b2..eec9fcca 100644 --- a/modules/serviceCenter/services/serviceAgent/featureDataProvider.py +++ b/modules/serviceCenter/services/serviceAgent/featureDataProvider.py @@ -8,6 +8,7 @@ feature table. All queries are automatically filtered by featureInstanceId and mandateId so data isolation is guaranteed. """ +import asyncio import hashlib import logging import json @@ -62,18 +63,36 @@ _ALLOWED_AGGREGATES = {"SUM", "COUNT", "AVG", "MIN", "MAX"} class FeatureDataProvider: """Reads feature-instance data from the DB using DATA_OBJECTS metadata.""" - def __init__(self, dbConnector, neutralizeFields: Optional[Dict[str, List[str]]] = None): + def __init__( + self, + dbConnector, + neutralizeFields: Optional[Dict[str, List[str]]] = None, + neutralizePolicy: Optional[Dict[str, Dict[str, Any]]] = None, + neutralizationService: Optional[Any] = None, + ): """ Args: dbConnector: A connectorDbPostgre.DatabaseConnector with an open connection. - neutralizeFields: Per-table field names whose values must be replaced - with placeholders before returning to the AI, e.g. - ``{"TrusteePosition": ["firstName", "lastName", "address"]}``. + neutralizeFields: LEGACY per-table field names whose values are replaced + with a whole-value placeholder ``[NEUT..]``. Kept for + backward compatibility; superseded by ``neutralizePolicy``. + neutralizePolicy: Per-table type/inheritance-aware policy, e.g. + ``{"TrusteePosition": {"tableActive": True, "explicitFields": {"iban"}}}``. + * ``tableActive`` -- effective (own/inherited) table-level neutralize flag. + * ``explicitFields`` -- fields whose neutralize flag is set EXPLICITLY. + Applied via :meth:`finalizeRowsAsync` following the A2 rules: + strings substring-neutralized when effective (explicit or inherited), + binary dropped, other scalars only when explicit. + neutralizationService: The mandate/instance-scoped NeutralizationService + used for substring neutralization of string cells (reuses the standard + neutralization engine; no external LLM is introduced here). """ self._db = dbConnector self._neutralizeFields: Dict[str, Set[str]] = { tbl: set(fields) for tbl, fields in (neutralizeFields or {}).items() } + self._neutralizePolicy: Dict[str, Dict[str, Any]] = neutralizePolicy or {} + self._neutralizer = neutralizationService # ------------------------------------------------------------------ # public API (called by FeatureDataAgent tools) @@ -108,12 +127,27 @@ class FeatureDataProvider: logger.warning(f"getActualColumns({tableName}) failed: {e}") return [] - def _applyFieldNeutralization(self, tableName: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Neutralize sensitive field values in query results before they reach the AI.""" - fieldsToNeut = self._neutralizeFields.get(tableName) - if not fieldsToNeut: - return rows - return [_neutralizeRowFields(row, fieldsToNeut) for row in rows] + async def finalizeRowsAsync(self, tableName: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Make raw DB rows AI-safe: apply the field-neutralization policy and JSON-serialize. + + The query methods (``browseTable``/``queryTable``/``aggregateTable``) return RAW + rows so this step can see the real Python types (bytes vs str vs scalar). + + * When a rich ``neutralizePolicy`` exists for the table, the A2 type/inheritance + rules apply (see :func:`_neutralizeAndSerializeRows`). + * Otherwise the legacy whole-value behavior is preserved (``neutralizeFields``). + * With no neutralization at all, rows are just JSON-serialized. + + Always returns JSON-serializable rows. + """ + policy = self._neutralizePolicy.get(tableName) + if policy: + return await _neutralizeAndSerializeRows(rows, policy, self._neutralizer) + serialized = [_serializeRow(dict(r)) for r in rows] + legacyFields = self._neutralizeFields.get(tableName) + if legacyFields: + serialized = [_neutralizeRowFields(row, legacyFields) for row in serialized] + return serialized def browseTable( self, @@ -162,9 +196,10 @@ class FeatureDataProvider: f'ORDER BY "id" LIMIT %s OFFSET %s' ) cur.execute(dataSql, allParams + [limit, offset]) - rows = [_serializeRow(dict(r)) for r in cur.fetchall()] + # Return RAW rows; neutralization + JSON-serialization happen in + # finalizeRowsAsync (needs the real Python types to apply A2 rules). + rows = [dict(r) for r in cur.fetchall()] - rows = self._applyFieldNeutralization(tableName, rows) result = {"rows": rows, "total": total, "limit": limit, "offset": offset} _debugQueryLog("browseTable", tableName, { "fields": fields, "limit": limit, "offset": offset, @@ -226,9 +261,8 @@ class FeatureDataProvider: f'FROM "{tableName}" WHERE {fullWhere}' ) cur.execute(sql, allParams) - rows = [_serializeRow(dict(r)) for r in cur.fetchall()] + rows = [dict(r) for r in cur.fetchall()] - rows = self._applyFieldNeutralization(tableName, rows) result = { "rows": rows, "aggregate": aggregate, @@ -300,9 +334,8 @@ class FeatureDataProvider: f'WHERE {fullWhere} {orderClause} LIMIT %s OFFSET %s' ) cur.execute(dataSql, allParams + [limit, offset]) - rows = [_serializeRow(dict(r)) for r in cur.fetchall()] + rows = [dict(r) for r in cur.fetchall()] - rows = self._applyFieldNeutralization(tableName, rows) result = {"rows": rows, "total": total, "limit": limit, "offset": offset} _debugQueryLog("queryTable", tableName, { "filters": filters, "fields": fields, "orderBy": orderBy, @@ -437,3 +470,142 @@ def _neutralizeRowFields(row: Dict[str, Any], fieldsToNeutralize: Set[str]) -> D shortHash = hashlib.sha256(str(val).encode()).hexdigest()[:8] row[field] = f"[{_PLACEHOLDER_PREFIX}.{field}.{shortHash}]" return row + + +# ------------------------------------------------------------------ +# A2: type / inheritance-aware field neutralization for source data +# ------------------------------------------------------------------ +# +# Rules (see wiki neutralization.md Failsafe 5/6): +# 1. STRING (incl. JSON/markdown/code -- anything textual): substring-neutralize +# via the private NeutralizationService whenever neutralize is EFFECTIVE for the +# field (explicit OR inherited). The placeholders stay embedded in the text so the +# record remains usable; the field name is passed as a type hint. +# 2. BINARY (bytes): never neutralized -- the column is DROPPED when neutralization +# applies to the table/field. +# 3. OTHER SCALARS (number/float/int/date/bool): neutralized (whole-value placeholder) +# ONLY when the field flag is set EXPLICITLY -- never via inheritance. + +_NEUT_CONCURRENCY = 4 + + +def _isStructuralField(key: str) -> bool: + """Identifiers / system columns are references, not PII content -- never neutralized. + + Excludes primary/foreign keys and audit columns so neutralization never corrupts + record references and never wastes an LLM call on a UUID/enum value. + """ + if key.startswith("_") or key.startswith("sys"): + return True + if key == "id" or key.endswith("Id") or key.endswith("_id"): + return True + return key in ("mandateId", "featureInstanceId", "instanceId", "createdBy", "updatedBy") + + +def _isTextValue(value: Any) -> bool: + """True for values that should be treated as neutralizable text (str/JSON-ish).""" + return isinstance(value, str) or isinstance(value, (dict, list)) + + +async def _neutralizeOneText(fieldName: str, text: str, neutralizer: Any) -> Optional[str]: + """Substring-neutralize a single text value, using the field name as a type hint. + + The hint (``": "``) is prepended so the private LLM can infer the entity + type for short PII columns, then stripped from the result. On any prefix mismatch + or error the cell is fail-safe redacted (never returns the raw value). + """ + prefix = f"{fieldName}: " + try: + result = await neutralizer.processTextAsync(prefix + text) + except Exception as e: # noqa: BLE001 - neutralization must fail closed + logger.warning("field neutralization failed for '%s': %s", fieldName, e) + return "[REDACTED]" + out = result.get("neutralized_text") if isinstance(result, dict) else None + if not isinstance(out, str): + return "[REDACTED]" + if out.startswith(prefix): + return out[len(prefix):] + # Engine altered the hint prefix (rare) -- fail closed rather than leak. + logger.warning("field neutralization prefix mismatch for '%s'; redacting", fieldName) + return "[REDACTED]" + + +async def _neutralizeAndSerializeRows( + rows: List[Dict[str, Any]], + policy: Dict[str, Any], + neutralizer: Any, +) -> List[Dict[str, Any]]: + """Apply the A2 field-neutralization rules to raw rows and JSON-serialize them.""" + tableActive = bool(policy.get("tableActive")) + explicitFields: Set[str] = set(policy.get("explicitFields") or []) + + outRows: List[Dict[str, Any]] = [] + # (fieldName, originalText) -> neutralizedText (dedup across the whole result set) + pending: Dict[tuple, Optional[str]] = {} + cellRefs: List[tuple] = [] # (rowIdx, key, fieldName, originalText) + + for row in rows: + out: Dict[str, Any] = {} + for key, value in row.items(): + fieldExplicit = key in explicitFields + fieldEffective = fieldExplicit or tableActive + + if value is None: + out[key] = None + continue + + # Identifiers / system columns: serialize but never neutralize. + if _isStructuralField(key): + out[key] = value.isoformat() if hasattr(value, "isoformat") else ( + f"" if isinstance(value, (bytes, bytearray)) else value + ) + continue + + if isinstance(value, (bytes, bytearray)): + # Rule 2: binary is dropped when neutralization applies; else legacy marker. + if tableActive or fieldExplicit: + continue + out[key] = f"" + continue + + if _isTextValue(value): + textVal = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + if fieldEffective and textVal != "": + pending.setdefault((key, textVal), None) + cellRefs.append((len(outRows), key, key, textVal)) + out[key] = textVal + continue + + # Rule 3: other scalars (number/float/int/date/bool) -- explicit only. + serialized = value.isoformat() if hasattr(value, "isoformat") else value + if fieldExplicit: + shortHash = hashlib.sha256(str(value).encode()).hexdigest()[:8] + out[key] = f"[{_PLACEHOLDER_PREFIX}.{key}.{shortHash}]" + else: + out[key] = serialized + outRows.append(out) + + if not cellRefs: + return outRows + + if neutralizer is None or not hasattr(neutralizer, "processTextAsync"): + # Fail-safe: neutralization required but no engine -> redact the affected cells. + for rowIdx, key, _fieldName, _origText in cellRefs: + outRows[rowIdx][key] = "[REDACTED]" + return outRows + + sem = asyncio.Semaphore(_NEUT_CONCURRENCY) + + async def _resolvePair(fieldName: str, origText: str) -> None: + async with sem: + pending[(fieldName, origText)] = await _neutralizeOneText(fieldName, origText, neutralizer) + + await asyncio.gather(*[ + _resolvePair(fieldName, origText) for (fieldName, origText) in pending.keys() + ]) + + for rowIdx, key, fieldName, origText in cellRefs: + neutralized = pending.get((fieldName, origText)) + if neutralized is not None: + outRows[rowIdx][key] = neutralized + return outRows diff --git a/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py b/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py index dd8c80ec..bcfaff26 100644 --- a/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py +++ b/modules/serviceCenter/services/serviceAgent/toolboxRegistry.py @@ -231,6 +231,22 @@ def _registerDefaultToolboxes() -> None: "trustee_refreshAccountingData", ], ), + ToolboxDefinition( + id="neutralization", + label="Neutralization / Reveal", + description=( + "Privacy-sensitive de-neutralization. NOT active by default - must be " + "explicitly requested. Contains revealDocument, which resolves " + "neutralization placeholders ([type.uuid]) back to cleartext using ONLY " + "the local mapping (no external LLM) and returns the result as a " + "transient one-time download. Cleartext is never saved, indexed, or kept " + "in the chat history." + ), + isDefault=False, + tools=[ + "revealDocument", + ], + ), ] for tb in defaults: _toolboxRegistry.registerToolbox(tb) diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index d4e5ccdb..d5054921 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -160,8 +160,11 @@ class AiService: 3. billingCallback on aiObjects: records one billing transaction per model call with exact provider + model name (set before AI call, invoked by _callWithModel) - NEUTRALIZATION: If enabled, prompt text is neutralized before the AI call - and placeholders in the response are rehydrated afterwards. + NEUTRALIZATION: If enabled, prompt text is neutralized before the AI call. + The response is persisted exactly as returned by the model (placeholders are + NOT rehydrated/re-saved with cleartext -- that would defeat neutralization). + De-neutralization for download is an explicit, transient action via the + agent's `revealDocument` tool (no save/index). """ await self.ensureAiObjectsInitialized() @@ -241,7 +244,9 @@ class AiService: """Streaming variant of callAi. Yields str deltas during generation, then final AiCallResponse. NEUTRALIZATION: If enabled, prompt text is neutralized before streaming. - Rehydration happens on the final AiCallResponse (not on individual str deltas). + The streamed/persisted response keeps placeholders as returned by the model + (no cleartext re-hydration into storage). Use the agent's `revealDocument` + tool for an explicit, transient de-neutralization for download. """ await self.ensureAiObjectsInitialized() @@ -623,7 +628,7 @@ detectedIntent-Werte: return basePrompt # ========================================================================= - # NEUTRALIZATION: Centralized prompt neutralization / response rehydration + # NEUTRALIZATION: Centralized prompt neutralization (no response rehydration) # ========================================================================= async def _hasNeutralizationModel(self) -> bool: @@ -920,20 +925,6 @@ detectedIntent-Werte: logger.info(f"_neutralizeRequest complete: neutralized={_wasNeutralized}, excluded={len(excludedDocs)}") return request, _wasNeutralized, excludedDocs - def _rehydrateResponse(self, responseText: str) -> str: - """Replace neutralization placeholders with original values in AI response.""" - if not responseText: - return responseText - try: - neutralSvc = self._get_service("neutralization") - if not neutralSvc or not hasattr(neutralSvc, 'resolveText'): - return responseText - resolved = neutralSvc.resolveText(responseText) - return resolved if resolved else responseText - except Exception as e: - logger.warning(f"Response rehydration failed: {e}") - return responseText - def _preflightBillingCheck(self) -> None: """ Pre-flight billing validation - like a 0 CHF credit card authorization check. @@ -1689,7 +1680,8 @@ Respond with ONLY a JSON object in this exact format: language: str, title: str, userPrompt: str, - parentOperationId: str + parentOperationId: str, + documentTheme: Optional[str] = None ) -> List[RenderedDocument]: """ Phase 5E: Rendert gefüllte Struktur zum Ziel-Format. @@ -1741,7 +1733,8 @@ Respond with ONLY a JSON object in this exact format: title, userPrompt, self, - parentOperationId=renderOperationId # Parent-Referenz für ChatLog-Hierarchie + parentOperationId=renderOperationId, # Parent-Referenz für ChatLog-Hierarchie + documentTheme=documentTheme ) # ChatLog abschließen @@ -1783,7 +1776,8 @@ Respond with ONLY a JSON object in this exact format: outputFormat: Optional[str] = None, title: Optional[str] = None, parentOperationId: Optional[str] = None, - generationIntent: Optional[str] = None # NEW: Explicit intent from action (skips detection) + generationIntent: Optional[str] = None, # NEW: Explicit intent from action (skips detection) + documentTheme: Optional[str] = None # Named style preset for document rendering ) -> AiResponse: """ Unified AI content generation with explicit intent requirement. @@ -1802,6 +1796,8 @@ Respond with ONLY a JSON object in this exact format: parentOperationId: Optional parent operation ID for hierarchical logging generationIntent: REQUIRED explicit intent ("document" | "code" | "image") from action. NO auto-detection - actions must explicitly specify intent. + documentTheme: Optional named style preset (general/finance/legal/technical/ + hr/marketing) forwarded to the renderer for document generation. Returns: AiResponse with content, metadata, and optional documents @@ -1872,7 +1868,8 @@ Respond with ONLY a JSON object in this exact format: contentParts=contentParts, outputFormat=outputFormat, title=title, - parentOperationId=parentOperationId + parentOperationId=parentOperationId, + documentTheme=documentTheme ) # DATA_EXTRACT: Extract content from documents and process with AI (no structure generation) @@ -2088,7 +2085,8 @@ Respond with ONLY a JSON object in this exact format: contentParts: Optional[List[ContentPart]], outputFormat: str, title: str, - parentOperationId: Optional[str] + parentOperationId: Optional[str], + documentTheme: Optional[str] = None ) -> AiResponse: """Handle document generation using document generation path.""" from modules.serviceCenter.services.serviceGeneration.paths.documentPath import DocumentGenerationPath @@ -2105,7 +2103,8 @@ Respond with ONLY a JSON object in this exact format: contentParts=contentParts, outputFormat=outputFormat, title=title or "Generated Document", - parentOperationId=parentOperationId + parentOperationId=parentOperationId, + documentTheme=documentTheme ) diff --git a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py index 90c9a347..794f7dec 100644 --- a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py +++ b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py @@ -26,7 +26,7 @@ from modules.interfaces.interfaceDbBilling import getInterface as getBillingInte logger = logging.getLogger(__name__) -# Markup percentage for internal pricing (+50% für Infrastruktur und Platform Service + 50% für Währungsrisiko ==> Faktor 2.0) +# Markup percentage on the AI base price (400% ==> Faktor 5.0: Infrastruktur, Platform Service, Währungsrisiko) BILLING_MARKUP_PERCENT = 400 # Singleton cache @@ -150,7 +150,7 @@ class BillingService: if basePriceCHF <= 0: return 0.0 - # Apply markup (50% = multiply by 1.5) + # Apply markup (400% = multiply by 5.0) markup_multiplier = 1 + (BILLING_MARKUP_PERCENT / 100) return round(basePriceCHF * markup_multiplier, 6) diff --git a/modules/serviceCenter/services/serviceGeneration/mainServiceGeneration.py b/modules/serviceCenter/services/serviceGeneration/mainServiceGeneration.py index 6afcc0a8..5bf86aee 100644 --- a/modules/serviceCenter/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/serviceCenter/services/serviceGeneration/mainServiceGeneration.py @@ -383,7 +383,7 @@ class GenerationService: 'workflowId': 'unknown' } - async def renderReport(self, extractedContent: Dict[str, Any], outputFormat: str, language: str, title: str, userPrompt: str = None, aiService=None, parentOperationId: Optional[str] = None, style: Optional[Dict[str, Any]] = None) -> List[RenderedDocument]: + async def renderReport(self, extractedContent: Dict[str, Any], outputFormat: str, language: str, title: str, userPrompt: str = None, aiService=None, parentOperationId: Optional[str] = None, style: Optional[Dict[str, Any]] = None, documentTheme: Optional[str] = None, imageResolver=None) -> List[RenderedDocument]: """ Render extracted JSON content to the specified output format. Processes EACH document separately and calls renderer for each. @@ -401,13 +401,20 @@ class GenerationService: aiService: AI service instance for generation prompt creation parentOperationId: Optional parent operation ID for hierarchical logging style: Optional style overrides (deep-merged with DEFAULT_STYLE) + documentTheme: Optional named theme preset (general/finance/legal/ + technical/hr/marketing). Resolved as DEFAULT_STYLE <- preset <- style, + so an explicit ``style`` override always wins. + imageResolver: Optional callable ``fileId -> bytes`` for lazy, on-demand + resolution of block images that carry only a ``fileId`` (no embedded + base64). Lets large documents avoid holding every image's bytes in + the JSON simultaneously. When None, images must be pre-embedded. Returns: List of RenderedDocument objects. Each RenderedDocument represents one rendered file (main document or supporting file) """ try: - resolvedStyle = resolveStyle(style) + resolvedStyle = resolveStyle(style, documentTheme) # Validate JSON input if not isinstance(extractedContent, dict): raise ValueError("extractedContent must be a JSON dictionary") @@ -451,6 +458,11 @@ class GenerationService: if not renderer: logger.warning(f"Unsupported format '{docFormat}' for document {doc.get('id', docIndex)}, skipping") continue + # Provide the per-render lazy image resolver (fileId -> bytes) so + # renderers can fetch block images on demand instead of relying on + # all bytes being pre-embedded in the document JSON. + if imageResolver is not None: + renderer._imageResolver = imageResolver # Check output style classification (code/document/image/etc.) from renderer from .renderers.registry import getOutputStyle diff --git a/modules/serviceCenter/services/serviceGeneration/paths/documentPath.py b/modules/serviceCenter/services/serviceGeneration/paths/documentPath.py index 4fc6c9d5..f7d146f5 100644 --- a/modules/serviceCenter/services/serviceGeneration/paths/documentPath.py +++ b/modules/serviceCenter/services/serviceGeneration/paths/documentPath.py @@ -34,7 +34,8 @@ class DocumentGenerationPath: contentParts: Optional[List[ContentPart]] = None, outputFormat: str = "txt", title: Optional[str] = None, - parentOperationId: Optional[str] = None + parentOperationId: Optional[str] = None, + documentTheme: Optional[str] = None ) -> AiResponse: """ Generate document using existing chapter/section model. @@ -165,7 +166,8 @@ class DocumentGenerationPath: language, # Global fallback (per-document language extracted from structure in renderReport) title or "Generated Document", userPrompt, - docOperationId + docOperationId, + documentTheme=documentTheme ) # Baue Response: Konvertiere alle gerenderten Dokumente zu DocumentData diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/documentRendererBaseTemplate.py b/modules/serviceCenter/services/serviceGeneration/renderers/documentRendererBaseTemplate.py index 52daae29..35401f27 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/documentRendererBaseTemplate.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/documentRendererBaseTemplate.py @@ -25,6 +25,42 @@ class BaseRenderer(ABC): def __init__(self, services=None): self.logger = logger self.services = services # Add services attribute + # Optional per-render lazy image resolver: fileId -> raw bytes. Set by + # GenerationService.renderReport. When present, block images that only + # carry a fileId (no embedded base64Data) are fetched on demand during + # rendering, so the document JSON never holds all image bytes at once. + self._imageResolver = None + + def _lazyResolveImageBase64(self, *sources) -> str: + """Resolve a fileId/_fileRef (found in any of the given dicts) to base64. + + Returns "" when no resolver is configured, no id is present, or the + fetch fails/empty. The raw bytes are held only transiently here; nothing + is written back into the source JSON, keeping peak memory ~ one image. + """ + resolver = getattr(self, "_imageResolver", None) + if not callable(resolver): + return "" + fileId = "" + for src in sources: + if isinstance(src, dict): + fileId = src.get("fileId") or src.get("_fileRef") or "" + if fileId: + break + if not fileId: + return "" + try: + rawBytes = resolver(fileId) + except Exception as e: # noqa: BLE001 - a single bad image must not abort the doc + self.logger.warning(f"lazy image resolve failed for fileId={fileId}: {e}") + return "" + if not rawBytes: + return "" + try: + return base64.b64encode(rawBytes).decode("ascii") + except Exception as e: # noqa: BLE001 + self.logger.warning(f"lazy image encode failed for fileId={fileId}: {e}") + return "" @classmethod def getSupportedFormats(cls) -> List[str]: diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererDocx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererDocx.py index ab37f756..37d191e1 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererDocx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererDocx.py @@ -158,8 +158,14 @@ class RendererDocx(BaseRenderer): # Fallback to metadata.title only if title parameter is empty document_title = title if title else metadata.get("title", "Generated Document") + # A cover_page section renders its own title page; skip the standalone + # title in that case so the title is not duplicated. + hasCoverPage = any( + (s.get("content_type") if isinstance(s, dict) else "") == "cover_page" + for s in sections + ) # Add document title using Title style - if document_title: + if document_title and not hasCoverPage: doc.add_paragraph(document_title, style='Title') # Process each section in order @@ -377,6 +383,10 @@ class RendererDocx(BaseRenderer): self._renderJsonCodeBlock(doc, element, styles) elif element_type == "image": self._renderJsonImage(doc, element, styles) + elif element_type == "cover_page" or section_type == "cover_page": + self._renderCoverPage(doc, element, styles) + elif element_type == "image_grid" or section_type == "image_grid": + self._renderImageGrid(doc, element, styles) else: # Fallback: if element_type not set, use section_type if section_type == "table": @@ -1030,6 +1040,89 @@ class RendererDocx(BaseRenderer): except Exception as e: self.logger.warning(f"Error rendering code block: {str(e)}") + def _imageStreamFromContent(self, content: Dict[str, Any]): + """Return a BytesIO of image bytes from base64Data or a lazily-resolved fileId. + + Returns None when nothing resolvable is present. Kept small so layout + primitives (cover/grid) share the same lazy-resolution path as block images. + """ + if not isinstance(content, dict): + return None + b64 = content.get("base64Data") or content.get("logoBase64") or "" + if not b64: + b64 = self._lazyResolveImageBase64(content) + if not b64: + return None + try: + return io.BytesIO(base64.b64decode(b64)) + except Exception: + return None + + def _renderCoverPage(self, doc: Document, element: Dict[str, Any], styles: Dict[str, Any]) -> None: + """Render a cover/title page in DOCX, ending with a page break.""" + try: + from docx.enum.text import WD_ALIGN_PARAGRAPH + content = element.get("content", element) if isinstance(element, dict) else {} + if not isinstance(content, dict): + content = {} + for _ in range(6): + doc.add_paragraph("") + logoStream = self._imageStreamFromContent(content) + if logoStream is not None: + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.add_run().add_picture(logoStream, width=Inches(2.0)) + title = (content.get("title") or "").strip() + if title: + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + run = p.add_run(title) + run.bold = True + run.font.size = Pt(28) + for key, sizePt in (("subtitle", 16), ("author", 12), ("date", 12)): + val = (content.get(key) or "").strip() + if not val: + continue + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.add_run(val).font.size = Pt(sizePt) + doc.add_page_break() + except Exception as e: + self.logger.warning(f"Error rendering cover_page: {e}") + + def _renderImageGrid(self, doc: Document, element: Dict[str, Any], styles: Dict[str, Any]) -> None: + """Render an image grid (N columns) as a borderless DOCX table of pictures.""" + try: + content = element.get("content", element) if isinstance(element, dict) else {} + if not isinstance(content, dict): + return + images = content.get("images") or [] + if not isinstance(images, list) or not images: + return + try: + columns = max(1, int(content.get("columns", 2))) + except (TypeError, ValueError): + columns = 2 + + streams = [s for s in (self._imageStreamFromContent(i) for i in images) if s is not None] + if not streams: + return + import math + nrows = math.ceil(len(streams) / columns) + table = doc.add_table(rows=nrows, cols=columns) + cellWidthInches = max(1.0, 6.5 / columns - 0.1) + for idx, stream in enumerate(streams): + r, c = divmod(idx, columns) + cell = table.cell(r, c) + para = cell.paragraphs[0] + try: + para.add_run().add_picture(stream, width=Inches(cellWidthInches)) + except Exception as ie: + self.logger.warning(f"image_grid cell failed: {ie}") + doc.add_paragraph("") + except Exception as e: + self.logger.warning(f"Error rendering image_grid: {e}") + def _renderJsonImage(self, doc: Document, image_data: Dict[str, Any], styles: Dict[str, Any]) -> None: """Render a JSON image to DOCX.""" try: @@ -1056,6 +1149,11 @@ class RendererDocx(BaseRenderer): if not caption: caption = image_data.get("caption", "") + # Last resort: lazily resolve a fileId reference to bytes on demand + # (large-document path - keeps image bytes out of the document JSON). + if not base64_data: + base64_data = self._lazyResolveImageBase64(content if isinstance(content, dict) else None, image_data) + # CRITICAL: Ensure we don't render base64 data as text # If base64_data looks like it might be rendered elsewhere, skip it if not base64_data: diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py index 7ec05c5c..9bfe3788 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py @@ -165,6 +165,10 @@ class RendererPdf(BaseRenderer): async def _generatePdfFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, unifiedStyle: Dict[str, Any] = None) -> str: """Generate PDF content from structured JSON document using AI-generated styling.""" + # Large-document streaming: image flowables are backed by temp files (reportlab + # reads them from disk at build time) so image bytes are not all resident in + # memory simultaneously. Collected here, deleted after the build. + self._tempImageFiles = [] try: # Get style set from unified style or legacy approach if unifiedStyle: @@ -210,7 +214,13 @@ class RendererPdf(BaseRenderer): document_title = (title or "").strip() if not document_title and isinstance(metadata, dict): document_title = (metadata.get("title") or "").strip() - if document_title: + # A cover_page section already renders its own title page; rendering the + # standalone document title on top of it would duplicate the title. + hasCoverPage = any( + (s.get("content_type") if isinstance(s, dict) else "") == "cover_page" + for s in sections + ) + if document_title and not hasCoverPage: story.append(self._paragraphFromInlineMarkdown(document_title, self._createDocumentTitleStyle(styles))) # Process each section (sections already extracted above) @@ -233,6 +243,18 @@ class RendererPdf(BaseRenderer): except Exception as e: self.logger.error(f"Error generating PDF from JSON: {str(e)}") raise Exception(f"PDF generation failed: {str(e)}") + finally: + self._cleanupTempImageFiles() + + def _cleanupTempImageFiles(self) -> None: + """Delete temp image files created for streamed (file-backed) PDF images.""" + import os + for path in getattr(self, "_tempImageFiles", []) or []: + try: + os.unlink(path) + except OSError: + pass + self._tempImageFiles = [] def _buildPdfWithOverflowGuard(self, doc, story: List[Any], buffer) -> None: """Try doc.build(); on 'too large on page' LayoutError, drop the offending @@ -790,6 +812,10 @@ class RendererPdf(BaseRenderer): all_elements.extend(self._renderJsonCodeBlock(element, styles)) elif element_type == "image": all_elements.extend(self._renderJsonImage(element, styles)) + elif element_type == "cover_page" or section_type == "cover_page": + all_elements.extend(self._renderCoverPage(element, styles)) + elif element_type == "image_grid" or section_type == "image_grid": + all_elements.extend(self._renderImageGrid(element, styles)) else: # Fallback: if element_type not set, use section_type as fallback if section_type == "table": @@ -1057,6 +1083,83 @@ class RendererPdf(BaseRenderer): self.logger.warning(f"Error rendering code block: {str(e)}") return [] + def _renderCoverPage(self, element: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]: + """Render a cover/title page: centered title, subtitle, author, date, optional logo. + + Content keys: title, subtitle, author, date, logo (fileId/_fileRef). Ends with a + page break so the body starts on the next page (legal filing / report front page). + """ + from reportlab.platypus import Spacer, PageBreak + content = element.get("content", element) if isinstance(element, dict) else {} + if not isinstance(content, dict): + content = {} + out: List[Any] = [Spacer(1, 160)] + # Optional logo at the top of the cover. + logoB64 = self._lazyResolveImageBase64(content) or content.get("logoBase64", "") + if logoB64: + out = self._renderJsonImage({"content": {"base64Data": logoB64, "altText": "Logo"}}, styles) + out.append(Spacer(1, 60)) + title = (content.get("title") or "").strip() + if title: + out.append(self._paragraphFromInlineMarkdown(title, self._createDocumentTitleStyle(styles))) + out.append(Spacer(1, 18)) + for key, sizePt in (("subtitle", 16), ("author", 12), ("date", 12)): + val = (content.get(key) or "").strip() + if not val: + continue + st = ParagraphStyle(f"cover_{key}", parent=self._createNormalStyle(styles), alignment=1, fontSize=sizePt) + out.append(Paragraph(self._escapeReportlabXml(val), st)) + out.append(Spacer(1, 8)) + out.append(PageBreak()) + return out + + def _renderImageGrid(self, element: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]: + """Render an image grid (N columns) -- marketing-style image arrangement. + + Content: ``columns`` (int, default 2) and ``images`` (list of dicts each carrying + base64Data or a fileId/_fileRef resolved lazily). Images are laid out in a + borderless reportlab Table so they sit side by side. + """ + from reportlab.platypus import Table, Spacer + content = element.get("content", element) if isinstance(element, dict) else {} + if not isinstance(content, dict): + return [] + images = content.get("images") or [] + if not isinstance(images, list) or not images: + return [] + try: + columns = max(1, int(content.get("columns", 2))) + except (TypeError, ValueError): + columns = 2 + + cellFlowables: List[Any] = [] + for img in images: + if not isinstance(img, dict): + continue + rendered = self._renderJsonImage({"content": img}, styles) + # Keep only the image flowable(s); skip caption paragraphs for a clean grid. + from reportlab.platypus import Image as ReportLabImage + imgsOnly = [f for f in rendered if isinstance(f, ReportLabImage)] + cellFlowables.append(imgsOnly[0] if imgsOnly else (rendered[0] if rendered else Spacer(1, 1))) + if not cellFlowables: + return [] + + # Pad to a full final row, then chunk into rows of `columns`. + while len(cellFlowables) % columns != 0: + cellFlowables.append(Spacer(1, 1)) + rows = [cellFlowables[i:i + columns] for i in range(0, len(cellFlowables), columns)] + colW = _PDF_CONTENT_WIDTH_PT / columns + # Scale each image to fit its column cell. + from reportlab.platypus import Image as ReportLabImage + for row in rows: + for cell in row: + if isinstance(cell, ReportLabImage) and getattr(cell, "drawWidth", 0) > colW - 8: + scale = (colW - 8) / cell.drawWidth + cell.drawWidth *= scale + cell.drawHeight *= scale + table = Table(rows, colWidths=[colW] * columns) + return [table, Spacer(1, 10)] + def _renderJsonImage(self, image_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]: """Render a JSON image to PDF elements using reportlab.""" try: @@ -1094,6 +1197,11 @@ class RendererPdf(BaseRenderer): if match: base64_data = match.group(1) + # Last resort: lazily resolve a fileId reference to bytes on demand + # (large-document path - keeps image bytes out of the document JSON). + if not base64_data: + base64_data = self._lazyResolveImageBase64(content if isinstance(content, dict) else None, image_data) + if not base64_data: self.logger.warning(f"No base64 data found for image. Alt text: {alt_text}") return [Paragraph(f"[Image: {alt_text}]", self._createNormalStyle(styles))] @@ -1183,8 +1291,20 @@ class RendererPdf(BaseRenderer): imgHeight = 3 * inch # ~216 points, safe for ~751pt available height imageStream.seek(0) - # Create reportlab Image - reportlabImage = ReportLabImage(imageStream, width=imgWidth, height=imgHeight) + # Create reportlab Image from a TEMP FILE rather than the in-memory + # stream: reportlab reads file-backed images lazily at build time, so + # the bytes of all images are not held in memory at once (large-doc path). + import tempfile + imageStream.seek(0) + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".img") + try: + tmp.write(imageStream.read()) + finally: + tmp.close() + if not hasattr(self, "_tempImageFiles") or self._tempImageFiles is None: + self._tempImageFiles = [] + self._tempImageFiles.append(tmp.name) + reportlabImage = ReportLabImage(tmp.name, width=imgWidth, height=imgHeight) elements = [reportlabImage] diff --git a/modules/serviceCenter/services/serviceGeneration/styleDefaults.py b/modules/serviceCenter/services/serviceGeneration/styleDefaults.py index 6d890f29..e6447c73 100644 --- a/modules/serviceCenter/services/serviceGeneration/styleDefaults.py +++ b/modules/serviceCenter/services/serviceGeneration/styleDefaults.py @@ -57,6 +57,97 @@ DEFAULT_STYLE: Dict[str, Any] = { } +# ------------------------------------------------------------------ +# Theme presets (A3): named, purpose-specific style overrides that are +# deep-merged onto DEFAULT_STYLE. A preset only declares the keys it changes; +# everything else inherits the default. Explicit per-call `style` overrides +# always win over the preset. +# ------------------------------------------------------------------ +THEME_PRESETS: Dict[str, Dict[str, Any]] = { + # "general" intentionally empty -> identical to DEFAULT_STYLE. + "general": {}, + "finance": { + "fonts": {"primary": "Calibri"}, + "colors": {"primary": "#0B3D2E", "secondary": "#14532D", "accent": "#047857"}, + "documentTitle": {"color": "#0B3D2E", "align": "left"}, + "headings": { + "h1": {"color": "#0B3D2E"}, + "h2": {"color": "#0B3D2E"}, + "h3": {"color": "#14532D"}, + "h4": {"color": "#14532D"}, + }, + "table": {"headerBg": "#0B3D2E", "rowBandingEven": "#ECFDF5"}, + }, + "legal": { + # Serif, sober, single-column, justified body, no logo banner. + "fonts": {"primary": "Times New Roman"}, + "colors": {"primary": "#1A1A1A", "secondary": "#333333", "accent": "#5A5A5A"}, + "documentTitle": {"color": "#1A1A1A", "align": "center", "sizePt": 20}, + "headings": { + "h1": {"color": "#1A1A1A", "sizePt": 16}, + "h2": {"color": "#1A1A1A", "sizePt": 14}, + "h3": {"color": "#333333", "sizePt": 12}, + "h4": {"color": "#333333", "sizePt": 11}, + }, + "paragraph": {"sizePt": 11, "lineSpacing": 1.5, "color": "#1A1A1A", "align": "justify"}, + "table": {"headerBg": "#333333", "rowBandingEven": "#F5F5F5", "borderColor": "#999999"}, + "page": {"showPageNumbers": True}, + }, + "technical": { + "fonts": {"primary": "Arial", "monospace": "Consolas"}, + "colors": {"primary": "#0F172A", "secondary": "#1E293B", "accent": "#2563EB"}, + "documentTitle": {"color": "#0F172A", "align": "left"}, + "headings": { + "h1": {"color": "#0F172A"}, + "h2": {"color": "#1E293B"}, + "h3": {"color": "#1E293B"}, + "h4": {"color": "#334155"}, + }, + "paragraph": {"sizePt": 10, "lineSpacing": 1.2}, + "codeBlock": {"fontSizePt": 9, "background": "#0F172A"}, + "table": {"headerBg": "#1E293B", "rowBandingEven": "#EEF2FF"}, + }, + "hr": { + "fonts": {"primary": "Calibri"}, + "colors": {"primary": "#5B21B6", "secondary": "#6D28D9", "accent": "#9333EA"}, + "documentTitle": {"color": "#5B21B6", "align": "center"}, + "headings": { + "h1": {"color": "#5B21B6"}, + "h2": {"color": "#6D28D9"}, + "h3": {"color": "#7C3AED"}, + "h4": {"color": "#7C3AED"}, + }, + "table": {"headerBg": "#5B21B6", "rowBandingEven": "#F5F3FF"}, + }, + "marketing": { + # Bold, image-friendly, generous spacing, larger title. + "fonts": {"primary": "Verdana"}, + "colors": {"primary": "#BE123C", "secondary": "#E11D48", "accent": "#F59E0B"}, + "documentTitle": {"color": "#BE123C", "sizePt": 34, "align": "center", "spaceAfterPt": 24}, + "headings": { + "h1": {"color": "#BE123C", "sizePt": 24}, + "h2": {"color": "#E11D48", "sizePt": 19}, + "h3": {"color": "#E11D48", "sizePt": 15}, + "h4": {"color": "#9F1239", "sizePt": 13}, + }, + "paragraph": {"sizePt": 12, "lineSpacing": 1.3}, + "image": {"defaultWidthPt": 540, "maxWidthPt": 900, "alignment": "center"}, + "table": {"headerBg": "#BE123C", "rowBandingEven": "#FFF1F2"}, + }, +} + + +def resolveTheme(themeName: str | None) -> Dict[str, Any]: + """Return the partial style override for a named theme preset. + + Unknown / empty names fall back to ``{}`` (i.e. plain DEFAULT_STYLE). + The lookup is case-insensitive. + """ + if not themeName: + return {} + return dict(THEME_PRESETS.get(str(themeName).strip().lower(), {})) + + def _deepMerge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]: """Recursively merge override into base. Both dicts left unchanged; returns new dict.""" result = {} @@ -76,8 +167,17 @@ def _deepMerge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any] return result -def resolveStyle(agentStyle: dict | None) -> Dict[str, Any]: - """Deep-merge DEFAULT_STYLE <- agentStyle. Returns fully resolved style dict.""" - if not agentStyle: - return dict(DEFAULT_STYLE) - return _deepMerge(DEFAULT_STYLE, agentStyle) +def resolveStyle(agentStyle: dict | None, documentTheme: str | None = None) -> Dict[str, Any]: + """Resolve the effective style: ``DEFAULT_STYLE <- themePreset <- agentStyle``. + + Precedence (lowest to highest): platform defaults, the named ``documentTheme`` + preset, then any explicit per-call ``agentStyle`` override. With no theme and + no override this returns plain :data:`DEFAULT_STYLE`. + """ + resolved = dict(DEFAULT_STYLE) + themeOverride = resolveTheme(documentTheme) + if themeOverride: + resolved = _deepMerge(resolved, themeOverride) + if agentStyle: + resolved = _deepMerge(resolved, agentStyle) + return resolved diff --git a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py index d3fddeb1..6d5404e8 100644 --- a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py +++ b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py @@ -134,6 +134,68 @@ def _parseInlineRuns(text: str) -> list: return runs if runs else [{"type": "text", "value": text}] +def _imageRefToDict(token: str) -> Optional[Dict[str, Any]]: + """Parse one image reference line into an image content dict. + + Accepts markdown image syntax ``![alt](file:ID "200pt")`` / ``![alt](https://...)`` + or a bare ``file:ID`` / URL. Returns None for blank lines. + """ + token = (token or "").strip() + if not token: + return None + m = re.match(r"^!\[([^\]]*)\]\(([^)\"]+)(?:\s+\"(\d+)pt\")?\)\s*$", token) + if m: + alt = (m.group(1) or "").strip() or "Image" + src = (m.group(2) or "").strip() + widthStr = m.group(3) + else: + alt, src, widthStr = "Image", token, None + fileId = src[5:] if src.startswith("file:") else "" + out: Dict[str, Any] = {"altText": alt, "base64Data": "", "_fileRef": fileId, "_srcUrl": src if not fileId else ""} + if widthStr: + out["widthPt"] = int(widthStr) + return out + + +def _parseCoverPageBlock(codeLines: List[str]) -> Dict[str, Any]: + """Parse a ```cover_page fenced block of ``key: value`` lines. + + Recognized keys: title, subtitle, author, date, logo (``file:ID`` or URL). + """ + content: Dict[str, Any] = {} + for raw in codeLines: + if ":" not in raw: + continue + key, _, value = raw.partition(":") + key = key.strip().lower() + value = value.strip() + if not value: + continue + if key in ("title", "subtitle", "author", "date"): + content[key] = value + elif key == "logo": + content["_fileRef"] = value[5:] if value.startswith("file:") else "" + if not content["_fileRef"]: + content["_srcUrl"] = value + return content + + +def _parseImageGridBlock(codeLines: List[str]) -> Dict[str, Any]: + """Parse a ```image_grid fenced block: optional ``columns: N`` plus image refs.""" + columns = 2 + images: List[Dict[str, Any]] = [] + for raw in codeLines: + stripped = raw.strip() + m = re.match(r"^columns\s*:\s*(\d+)\s*$", stripped, re.IGNORECASE) + if m: + columns = max(1, int(m.group(1))) + continue + img = _imageRefToDict(stripped) + if img: + images.append(img) + return {"columns": columns, "images": images} + + def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]: """ Convert markdown content to the standard document JSON format with Inline-Run model. @@ -178,6 +240,19 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D codeLines.append(lines[i]) i += 1 i += 1 + # Layout primitives are authored as fenced blocks with a special "language". + if lang == "cover_page": + sections.append({ + "id": _nextId(), "content_type": "cover_page", "order": order, + "elements": [{"content": _parseCoverPageBlock(codeLines)}], + }) + continue + if lang == "image_grid": + sections.append({ + "id": _nextId(), "content_type": "image_grid", "order": order, + "elements": [{"content": _parseImageGridBlock(codeLines)}], + }) + continue sections.append({ "id": _nextId(), "content_type": "code_block", "order": order, "elements": [{"content": {"code": "\n".join(codeLines), "language": lang}}], diff --git a/modules/serviceCenter/services/serviceKnowledge/_buildTree.py b/modules/serviceCenter/services/serviceKnowledge/_buildTree.py index 63bcc0f1..e4aad028 100644 --- a/modules/serviceCenter/services/serviceKnowledge/_buildTree.py +++ b/modules/serviceCenter/services/serviceKnowledge/_buildTree.py @@ -10,7 +10,7 @@ The UDB shows three logical hierarchies as a single user-facing tree: For every visible node the UI needs: - a stable `key` (used both for expand-state and as parent reference) - a `kind`, `label`, optional `icon` - - effective values for all three flags (neutralize, scope, ragIndexEnabled) + - effective values for flags (neutralize, ragIndexEnabled) - whether a backing DB record exists (`dataSourceId` + `modelType`) - whether the node has children to expand diff --git a/modules/serviceCenter/services/serviceKnowledge/_inheritFlags.py b/modules/serviceCenter/services/serviceKnowledge/_inheritFlags.py index b211839f..35de8409 100644 --- a/modules/serviceCenter/services/serviceKnowledge/_inheritFlags.py +++ b/modules/serviceCenter/services/serviceKnowledge/_inheritFlags.py @@ -1,6 +1,6 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. -"""Cascade-inherit semantics for DataSource flags (neutralize, ragIndexEnabled, scope). +"""Cascade-inherit semantics for DataSource flags (neutralize, ragIndexEnabled). Three-state flags allow tree elements to either set an explicit value or inherit the value from their nearest ancestor in the path hierarchy. @@ -19,7 +19,8 @@ Path-traversal rules: - Sub-elements have paths like `/folder1/sub`. Their parent path is the longest prefix path that exists as a DataSource record (string-based). - If no ancestor with an explicit value exists, the default is `False` - (or `'personal'` for scope) — matching the legacy behavior of NULL = inherit. + — matching the legacy behavior of NULL = inherit. + (scope was removed from DataSource in 2026-06 for privacy reasons.) """ import logging @@ -27,7 +28,7 @@ from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple logger = logging.getLogger(__name__) -_INHERITABLE_FLAGS = ("neutralize", "ragIndexEnabled", "scope") +_INHERITABLE_FLAGS = ("neutralize", "ragIndexEnabled") _INHERITABLE_FDS_FLAGS = ("neutralize", "ragIndexEnabled") # Connection-root DataSources carry the authority as their sourceType @@ -56,8 +57,6 @@ def _normalisePath(path: Optional[str]) -> str: def _flagDefault(flag: str) -> Any: - if flag == "scope": - return "personal" return False @@ -545,7 +544,8 @@ def resolveEffectiveForPath( """Resolve effective flags for ANY (connectionId, sourceType, path) tuple. Works whether or not a DataSource record exists for this exact path. - Returns dict with effectiveNeutralize, effectiveScope, effectiveRagIndexEnabled. + Returns dict with effectiveNeutralize, effectiveRagIndexEnabled. + (effectiveScope removed 2026-06 — personal sources have no scope.) """ normPath = _normalisePath(path) exactRecord = None @@ -561,7 +561,6 @@ def resolveEffectiveForPath( if exactRecord: return { "effectiveNeutralize": getEffectiveFlag(exactRecord, "neutralize", allDs, mode=mode), - "effectiveScope": getEffectiveFlag(exactRecord, "scope", allDs, mode=mode), "effectiveRagIndexEnabled": getEffectiveFlag(exactRecord, "ragIndexEnabled", allDs, mode=mode), } @@ -571,12 +570,10 @@ def resolveEffectiveForPath( "sourceType": sourceType, "path": normPath, "neutralize": None, - "scope": None, "ragIndexEnabled": None, } return { "effectiveNeutralize": getEffectiveFlag(virtualRec, "neutralize", allDs, mode=mode), - "effectiveScope": getEffectiveFlag(virtualRec, "scope", allDs, mode=mode), "effectiveRagIndexEnabled": getEffectiveFlag(virtualRec, "ragIndexEnabled", allDs, mode=mode), } diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py index 056a97cb..df80898b 100644 --- a/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py +++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py @@ -163,7 +163,7 @@ def _loadRagEnabledDataSources(connectionId: str, dataSourceIds: Optional[list] over all DataSources whose effective value resolves to True, including inherited ones. - Returned dicts carry **resolved** flags (`neutralize`, `scope`) so the + Returned dicts carry **resolved** flags (`neutralize`) so the downstream walkers can keep reading `ds.get("neutralize")` directly without having to know about the inheritance chain. @@ -185,7 +185,6 @@ def _loadRagEnabledDataSources(connectionId: str, dataSourceIds: Optional[list] continue dsCopy = dict(ds) if isinstance(ds, dict) else {**ds.__dict__} dsCopy["neutralize"] = getEffectiveFlag(ds, "neutralize", allDs) - dsCopy["scope"] = getEffectiveFlag(ds, "scope", allDs) dsCopy["ragIndexEnabled"] = True if connectionRootLimits: diff --git a/modules/serviceCenter/services/serviceKnowledge/subFeatureBootstrap.py b/modules/serviceCenter/services/serviceKnowledge/subFeatureBootstrap.py index e2aba02b..88a59408 100644 --- a/modules/serviceCenter/services/serviceKnowledge/subFeatureBootstrap.py +++ b/modules/serviceCenter/services/serviceKnowledge/subFeatureBootstrap.py @@ -152,7 +152,6 @@ async def _featureBootstrapHandler( try: dbConnector = _getFeatureDbConnector(featureCode) - provider = FeatureDataProvider(dbConnector) rootUser = getRootUser() ctx = ServiceCenterContext( @@ -162,6 +161,22 @@ async def _featureBootstrapHandler( ) knowledgeService = getService("knowledge", ctx) + # A2: index the SAME neutralized text the query path returns (parity). + neutralizationService = getService("neutralization", ctx) + neutralizePolicy = None + if effectiveNeutralize or neutralizeFields: + neutralizePolicy = { + tableName: { + "tableActive": bool(effectiveNeutralize), + "explicitFields": set(neutralizeFields), + } + } + provider = FeatureDataProvider( + dbConnector, + neutralizePolicy=neutralizePolicy, + neutralizationService=neutralizationService, + ) + extraFilters = [ {"field": k, "op": "=", "value": v} for k, v in recordFilter.items() @@ -186,12 +201,16 @@ async def _featureBootstrapHandler( if not rows: break + # Apply the A2 field-neutralization policy + JSON-serialize (same as + # the sub-agent query path) before building the embedding text. + rows = await provider.finalizeRowsAsync(tableName, rows) + for row in rows: rowId = row.get("id", "") if not rowId: continue - textContent = _serializeRowToText(row, neutralizeFields if effectiveNeutralize else None) + textContent = _serializeRowToText(row) if not textContent.strip(): fdsSkipped += 1 continue diff --git a/modules/serviceCenter/services/serviceKnowledge/udbNodes.py b/modules/serviceCenter/services/serviceKnowledge/udbNodes.py index 2d12554e..d0678e99 100644 --- a/modules/serviceCenter/services/serviceKnowledge/udbNodes.py +++ b/modules/serviceCenter/services/serviceKnowledge/udbNodes.py @@ -77,10 +77,11 @@ class UdbNode(ABC): def supportsFlag(self, flag: str) -> bool: """Whether this node carries a value for `flag` at all. - Subclasses override to restrict (e.g. FDS has no scope; fdsField - only has neutralize). + Subclasses override to restrict (e.g. fdsField only has neutralize). + Scope was removed from DataSource nodes (privacy, 2026-06) and never + existed on FDS nodes. Only Files (folder-files) retain scope. """ - return flag in ("neutralize", "scope", "ragIndexEnabled") + return flag in ("neutralize", "ragIndexEnabled") @abstractmethod def canEdit(self, context: Any, rootIf: Any) -> bool: @@ -185,7 +186,7 @@ class SyntheticContainerNode(UdbNode): "dataSourceId": None, "modelType": None, "effectiveNeutralize": self.getEffectiveFlag("neutralize", allDs, allFds, "aggregate"), - "effectiveScope": self.getEffectiveFlag("scope", allDs, allFds, "aggregate") or "personal", + "effectiveScope": "personal", "effectiveRagIndexEnabled": self.getEffectiveFlag("ragIndexEnabled", allDs, allFds, "aggregate"), "supportsRag": False, "canBeAdded": False, @@ -248,12 +249,14 @@ class _DataSourceFamilyNode(UdbNode): return _isConnectionOwner(rootIf, str(context.user.id), self.connectionId) def getEffectiveFlag(self, flag, allDs, allFds, mode="aggregate") -> Any: + if not self.supportsFlag(flag): + return False from modules.serviceCenter.services.serviceKnowledge._inheritFlags import ( resolveEffectiveForPath, ) out = resolveEffectiveForPath(self.connectionId, self.sourceType, self.path, allDs, mode=mode) key = "effective" + flag[0].upper() + flag[1:] - return out.get(key, False if flag != "scope" else "personal") + return out.get(key, False) def setFlag(self, flag, value, rootIf) -> List[str]: from modules.datamodels.datamodelDataSource import DataSource @@ -371,7 +374,7 @@ def _dsDict(node: _DataSourceFamilyNode, allDs: List[Dict[str, Any]]) -> Dict[st "dataSourceId": node.rec.get("id") if node.rec else None, "modelType": "DataSource" if node.rec else None, "effectiveNeutralize": node.getEffectiveFlag("neutralize", allDs, [], "aggregate"), - "effectiveScope": node.getEffectiveFlag("scope", allDs, [], "aggregate"), + "effectiveScope": "personal", "effectiveRagIndexEnabled": node.getEffectiveFlag("ragIndexEnabled", allDs, [], "aggregate"), "supportsRag": True, "canBeAdded": node.rec is None, @@ -900,8 +903,6 @@ def _aggregateFromChildren(node: UdbNode, flag: str, """ children = node.getLogicalChildren(allDs, allFds, None, None) if not children: - if flag == "scope": - return "personal" return False seen = set() last: Any = None @@ -917,7 +918,7 @@ def _aggregateFromChildren(node: UdbNode, flag: str, if len(seen) > 1: return "mixed" if not seen: - return "personal" if flag == "scope" else False + return False return last diff --git a/modules/shared/gdprDeletion.py b/modules/shared/gdprDeletion.py index 45a9ea43..c6f8d5ca 100644 --- a/modules/shared/gdprDeletion.py +++ b/modules/shared/gdprDeletion.py @@ -575,9 +575,6 @@ def _deleteUserDataFromFeatureDatabases(userId: str, currentUser) -> Dict[str, A elif featureCode == "realestate": from modules.features.realestate.interfaceFeatureRealEstate import getInterface as getRealEstateInterface featureInterface = getRealEstateInterface(currentUser) - elif featureCode == "chatbot": - from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface - featureInterface = getChatbotInterface(currentUser) elif featureCode == "neutralization": from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutralizerInterface featureInterface = getNeutralizerInterface(currentUser) diff --git a/modules/shared/i18nRegistry.py b/modules/shared/i18nRegistry.py index 06ccb20e..cb2d070f 100644 --- a/modules/shared/i18nRegistry.py +++ b/modules/shared/i18nRegistry.py @@ -440,7 +440,6 @@ def _registerFeatureUiLabels(): "modules.features.workspace.mainWorkspace", "modules.features.realEstate.mainRealEstate", "modules.features.neutralization.mainNeutralization", - "modules.features.chatbot.mainChatbot", ) added = 0 for modPath in _featureModulePaths: @@ -481,7 +480,6 @@ def _registerRbacLabels(): "modules.features.workspace.mainWorkspace", "modules.features.realEstate.mainRealEstate", "modules.features.neutralization.mainNeutralization", - "modules.features.chatbot.mainChatbot", ) added = 0 diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index aacc6d3c..21d0cbee 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -330,16 +330,6 @@ NAVIGATION_SECTIONS = [ "adminOnly": True, "sysAdminOnly": True, }, - { - "id": "admin-stt-benchmark", - "objectKey": "ui.admin.sttBenchmark", - "label": t("STT Benchmark"), - "icon": "FaMicrophone", - "path": "/admin/stt-benchmark", - "order": 92, - "adminOnly": True, - "sysAdminOnly": True, - }, { "id": "admin-languages", "objectKey": "ui.admin.languages", diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py index 5a1ff0eb..42962ad7 100644 --- a/modules/workflows/methods/methodAi/actions/generateDocument.py +++ b/modules/workflows/methods/methodAi/actions/generateDocument.py @@ -22,6 +22,7 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult: return ActionResult.isFailure(error="prompt is required") documentType = parameters.get("documentType") + documentTheme = parameters.get("documentTheme") or None # Prefer explicit outputFormat (flow UI); resultType remains for legacy / API callers. resultType = parameters.get("outputFormat") or parameters.get("resultType") if isinstance(resultType, str): @@ -82,7 +83,8 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult: outputFormat=resultType, # Can be None - AI determines from prompt title=title, parentOperationId=parentOperationId, - generationIntent="document" # NEW: Explicit intent, skips detection + generationIntent="document", # NEW: Explicit intent, skips detection + documentTheme=documentTheme # Named style preset for the renderer ) # Convert AiResponse to ActionResult diff --git a/modules/workflows/methods/methodAi/methodAi.py b/modules/workflows/methods/methodAi/methodAi.py index 64fc4f0f..aacfacaa 100644 --- a/modules/workflows/methods/methodAi/methodAi.py +++ b/modules/workflows/methods/methodAi/methodAi.py @@ -89,10 +89,10 @@ class MethodAi(MethodBase): name="documentTheme", type="str", frontendType=FrontendType.SELECT, - frontendOptions=["general", "finance", "legal", "technical", "hr"], + frontendOptions=["general", "finance", "legal", "technical", "hr", "marketing"], required=False, default="general", - description="Style hint for the document renderer (e.g. finance, legal). Used by the AI agent to choose colors and layout." + description="Named style preset for the document renderer (general/finance/legal/technical/hr/marketing). The agent forwards it to the renderDocument tool's documentTheme." ), "resultType": WorkflowActionParameter( name="resultType", @@ -118,7 +118,7 @@ class MethodAi(MethodBase): frontendType=FrontendType.CHECKBOX, required=False, default=False, - description="If true, uses fast simple AI call without document generation pipeline. Use for chatbot responses and simple text generation." + description="If true, uses fast simple AI call without document generation pipeline. Use for conversational responses and simple text generation." ), "contentParts": WorkflowActionParameter( name="contentParts", @@ -385,6 +385,15 @@ class MethodAi(MethodBase): required=False, description="Type of document (content hint for the model); used as title fallback when title is empty." ), + "documentTheme": WorkflowActionParameter( + name="documentTheme", + type="str", + frontendType=FrontendType.SELECT, + frontendOptions=["general", "finance", "legal", "technical", "hr", "marketing"], + required=False, + default="general", + description="Named style preset applied by the renderer (colors, fonts, spacing): general, finance, legal, technical, hr, marketing." + ), "resultType": WorkflowActionParameter( name="resultType", type="str", diff --git a/modules/workflows/methods/methodChatbot/__init__.py b/modules/workflows/methods/methodChatbot/__init__.py deleted file mode 100644 index a5940ab0..00000000 --- a/modules/workflows/methods/methodChatbot/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. - -from .methodChatbot import MethodChatbot - -__all__ = ['MethodChatbot'] diff --git a/modules/workflows/methods/methodChatbot/actions/__init__.py b/modules/workflows/methods/methodChatbot/actions/__init__.py deleted file mode 100644 index e76c67b4..00000000 --- a/modules/workflows/methods/methodChatbot/actions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. - diff --git a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py b/modules/workflows/methods/methodChatbot/actions/queryDatabase.py deleted file mode 100644 index 8622a5a9..00000000 --- a/modules/workflows/methods/methodChatbot/actions/queryDatabase.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. - -""" -Query Database action for Chatbot operations. -Executes SQL queries via the preprocessor connector. -""" - -import logging -import json -import time -from typing import Dict, Any -from modules.workflows.methods.methodBase import action -from modules.datamodels.datamodelChat import ActionResult, ActionDocument -from modules.connectors.connectorPreprocessor import PreprocessorConnector - -logger = logging.getLogger(__name__) - -@action -async def queryDatabase(self, parameters: Dict[str, Any]) -> ActionResult: - """ - Execute a SQL query via the preprocessor connector. - - Parameters: - - sqlQuery (str, required): SQL SELECT query to execute. Can also be extracted from analysis_result document if provided in documentList. - """ - try: - # Init progress logger - workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" - operationId = f"chatbot_query_db_{workflowId}_{int(time.time())}" - - # Start progress tracking - parentOperationId = parameters.get('parentOperationId') - self.services.chat.progressLogStart( - operationId, - "Database Query", - "Executing SQL Query", - "Preprocessing API", - parentOperationId=parentOperationId - ) - - # Get SQL query from parameters or extract from documentList - sqlQuery = parameters.get("sqlQuery") - - # If sqlQuery not provided, try to extract from documentList (analysis_result) - if not sqlQuery: - documentListParam = parameters.get("documentList") - if documentListParam: - # Get documents from previous task - from modules.datamodels.datamodelDocref import DocumentReferenceList - if isinstance(documentListParam, str): - docList = DocumentReferenceList.from_string_list([documentListParam]) - elif isinstance(documentListParam, list): - docList = DocumentReferenceList.from_string_list(documentListParam) - else: - docList = documentListParam - - # Get documents from workflow - documents = self.services.chat.getChatDocumentsFromDocumentList(docList) - - # Try to extract SQL query from JSON document - for doc in documents: - try: - # ChatDocument objects have fileId - get file data from database - if hasattr(doc, 'fileId') and doc.fileId: - # Get file data from database - fileData = self.services.interfaceDbComponent.getFileData(doc.fileId) - if fileData: - # Decode bytes if needed - if isinstance(fileData, bytes): - docData = fileData.decode('utf-8') - else: - docData = str(fileData) - - # Try to parse as JSON - analysisData = json.loads(docData) - sqlQuery = analysisData.get("sqlQuery") - - if sqlQuery: - logger.info(f"Extracted SQL query from analysis_result document: {sqlQuery[:100]}...") - break - except (json.JSONDecodeError, AttributeError, KeyError, TypeError) as e: - logger.debug(f"Could not parse document as JSON: {e}") - continue - - if not sqlQuery: - return ActionResult.isFailure(error="SQL query is required. Provide sqlQuery parameter or analysis_result document with sqlQuery field.") - - # Update progress - self.services.chat.progressLogUpdate(operationId, 0.3, "Validating query") - - # Validate: only SELECT queries allowed - sqlNormalized = sqlQuery.strip().upper() - if not sqlNormalized.startswith("SELECT"): - return ActionResult.isFailure(error="Only SELECT queries are allowed") - forbiddenKeywords = ["INSERT", "UPDATE", "DELETE", "DROP", "ALTER", "CREATE", "TRUNCATE", "EXEC", "EXECUTE"] - for kw in forbiddenKeywords: - if f" {kw} " in f" {sqlNormalized} " or sqlNormalized.startswith(f"{kw} "): - return ActionResult.isFailure(error=f"Forbidden SQL keyword detected: {kw}") - - # Initialize connector - connector = PreprocessorConnector() - - # Update progress - self.services.chat.progressLogUpdate(operationId, 0.5, "Executing query") - - try: - result = await connector.executeQuery(sqlQuery) - except Exception: - await connector.close() - raise - - # Update progress - self.services.chat.progressLogUpdate(operationId, 0.8, "Formatting results") - - # Generate meaningful filename - meaningful_name = self._generateMeaningfulFileName( - base_name="database_query", - extension="txt", - action_name="queryDatabase" - ) - - # Create validation metadata - validationMetadata = self._createValidationMetadata( - "queryDatabase", - sqlQuery=sqlQuery[:200] if len(sqlQuery) > 200 else sqlQuery, # Truncate for metadata - resultLength=len(result) - ) - - # Create action document - document = ActionDocument( - documentName=meaningful_name, - documentData=result, - mimeType="text/plain", - validationMetadata=validationMetadata - ) - - # Complete progress tracking - self.services.chat.progressLogFinish(operationId, True) - - # Close connector - await connector.close() - - return ActionResult.isSuccess(documents=[document]) - - except Exception as e: - logger.error(f"Error executing database query: {str(e)}") - - try: - self.services.chat.progressLogFinish(operationId, False) - except Exception: - pass - - return ActionResult.isFailure( - error=str(e) - ) - diff --git a/modules/workflows/methods/methodChatbot/methodChatbot.py b/modules/workflows/methods/methodChatbot/methodChatbot.py deleted file mode 100644 index cc44428e..00000000 --- a/modules/workflows/methods/methodChatbot/methodChatbot.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2025 Patrick Motsch -# All rights reserved. - -import logging -from modules.workflows.methods.methodBase import MethodBase -from modules.datamodels.datamodelWorkflowActions import WorkflowActionDefinition, WorkflowActionParameter -from modules.shared.frontendTypes import FrontendType - -# Import actions -from .actions.queryDatabase import queryDatabase - -logger = logging.getLogger(__name__) - -class MethodChatbot(MethodBase): - """Chatbot operations methods.""" - - def __init__(self, services): - super().__init__(services) - self.name = "chatbot" - self.description = "Chatbot operations" - - # RBAC-Integration: Action-Definitionen mit actionId - self._actions = { - "queryDatabase": WorkflowActionDefinition( - actionId="chatbot.queryDatabase", - description="Execute a SQL SELECT query via the preprocessor connector. Returns formatted query results.", - dynamicMode=False, - outputType="QueryResult", - parameters={ - "sqlQuery": WorkflowActionParameter( - name="sqlQuery", - type="str", - uiHint="textarea", - frontendType=FrontendType.TEXTAREA, - required=False, - description="SQL SELECT query to execute. If not provided, will attempt to extract from analysis_result document in documentList." - ), - "documentList": WorkflowActionParameter( - name="documentList", - type="DocumentList", - frontendType=FrontendType.DOCUMENT_REFERENCE, - required=False, - description="Document reference(s) containing analysis_result with sqlQuery field. Used if sqlQuery parameter is not provided." - ) - }, - execute=queryDatabase.__get__(self, self.__class__) - ) - } - - # Validate actions after definition - self._validateActions() - - # Register actions as methods (optional, für direkten Zugriff) - self.queryDatabase = queryDatabase.__get__(self, self.__class__) - diff --git a/requirements.txt b/requirements.txt index 0e330d51..70effc91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -81,7 +81,7 @@ pytest>=8.0.0 pytest-asyncio>=0.21.0 ## Configuration Validation -jsonschema>=4.0.0 # Required for chatbot workflow config validation +jsonschema>=4.0.0 # Required for feature/workflow config validation ## Streaming JSON parser (memory-safe import of large DB exports) ijson>=3.2.0 @@ -123,11 +123,4 @@ extract-msg>=0.55.0 pyproj>=3.6.0 # For coordinate transformations (EPSG:2056 <-> EPSG:4326) shapely>=2.0.0 # For geometric operations (intersections, area calculations) geopandas>=0.14.0 # For reading and querying GeoPackage files -fiona>=1.9.0 # Required by geopandas for reading GeoPackage files - -## LangChain & LangGraph for chatbot workflow -langchain==1.2.8 -langchain-core==1.2.8 -langgraph==1.0.7 -langchain-tavily==0.2.17 -nest-asyncio>=1.6.0 # For running async code in sync context (LangGraph compatibility) \ No newline at end of file +fiona>=1.9.0 # Required by geopandas for reading GeoPackage files \ No newline at end of file diff --git a/scripts/script_db_export_migration.py b/scripts/script_db_export_migration.py index f286327a..5a2f9214 100644 --- a/scripts/script_db_export_migration.py +++ b/scripts/script_db_export_migration.py @@ -11,7 +11,6 @@ erstellt: _structure.json Datenbanken: - poweron_app (User, Mandate, RBAC, Features, etc.) - poweron_chat (Chat-Konversationen und Nachrichten) - - poweron_chatbot (Chatbot-Feature: Konversationen, Nachrichten, Logs) - poweron_management (Workflows, Prompts, Connections, etc.) - poweron_realestate (Real Estate Daten) - poweron_trustee (Trustee Daten) @@ -104,7 +103,6 @@ ALL_DATABASES = [ "poweron_app", "poweron_billing", "poweron_chat", - "poweron_chatbot", "poweron_commcoach", "poweron_knowledge", "poweron_management", @@ -122,7 +120,6 @@ ALL_DATABASES = [ DATABASE_CONFIG = { "poweron_app": "DB_APP", "poweron_chat": "DB_CHAT", - "poweron_chatbot": "DB_CHATBOT", "poweron_management": "DB_MANAGEMENT", "poweron_realestate": "DB_REALESTATE", "poweron_trustee": "DB_TRUSTEE", @@ -772,7 +769,6 @@ def main(): Datenbanken: poweron_app - User, Mandate, RBAC, Features poweron_chat - Chat-Konversationen - poweron_chatbot - Chatbot-Feature poweron_management - Workflows, Prompts, Connections poweron_realestate - Real Estate Daten poweron_trustee - Trustee Daten @@ -781,7 +777,7 @@ Beispiele: python script_db_export_migration.py python script_db_export_migration.py --pretty python script_db_export_migration.py -o backup.json --pretty - python script_db_export_migration.py --db poweron_app,poweron_chat,poweron_chatbot + python script_db_export_migration.py --db poweron_app,poweron_chat,poweron_management python script_db_export_migration.py --exclude Token,AuthEvent --include-meta python script_db_export_migration.py --summary """ diff --git a/scripts/script_db_init_chatbot.py b/scripts/script_db_init_chatbot.py deleted file mode 100644 index 907c3a4b..00000000 --- a/scripts/script_db_init_chatbot.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python3 -""" -Initialize poweron_chatbot database for the Chatbot feature. - -Creates the poweron_chatbot database if it does not exist. -Uses DB_CHATBOT_* config (falls back to DB_*). -Tables (ChatbotConversation, ChatbotMessage, ChatbotDocument, ChatbotLog) are -auto-created by the connector on first use. - -Usage: - python script_db_init_chatbot.py [--dry-run] -""" - -import os -import sys -import argparse -import logging -from pathlib import Path - -scriptPath = Path(__file__).resolve() -gatewayPath = scriptPath.parent.parent -sys.path.insert(0, str(gatewayPath)) -os.chdir(str(gatewayPath)) - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", -) -logger = logging.getLogger(__name__) - -import psycopg2 -from modules.shared.configuration import APP_CONFIG - -DB_NAME = "poweron_chatbot" -CONFIG_PREFIX = "DB_CHATBOT" - - -def _get_config(): - """Get DB config: DB_CHATBOT_* with fallback to DB_*.""" - host = APP_CONFIG.get(f"{CONFIG_PREFIX}_HOST") or APP_CONFIG.get("DB_HOST", "localhost") - port = int(APP_CONFIG.get(f"{CONFIG_PREFIX}_PORT") or APP_CONFIG.get("DB_PORT", "5432")) - user = APP_CONFIG.get(f"{CONFIG_PREFIX}_USER") or APP_CONFIG.get("DB_USER") - password = ( - APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD_SECRET") - or APP_CONFIG.get(f"{CONFIG_PREFIX}_PASSWORD") - or APP_CONFIG.get("DB_PASSWORD_SECRET") - or APP_CONFIG.get("DB_PASSWORD") - ) - return {"host": host, "port": port, "user": user, "password": password} - - -def init_chatbot_db(dry_run: bool = False) -> bool: - """Create poweron_chatbot database if it does not exist.""" - config = _get_config() - if not config["user"] or not config["password"]: - logger.error("DB_USER and DB_PASSWORD (or DB_CHATBOT_*) required") - return False - - try: - conn = psycopg2.connect( - host=config["host"], - port=config["port"], - database="postgres", - user=config["user"], - password=config["password"], - ) - conn.autocommit = True - - with conn.cursor() as cur: - cur.execute( - "SELECT 1 FROM pg_database WHERE datname = %s", - (DB_NAME,), - ) - exists = cur.fetchone() is not None - - if exists: - logger.info(f"Database {DB_NAME} already exists") - else: - if dry_run: - logger.info(f"[DRY-RUN] Would create database {DB_NAME}") - else: - cur.execute(f'CREATE DATABASE "{DB_NAME}"') - logger.info(f"Created database {DB_NAME}") - - conn.close() - return True - except Exception as e: - logger.error(f"Failed to init {DB_NAME}: {e}") - return False - - -def main(): - parser = argparse.ArgumentParser(description="Initialize poweron_chatbot database") - parser.add_argument("--dry-run", action="store_true", help="Do not create, only report") - args = parser.parse_args() - ok = init_chatbot_db(dry_run=args.dry_run) - sys.exit(0 if ok else 1) - - -if __name__ == "__main__": - main() diff --git a/scripts/script_migrate_user_uid.py b/scripts/script_migrate_user_uid.py index 07f9b443..e36f483d 100644 --- a/scripts/script_migrate_user_uid.py +++ b/scripts/script_migrate_user_uid.py @@ -47,7 +47,6 @@ ALL_DATABASES = [ "poweron_billing", "poweron_workspace", "poweron_graphicaleditor", - "poweron_chatbot", "poweron_trustee", "poweron_commcoach", "poweron_neutralization", diff --git a/tests/demo/README.md b/tests/demo/README.md index 6887f94a..9d68586f 100644 --- a/tests/demo/README.md +++ b/tests/demo/README.md @@ -30,6 +30,5 @@ pytest tests/demo/test_demo_uc1_trustee.py -v | `test_demo_bootstrap.py` | Idempotent load/remove, mandates, user, features, RMA, neutralization | | `test_demo_uc1_trustee.py` | Trustee instances, RMA config, system workflow templates | | `test_demo_uc2_realestate.py` | Workspace instances for agent demo | -| `test_demo_uc3_chatbot.py` | Chatbot instance, knowledge-base files | | `test_demo_uc4_i18n.py` | i18n readiness, Spanish not pre-installed | | `test_demo_neutralization.py` | Neutralization config enabled, test PDF exists | diff --git a/tests/demo/test_demo_bootstrap.py b/tests/demo/test_demo_bootstrap.py index 09076e57..3ac6073e 100644 --- a/tests/demo/test_demo_bootstrap.py +++ b/tests/demo/test_demo_bootstrap.py @@ -60,19 +60,6 @@ class TestDemoBootstrap: instances = _getFeatureInstances(db, mid, featureCode) assert len(instances) >= 1, f"Feature '{featureCode}' missing in Alpina Treuhand AG" - def test_alpinaNoChatbot(self, db, mandateAlpina): - """Alpina should NOT have a chatbot instance.""" - mid = mandateAlpina.get("id") - instances = _getFeatureInstances(db, mid, "chatbot") - assert len(instances) == 0, "Alpina Treuhand should not have chatbot" - - def test_happylifeNoChatbot(self, db, mandateHappylife): - """HappyLife also should NOT have a chatbot instance — chatbot was - removed from the InvestorDemo on 2026-04-20 (see changelog).""" - mid = mandateHappylife.get("id") - instances = _getFeatureInstances(db, mid, "chatbot") - assert len(instances) == 0, "HappyLife should no longer have chatbot (removed 2026-04-20)" - class TestDemoBootstrapRma: diff --git a/tests/demo/test_demo_uc3_chatbot.py b/tests/demo/test_demo_uc3_chatbot.py deleted file mode 100644 index 0248bd5d..00000000 --- a/tests/demo/test_demo_uc3_chatbot.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -T-UC3: Knowledge Chatbot. - -The chatbot feature instance was removed from the InvestorDemo on -2026-04-20 (see changelog) — neither HappyLife nor Alpina bootstrap a -chatbot today; the actual RAG demo runs via workspace. We still verify -the knowledge-base demo files are present and that the bootstrap does -NOT (re)create chatbot instances in either mandate. -""" - -import pytest -from pathlib import Path -from tests.demo.conftest import _getFeatureInstances - - -class TestChatbotSetup: - - def test_chatbotNotInHappylife(self, db, mandateHappylife): - """HappyLife should NOT have a chatbot instance (removed 2026-04-20).""" - mid = mandateHappylife.get("id") - instances = _getFeatureInstances(db, mid, "chatbot") - assert len(instances) == 0, "HappyLife should no longer bootstrap a chatbot instance" - - def test_chatbotNotInAlpina(self, db, mandateAlpina): - """Alpina should NOT have a chatbot instance.""" - mid = mandateAlpina.get("id") - instances = _getFeatureInstances(db, mid, "chatbot") - assert len(instances) == 0, "Alpina should not have chatbot" - - -class TestKnowledgeBaseFiles: - - def test_knowledgeBaseFilesExist(self): - """Knowledge-base documents must exist in demoData.""" - kbDir = Path(__file__).resolve().parent.parent.parent / "demoData" / "knowledge-base" - assert kbDir.exists(), f"knowledge-base dir not found at {kbDir}" - files = list(kbDir.iterdir()) - docs = [f for f in files if f.suffix in (".md", ".html", ".pdf", ".docx", ".txt")] - assert len(docs) >= 3, f"Expected at least 3 knowledge-base docs, found {len(docs)}: {[f.name for f in docs]}" diff --git a/tests/serviceGeneration/test_large_document_render.py b/tests/serviceGeneration/test_large_document_render.py new file mode 100644 index 00000000..8b757e64 --- /dev/null +++ b/tests/serviceGeneration/test_large_document_render.py @@ -0,0 +1,154 @@ +# Copyright (c) 2026 Patrick Motsch +# All rights reserved. +"""A3 / AC15: lazy file-reference image resolution for large documents. + +Block images may carry only a ``fileId`` (no embedded base64). The renderer +fetches the bytes on demand via an injected ``_imageResolver``, so the document +JSON never holds every image's bytes simultaneously. These tests exercise the +PDF and DOCX block-image paths directly (real reportlab / python-docx), without +the full GenerationService/services plumbing. +""" + +import io + +import pytest + +from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle +from modules.serviceCenter.services.serviceGeneration.renderers.rendererPdf import RendererPdf +from modules.serviceCenter.services.serviceGeneration.renderers.rendererDocx import RendererDocx + + +def _tinyPng() -> bytes: + from PIL import Image as PILImage + + buf = io.BytesIO() + PILImage.new("RGB", (8, 8), (200, 30, 30)).save(buf, format="PNG") + return buf.getvalue() + + +class _ResolverSpy: + """Records the fileIds it was asked for and returns fixed PNG bytes.""" + + def __init__(self, data: bytes): + self._data = data + self.calls = [] + + def __call__(self, fileId): + self.calls.append(fileId) + return self._data + + +def _imageElement(fileId: str): + return {"type": "image", "content": {"fileId": fileId, "altText": "Image"}} + + +# ── Base helper ──────────────────────────────────────────────────── + +def test_base_helper_resolves_bytes_to_base64(): + png = _tinyPng() + r = RendererPdf() + r._imageResolver = _ResolverSpy(png) + b64 = r._lazyResolveImageBase64({"fileId": "img-1"}) + import base64 as _b64 + assert _b64.b64decode(b64) == png + + +def test_base_helper_without_resolver_returns_empty(): + r = RendererPdf() + assert r._lazyResolveImageBase64({"fileId": "img-1"}) == "" + + +def test_base_helper_without_fileid_returns_empty(): + r = RendererPdf() + r._imageResolver = _ResolverSpy(_tinyPng()) + assert r._lazyResolveImageBase64({"altText": "no id"}) == "" + + +# ── DOCX block images ────────────────────────────────────────────── + +def test_docx_lazy_resolves_block_images(): + from docx import Document + + spy = _ResolverSpy(_tinyPng()) + r = RendererDocx() + r._imageResolver = spy + + doc = Document() + n = 5 + elements = [_imageElement(f"img-{i}") for i in range(n)] + for el in elements: + r._renderJsonImage(doc, el, {}) + + # All N images embedded, resolver hit once per image, in order. + assert len(doc.inline_shapes) == n + assert spy.calls == [f"img-{i}" for i in range(n)] + # Stays lazy: the source JSON never gained embedded base64. + assert all("base64Data" not in el["content"] for el in elements) + + +def test_docx_without_resolver_embeds_no_image(): + from docx import Document + + r = RendererDocx() # no _imageResolver + doc = Document() + r._renderJsonImage(doc, _imageElement("img-1"), {}) + assert len(doc.inline_shapes) == 0 + + +# ── PDF block images ─────────────────────────────────────────────── + +def test_pdf_lazy_resolves_block_image(): + from reportlab.platypus import Image as ReportLabImage + + spy = _ResolverSpy(_tinyPng()) + r = RendererPdf() + r._imageResolver = spy + styles = r._convertUnifiedStyleToInternal(resolveStyle(None)) + + el = _imageElement("img-1") + flowables = r._renderJsonImage(el, styles) + + assert spy.calls == ["img-1"] + assert any(isinstance(f, ReportLabImage) for f in flowables) + assert "base64Data" not in el["content"] + r._cleanupTempImageFiles() + + +def test_pdf_without_resolver_returns_placeholder(): + from reportlab.platypus import Image as ReportLabImage + + r = RendererPdf() # no _imageResolver + styles = r._convertUnifiedStyleToInternal(resolveStyle(None)) + flowables = r._renderJsonImage(_imageElement("img-1"), styles) + assert not any(isinstance(f, ReportLabImage) for f in flowables) + + +def test_pdf_images_are_file_backed_and_cleaned(): + """Step 2: PDF image flowables read from temp files (not in-memory streams), + so image bytes are not all resident at build time. Temp files are cleaned up.""" + import os + from reportlab.platypus import Image as ReportLabImage + + spy = _ResolverSpy(_tinyPng()) + r = RendererPdf() + r._imageResolver = spy + r._tempImageFiles = [] + styles = r._convertUnifiedStyleToInternal(resolveStyle(None)) + + n = 4 + images = [] + for i in range(n): + flowables = r._renderJsonImage(_imageElement(f"img-{i}"), styles) + images += [f for f in flowables if isinstance(f, ReportLabImage)] + + assert len(images) == n + # Each image is file-backed by a tracked temp file that currently exists on disk. + assert len(r._tempImageFiles) == n + for img in images: + assert img.filename in r._tempImageFiles + assert os.path.exists(img.filename) + + paths = list(r._tempImageFiles) + r._cleanupTempImageFiles() + assert all(not os.path.exists(p) for p in paths) + assert r._tempImageFiles == [] diff --git a/tests/serviceGeneration/test_layout_primitives.py b/tests/serviceGeneration/test_layout_primitives.py new file mode 100644 index 00000000..1c9e6c5e --- /dev/null +++ b/tests/serviceGeneration/test_layout_primitives.py @@ -0,0 +1,146 @@ +# Copyright (c) 2026 Patrick Motsch +# All rights reserved. +"""A3: layout primitives (cover_page, image_grid). + +Covers the markdown authoring conventions (fenced ```cover_page / ```image_grid +blocks) and the PDF/DOCX renderer handlers, using real reportlab / python-docx. +""" + +import base64 +import io + + +from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle +from modules.serviceCenter.services.serviceGeneration.renderers.rendererPdf import RendererPdf +from modules.serviceCenter.services.serviceGeneration.renderers.rendererDocx import RendererDocx +from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson +from modules.datamodels.datamodelJson import supportedSectionTypes + + +def _tinyPngB64() -> str: + from PIL import Image as PILImage + + buf = io.BytesIO() + PILImage.new("RGB", (8, 8), (30, 120, 200)).save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode("ascii") + + +def _imgContent(): + return {"base64Data": _tinyPngB64(), "altText": "Pic"} + + +# ── datamodel ─────────────────────────────────────────────────────── + +def test_layout_types_are_registered(): + assert "cover_page" in supportedSectionTypes + assert "image_grid" in supportedSectionTypes + + +# ── markdown authoring ────────────────────────────────────────────── + +def test_markdown_parses_cover_page_block(): + md = ( + "```cover_page\n" + "title: Klageschrift\n" + "subtitle: In Sachen A gegen B\n" + "author: Dr. Muster\n" + "date: 2026-06-02\n" + "logo: file:logo-1\n" + "```\n" + ) + doc = markdownToDocumentJson(md, "T") + sections = doc["documents"][0]["sections"] + assert len(sections) == 1 + sec = sections[0] + assert sec["content_type"] == "cover_page" + content = sec["elements"][0]["content"] + assert content["title"] == "Klageschrift" + assert content["subtitle"] == "In Sachen A gegen B" + assert content["author"] == "Dr. Muster" + assert content["date"] == "2026-06-02" + assert content["_fileRef"] == "logo-1" + + +def test_markdown_parses_image_grid_block(): + md = ( + "```image_grid\n" + "columns: 3\n" + '![Foto1](file:img-1 "200pt")\n' + "file:img-2\n" + "https://example.com/x.png\n" + "```\n" + ) + doc = markdownToDocumentJson(md, "T") + sec = doc["documents"][0]["sections"][0] + assert sec["content_type"] == "image_grid" + content = sec["elements"][0]["content"] + assert content["columns"] == 3 + assert len(content["images"]) == 3 + assert content["images"][0]["_fileRef"] == "img-1" + assert content["images"][0]["widthPt"] == 200 + assert content["images"][1]["_fileRef"] == "img-2" + assert content["images"][2]["_srcUrl"] == "https://example.com/x.png" + + +def test_markdown_plain_code_block_still_works(): + md = "```python\nprint('x')\n```\n" + doc = markdownToDocumentJson(md, "T") + sec = doc["documents"][0]["sections"][0] + assert sec["content_type"] == "code_block" + assert sec["elements"][0]["content"]["language"] == "python" + + +# ── PDF handlers ──────────────────────────────────────────────────── + +def test_pdf_cover_page_emits_page_break(): + from reportlab.platypus import PageBreak + + r = RendererPdf() + styles = r._convertUnifiedStyleToInternal(resolveStyle(None)) + el = {"content": {"title": "Report", "subtitle": "2026", "author": "PM"}} + flowables = r._renderCoverPage(el, styles) + assert any(isinstance(f, PageBreak) for f in flowables) + assert len(flowables) >= 2 + + +def test_pdf_image_grid_builds_table(): + from reportlab.platypus import Table + + r = RendererPdf() + r._tempImageFiles = [] + styles = r._convertUnifiedStyleToInternal(resolveStyle(None)) + el = {"content": {"columns": 2, "images": [_imgContent(), _imgContent(), _imgContent()]}} + flowables = r._renderImageGrid(el, styles) + tables = [f for f in flowables if isinstance(f, Table)] + assert len(tables) == 1 + # 3 images, 2 columns -> 2 rows (last cell padded). + assert len(tables[0]._cellvalues) == 2 + r._cleanupTempImageFiles() + + +# ── DOCX handlers ─────────────────────────────────────────────────── + +def test_docx_cover_page_renders_and_breaks(): + from docx import Document + + r = RendererDocx() + doc = Document() + before = len(doc.paragraphs) + r._renderCoverPage(doc, {"content": {"title": "Klageschrift", "author": "Dr. M"}}, {}) + # New paragraphs were added (spacers + title + author) and a page break exists. + assert len(doc.paragraphs) > before + xml = doc.element.xml + assert "w:br" in xml and 'type="page"' in xml + + +def test_docx_image_grid_builds_table_with_pictures(): + from docx import Document + + r = RendererDocx() + doc = Document() + r._renderImageGrid(doc, {"content": {"columns": 2, "images": [_imgContent(), _imgContent(), _imgContent()]}}, {}) + assert len(doc.tables) == 1 + table = doc.tables[0] + assert len(table.columns) == 2 + assert len(table.rows) == 2 + assert len(doc.inline_shapes) == 3 diff --git a/tests/serviceGeneration/test_style_resolver.py b/tests/serviceGeneration/test_style_resolver.py index 06f907ef..23027efe 100644 --- a/tests/serviceGeneration/test_style_resolver.py +++ b/tests/serviceGeneration/test_style_resolver.py @@ -1,7 +1,12 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. import pytest -from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle, DEFAULT_STYLE +from modules.serviceCenter.services.serviceGeneration.styleDefaults import ( + resolveStyle, + resolveTheme, + DEFAULT_STYLE, + THEME_PRESETS, +) def test_resolve_none_returns_defaults(): @@ -44,3 +49,45 @@ def test_override_document_title_partial_merge(): assert result["documentTitle"]["sizePt"] == 32 assert result["documentTitle"]["align"] == "center" assert result["headings"]["h1"]["sizePt"] == DEFAULT_STYLE["headings"]["h1"]["sizePt"] + + +# ── Theme presets (A3) ───────────────────────────────────────────── + +def test_resolve_theme_unknown_is_empty(): + assert resolveTheme(None) == {} + assert resolveTheme("does-not-exist") == {} + + +def test_resolve_theme_case_insensitive(): + assert resolveTheme("FINANCE") == THEME_PRESETS["finance"] + + +def test_general_theme_equals_defaults(): + assert resolveStyle(None, "general") == DEFAULT_STYLE + + +def test_theme_applies_preset_over_defaults(): + result = resolveStyle(None, "legal") + # legal preset changes the primary font to a serif and justifies body text + assert result["fonts"]["primary"] == "Times New Roman" + assert result["paragraph"]["align"] == "justify" + # untouched keys still come from DEFAULT_STYLE + assert result["page"]["format"] == DEFAULT_STYLE["page"]["format"] + + +def test_explicit_style_overrides_theme(): + # theme sets finance green; explicit style must win + result = resolveStyle({"colors": {"primary": "#FF0000"}}, "finance") + assert result["colors"]["primary"] == "#FF0000" + # non-overridden theme key still applies + assert result["table"]["headerBg"] == THEME_PRESETS["finance"]["table"]["headerBg"] + + +def test_marketing_theme_enlarges_title_and_images(): + result = resolveStyle(None, "marketing") + assert result["documentTitle"]["sizePt"] == 34 + assert result["image"]["defaultWidthPt"] == 540 + + +def test_unknown_theme_falls_back_to_defaults(): + assert resolveStyle(None, "rainbow") == DEFAULT_STYLE diff --git a/tests/unit/methods/test_action_signature_validator.py b/tests/unit/methods/test_action_signature_validator.py index 8e54fdcf..5607117a 100644 --- a/tests/unit/methods/test_action_signature_validator.py +++ b/tests/unit/methods/test_action_signature_validator.py @@ -264,7 +264,6 @@ def _instantiateMethod(methodCls): ("modules.workflows.methods.methodFile.methodFile", "MethodFile"), ("modules.workflows.methods.methodContext.methodContext", "MethodContext"), ("modules.workflows.methods.methodJira.methodJira", "MethodJira"), - ("modules.workflows.methods.methodChatbot.methodChatbot", "MethodChatbot"), ]) def test_methodSignaturesAreHealthy(modulePath, className): """Each shipping Method's _actions must validate against the catalog.""" diff --git a/tests/unit/serviceAgent/test_field_neutralization.py b/tests/unit/serviceAgent/test_field_neutralization.py new file mode 100644 index 00000000..6cd52974 --- /dev/null +++ b/tests/unit/serviceAgent/test_field_neutralization.py @@ -0,0 +1,149 @@ +# Copyright (c) 2026 Patrick Motsch +# All rights reserved. +"""A2: type/inheritance-aware field neutralization for feature source data. + +Tests the pure policy logic in +``modules.serviceCenter.services.serviceAgent.featureDataProvider._neutralizeAndSerializeRows`` +with a fake NeutralizationService (no DB, no real LLM). + +Rules under test: + 1. strings -> substring-neutralized when effective (explicit OR inherited) + 2. binary -> dropped when neutralization applies + 3. other scalars -> neutralized ONLY when the field flag is explicit +""" + +import asyncio +import datetime + +from modules.serviceCenter.services.serviceAgent.featureDataProvider import ( + _neutralizeAndSerializeRows, +) + + +class _FakeNeutralizer: + """Deterministic stand-in: replaces the token 'SECRET' with a placeholder. + + Records every text it was asked to process so tests can assert that the + field name was passed as a type hint (prefix ``": "``). + """ + + def __init__(self): + self.seen = [] + + async def processTextAsync(self, text, fileId=None): + self.seen.append(text) + neutralized = text.replace("SECRET", "[name.00000000-0000-0000-0000-000000000000]") + return {"neutralized_text": neutralized} + + +def _run(rows, policy, neutralizer): + return asyncio.run(_neutralizeAndSerializeRows(rows, policy, neutralizer)) + + +def test_string_substring_neutralized_when_inherited(): + """AC7: tableActive (inherited), empty explicitFields -> strings substring-neutralized.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "note": "Patient SECRET has a fever"}] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, fake) + assert out[0]["note"] == "Patient [name.00000000-0000-0000-0000-000000000000] has a fever" + # surrounding text preserved (NOT whole-value replaced) + assert out[0]["note"].startswith("Patient ") + # id is structural -> never neutralized + assert out[0]["id"] == "row-1" + + +def test_field_name_passed_as_type_hint(): + """AC8: the field name is prepended as a type hint and stripped from the result.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "lastName": "SECRET"}] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, fake) + assert out[0]["lastName"] == "[name.00000000-0000-0000-0000-000000000000]" + assert any(t.startswith("lastName: ") for t in fake.seen) + + +def test_binary_dropped_when_active(): + """AC9: binary columns are dropped (not masked) when neutralization applies.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "photo": b"\x89PNG\x00\x01", "note": "ok"}] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, fake) + assert "photo" not in out[0] + assert out[0]["id"] == "row-1" + + +def test_numeric_unchanged_when_only_inherited(): + """AC10: numeric/date fields are NOT neutralized via inheritance.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "amount": 4200, "bookedAt": datetime.date(2026, 1, 2)}] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, fake) + assert out[0]["amount"] == 4200 + assert out[0]["bookedAt"] == "2026-01-02" + + +def test_numeric_neutralized_when_explicit(): + """AC11: numeric field neutralized (whole-value placeholder) when explicitly flagged.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "amount": 4200}] + policy = {"tableActive": False, "explicitFields": {"amount"}} + out = _run(rows, policy, fake) + assert out[0]["amount"].startswith("[NEUT.amount.") + assert out[0]["amount"].endswith("]") + + +def test_failsafe_redacts_when_no_neutralizer(): + """No engine available but neutralization required -> fail closed ([REDACTED]).""" + rows = [{"id": "row-1", "note": "Patient SECRET"}] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, None) + assert out[0]["note"] == "[REDACTED]" + + +def test_dedup_across_rows_single_call_per_unique_value(): + """Identical (field,value) pairs are neutralized once and reused.""" + fake = _FakeNeutralizer() + rows = [ + {"id": "a", "note": "SECRET"}, + {"id": "b", "note": "SECRET"}, + ] + policy = {"tableActive": True, "explicitFields": set()} + out = _run(rows, policy, fake) + assert out[0]["note"] == out[1]["note"] + # 'note: SECRET' appears exactly once in the engine call log (deduped) + assert fake.seen.count("note: SECRET") == 1 + + +def test_inactive_table_leaves_strings_untouched(): + """No tableActive and no explicit fields -> strings pass through unchanged.""" + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "note": "Patient SECRET"}] + policy = {"tableActive": False, "explicitFields": set()} + out = _run(rows, policy, fake) + assert out[0]["note"] == "Patient SECRET" + assert fake.seen == [] + + +def test_rag_bootstrap_parity_with_query_path(): + """AC12: the RAG bootstrap serializes the SAME neutralized rows the query path returns. + + Both paths run rows through ``_neutralizeAndSerializeRows``; the bootstrap then + turns the finalized row into embedding text via ``_serializeRowToText``. This + asserts no raw secret leaks into the index text and structural fields are skipped. + """ + from modules.serviceCenter.services.serviceKnowledge.subFeatureBootstrap import _serializeRowToText + + fake = _FakeNeutralizer() + rows = [{"id": "row-1", "sysCreatedAt": "2026-01-01", "note": "Patient SECRET has a fever"}] + policy = {"tableActive": True, "explicitFields": set()} + + finalized = _run(rows, policy, fake) # identical to the sub-agent query path + indexText = _serializeRowToText(finalized[0]) + + # Neutralized placeholder is indexed, raw secret is not. + assert "[name.00000000-0000-0000-0000-000000000000]" in indexText + assert "SECRET" not in indexText + # Structural fields are excluded from the embedding text. + assert "id:" not in indexText + assert "sysCreatedAt" not in indexText diff --git a/tests/unit/services/test_inheritFlags.py b/tests/unit/services/test_inheritFlags.py index 40099e56..3b9ce395 100644 --- a/tests/unit/services/test_inheritFlags.py +++ b/tests/unit/services/test_inheritFlags.py @@ -27,7 +27,6 @@ def _ds(idVal: str, path: str, **flags) -> dict: "path": path, "neutralize": None, "ragIndexEnabled": None, - "scope": None, } base.update(flags) return base @@ -97,15 +96,6 @@ class TestEffectiveFlagWalk(unittest.TestCase): leaf = _ds("l", "/anything/anywhere") self.assertTrue(_inheritFlags.getEffectiveFlag(leaf, "neutralize", [root, leaf])) - def test_scope_inheritance_with_string_default(self): - root = _ds("r", "/", scope="mandate") - leaf = _ds("l", "/folder") - self.assertEqual(_inheritFlags.getEffectiveFlag(leaf, "scope", [root, leaf]), "mandate") - - def test_scope_default_personal_when_empty(self): - leaf = _ds("l", "/folder") - self.assertEqual(_inheritFlags.getEffectiveFlag(leaf, "scope", [leaf]), "personal") - def test_unknown_flag_raises(self): leaf = _ds("l", "/") with self.assertRaises(ValueError): @@ -158,20 +148,6 @@ class TestEffectiveFlagAggregate(unittest.TestCase): allDs = [root, child1, child2] self.assertEqual(_inheritFlags.getEffectiveFlag(root, "neutralize", allDs, mode="aggregate"), "mixed") - def test_mixed_scope(self): - root = _ds("r", "/", scope="personal") - child1 = _ds("c1", "/a", scope="team") - child2 = _ds("c2", "/b") # inherits personal from root - allDs = [root, child1, child2] - self.assertEqual(_inheritFlags.getEffectiveFlag(root, "scope", allDs, mode="aggregate"), "mixed") - - def test_all_scope_same_explicit_returns_concrete(self): - root = _ds("r", "/", scope="team") - child1 = _ds("c1", "/a", scope="team") - child2 = _ds("c2", "/b") # inherits team - allDs = [root, child1, child2] - self.assertEqual(_inheritFlags.getEffectiveFlag(root, "scope", allDs, mode="aggregate"), "team") - def test_connection_root_aggregate_cross_sourcetype(self): connRoot = _ds("conn", "/", sourceType="msft", neutralize=True) spExplicit = _ds("sp", "/", sourceType="sharepointFolder", neutralize=False) @@ -531,20 +507,20 @@ class TestBuildEffectiveByWorkspaceFds(unittest.TestCase): class TestResolveEffectiveForPath(unittest.TestCase): def test_with_exact_record(self): - root = _ds("r", "/", neutralize=True, scope="mandate", ragIndexEnabled=False) + root = _ds("r", "/", neutralize=True, ragIndexEnabled=False) leaf = _ds("l", "/folder/sub", neutralize=False) allDs = [root, leaf] result = _inheritFlags.resolveEffectiveForPath("conn-1", "sharepointFolder", "/folder/sub", allDs) self.assertEqual(result["effectiveNeutralize"], False) - self.assertEqual(result["effectiveScope"], "mandate") + self.assertNotIn("effectiveScope", result) self.assertEqual(result["effectiveRagIndexEnabled"], False) def test_without_record_inherits_from_ancestor(self): - root = _ds("r", "/", neutralize=True, scope="mandate", ragIndexEnabled=True) + root = _ds("r", "/", neutralize=True, ragIndexEnabled=True) allDs = [root] result = _inheritFlags.resolveEffectiveForPath("conn-1", "sharepointFolder", "/deep/path/file.txt", allDs) self.assertEqual(result["effectiveNeutralize"], True) - self.assertEqual(result["effectiveScope"], "mandate") + self.assertNotIn("effectiveScope", result) self.assertEqual(result["effectiveRagIndexEnabled"], True) def test_without_record_inherits_from_closest_ancestor(self): @@ -559,7 +535,7 @@ class TestResolveEffectiveForPath(unittest.TestCase): allDs: list = [] result = _inheritFlags.resolveEffectiveForPath("conn-1", "sharepointFolder", "/path", allDs) self.assertEqual(result["effectiveNeutralize"], False) - self.assertEqual(result["effectiveScope"], "personal") + self.assertNotIn("effectiveScope", result) self.assertEqual(result["effectiveRagIndexEnabled"], False) def test_connection_root_covers_service_subtree(self): @@ -620,14 +596,15 @@ class TestResolveEffectiveForFds(unittest.TestCase): result = _inheritFlags.resolveEffectiveForFds("fi-1", "*", None, allFds, mode="aggregate") self.assertEqual(result["effectiveRagIndexEnabled"], "mixed") - def test_inheritable_fds_flags_excludes_scope(self): + def test_inheritable_flags_and_fds_flags(self): self.assertIn("ragIndexEnabled", _inheritFlags._INHERITABLE_FDS_FLAGS) self.assertIn("neutralize", _inheritFlags._INHERITABLE_FDS_FLAGS) self.assertNotIn("scope", _inheritFlags._INHERITABLE_FDS_FLAGS) + self.assertNotIn("scope", _inheritFlags._INHERITABLE_FLAGS) # =========================================================================== -# FDS cascade resets RAG (in addition to neutralize and scope) +# FDS cascade resets RAG (in addition to neutralize) # =========================================================================== class TestCascadeResetFdsRag(unittest.TestCase): @@ -678,15 +655,6 @@ class TestVirtualCoordAggregate(unittest.TestCase): ) self.assertEqual(result["effectiveNeutralize"], "mixed") - def test_virtual_folder_mixed_scope(self): - child1 = _ds("c1", "/virtual/a", scope="mandate") - child2 = _ds("c2", "/virtual/b", scope="personal") - allDs = [child1, child2] - result = _inheritFlags.resolveEffectiveForPath( - "conn-1", "sharepointFolder", "/virtual", allDs, mode="aggregate", - ) - self.assertEqual(result["effectiveScope"], "mixed") - def test_virtual_folder_mixed_rag(self): child1 = _ds("c1", "/virtual/a", ragIndexEnabled=True) child2 = _ds("c2", "/virtual/b", ragIndexEnabled=False) diff --git a/tests/unit/services/test_udbNodes.py b/tests/unit/services/test_udbNodes.py index 8048eee8..a7454d85 100644 --- a/tests/unit/services/test_udbNodes.py +++ b/tests/unit/services/test_udbNodes.py @@ -46,10 +46,10 @@ class TestSupportsFlag(unittest.TestCase): self.assertFalse(n.supportsFlag("scope")) self.assertFalse(n.supportsFlag("ragIndexEnabled")) - def test_connection_supports_all_three(self): + def test_connection_supports_neutralize_and_rag(self): n = ConnectionNode("c1", "msft", label="m", parentKey="personalRoot", rec=None) self.assertTrue(n.supportsFlag("neutralize")) - self.assertTrue(n.supportsFlag("scope")) + self.assertFalse(n.supportsFlag("scope")) self.assertTrue(n.supportsFlag("ragIndexEnabled")) def test_fds_table_supports_neutralize_and_rag_but_not_scope(self): @@ -137,7 +137,7 @@ class TestGetEffectiveFlag(unittest.TestCase): def test_ds_walk_inherits_from_authority_root(self): root = { "id": "r", "connectionId": "c", "sourceType": "msft", "path": "/", - "userId": "user-1", "neutralize": True, "scope": None, "ragIndexEnabled": None, + "userId": "user-1", "neutralize": True, "ragIndexEnabled": None, } node = FolderNode( connectionId="c", service="sharepoint", sourceType="sharepointFolder", From 67806e5323c232e38c4c90990673d9e188e19afa Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 3 Jun 2026 10:17:59 +0200 Subject: [PATCH 3/3] fixes deploy --- tests/unit/connectors/test_connectorDbPostgre_pool.py | 3 +++ tests/unit/services/test_buildTree.py | 7 +++++-- tests/unit/services/test_featureDataAgent_schema.py | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/unit/connectors/test_connectorDbPostgre_pool.py b/tests/unit/connectors/test_connectorDbPostgre_pool.py index bb4170e9..99dbab43 100644 --- a/tests/unit/connectors/test_connectorDbPostgre_pool.py +++ b/tests/unit/connectors/test_connectorDbPostgre_pool.py @@ -41,6 +41,7 @@ import psycopg2.errors import pytest from pydantic import Field +import modules.connectors.connectorDbPostgre as _pgmod from modules.connectors.connectorDbPostgre import ( DatabaseConnector, _PoolRegistry, @@ -149,6 +150,7 @@ def liveConnector(): adminConn.close() closeAllPools() + _pgmod._shuttingDown = False connector = DatabaseConnector( dbHost=host, @@ -164,6 +166,7 @@ def liveConnector(): # Teardown: tear pools down, then drop the DB. closeAllPools() + _pgmod._shuttingDown = False adminConn = psycopg2.connect( host=host, port=port, database="postgres", user=user, password=password ) diff --git a/tests/unit/services/test_buildTree.py b/tests/unit/services/test_buildTree.py index d285e867..1f8c8da0 100644 --- a/tests/unit/services/test_buildTree.py +++ b/tests/unit/services/test_buildTree.py @@ -127,7 +127,9 @@ class TestGetChildrenForParents(unittest.TestCase): self.assertEqual(result["bogus|key"], []) def test_top_level_emits_personal_root_first(self): - with patch("modules.interfaces.interfaceDbApp.getRootInterface") as mockRoot: + with patch("modules.interfaces.interfaceDbApp.getRootInterface") as mockRoot, \ + patch("modules.serviceCenter.services.serviceKnowledge._buildTree._personalRootChildrenNodes", return_value=[]), \ + patch("modules.security.rbacCatalog.getCatalogService"): rootIf = MagicMock() rootIf.db.getRecordset.return_value = [] rootIf.getUserMandates.return_value = [] @@ -151,7 +153,8 @@ class TestGetChildrenForParents(unittest.TestCase): def test_top_level_emits_mandate_groups_inline(self): with patch("modules.interfaces.interfaceDbApp.getRootInterface") as mockRoot, \ - patch("modules.security.rbacCatalog.getCatalogService") as mockCatalog: + patch("modules.security.rbacCatalog.getCatalogService") as mockCatalog, \ + patch("modules.serviceCenter.services.serviceKnowledge._buildTree._personalRootChildrenNodes", return_value=[]): rootIf = MagicMock() rootIf.db.getRecordset.return_value = [] userMandate = MagicMock() diff --git a/tests/unit/services/test_featureDataAgent_schema.py b/tests/unit/services/test_featureDataAgent_schema.py index 616f46cc..2b84e08c 100644 --- a/tests/unit/services/test_featureDataAgent_schema.py +++ b/tests/unit/services/test_featureDataAgent_schema.py @@ -20,7 +20,7 @@ asked for the closing balance per period). from __future__ import annotations import asyncio -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -228,6 +228,7 @@ def _buildRegistryWithMockProvider(): provider.browseTable.return_value = {"rows": [], "total": 0, "limit": 50, "offset": 0} provider.queryTable.return_value = {"rows": [], "total": 0, "limit": 50, "offset": 0} provider.aggregateTable.return_value = {"rows": [], "aggregate": "SUM", "field": "x"} + provider.finalizeRowsAsync = AsyncMock(return_value=[]) registry = _buildSubAgentTools( provider=provider, featureInstanceId="fi-test",