diff --git a/app.py b/app.py index c112b869..1fb68bf0 100644 --- a/app.py +++ b/app.py @@ -324,6 +324,14 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"Feature catalog registration failed: {e}") + # Pre-warm service center modules (avoids first-request import latency) + try: + from modules.serviceCenter import preWarm + preWarm() + logger.info("Service center pre-warm completed") + except Exception as e: + logger.warning(f"Service center pre-warm failed (non-critical): {e}") + # Get event user for feature lifecycle (system-level user for background operations) rootInterface = getRootInterface() eventUser = rootInterface.getUserByUsername("event") diff --git a/docs/SERVICE_ARCHITECTURE_DOCUMENTATION.md b/docs/SERVICE_ARCHITECTURE_DOCUMENTATION.md new file mode 100644 index 00000000..b8cdd891 --- /dev/null +++ b/docs/SERVICE_ARCHITECTURE_DOCUMENTATION.md @@ -0,0 +1,318 @@ +# Gateway Service Architecture Documentation + +This document describes the structure, design patterns, and key components of the two service architectures in the gateway: + +1. **`modules/serviceCenter`** — the new service center (context-based DI, RBAC-aware) +2. **`modules/services`** — the legacy services hub (monolithic hub, eager loading) + +--- + +## 1. `modules/serviceCenter` — New Service Center + +### 1.1 File/Folder Structure + +``` +modules/serviceCenter/ +├── __init__.py # Public API: getService, preWarm, registerServiceObjects, can_access_service +├── context.py # ServiceCenterContext dataclass +├── registry.py # Service definitions (CORE_SERVICES, IMPORTABLE_SERVICES, RBAC) +├── resolver.py # Resolution logic, dependency injection, legacy fallback +├── core/ # Core services (internal building blocks, no RBAC) +│ ├── __init__.py +│ ├── serviceUtils/ +│ │ └── mainServiceUtils.py +│ ├── serviceSecurity/ +│ │ └── mainServiceSecurity.py +│ └── serviceStreaming/ +│ └── mainServiceStreaming.py +└── services/ # Feature-facing importable services (RBAC-protected) + ├── __init__.py + ├── serviceAi/ + ├── serviceBilling/ + ├── serviceChat/ + ├── serviceExtraction/ + ├── serviceGeneration/ + ├── serviceMessaging/ + ├── serviceNeutralization/ + ├── serviceSharepoint/ + ├── serviceTicket/ + └── serviceWeb/ +``` + +**Design distinction:** +- **Core services** — Internal utilities (utils, security, streaming). Never directly requested by features. No RBAC. Security and streaming are fully migrated; legacy hub delegates to service center. +- **Importable services** — Feature-facing. Have `objectKey` for RBAC (e.g. `service.web`, `service.extraction`). + +--- + +### 1.2 Service Registration and Discovery + +**Registration:** Services are declared statically in `registry.py`. + +- **CORE_SERVICES**: Internal services with `module`, `class`, `dependencies`. +- **IMPORTABLE_SERVICES**: Feature-facing services with `module`, `class`, `dependencies`, `objectKey`, `label`. +- **SERVICE_RBAC_OBJECTS**: Derived from IMPORTABLE_SERVICES for catalog registration. + +**Discovery:** No dynamic discovery. All services are explicitly listed in the registry. Adding a service requires editing `registry.py`. + +```python +# Example from registry.py +IMPORTABLE_SERVICES = { + "ai": { + "module": "modules.serviceCenter.services.serviceAi.mainServiceAi", + "class": "AiService", + "dependencies": ["chat", "utils", "extraction", "billing"], + "objectKey": "service.ai", + "label": {"en": "AI", "de": "KI", "fr": "IA"}, + }, + # ... +} +``` + +--- + +### 1.3 Dependency Injection and Factory Patterns + +**Constructor pattern:** Services receive two arguments from the resolver: +1. `context: ServiceCenterContext` — user, mandate_id, feature_instance_id, workflow +2. `get_service: Callable[[str], Any]` — function to resolve other services by key + +```python +# Service Center service constructor +def __init__(self, context, get_service: Callable[[str], Any]): + self._context = context + self._get_service = get_service +``` + +**Dependency resolution:** +- The resolver (`resolver.py`) builds a `get_service` callable that recursively resolves dependencies. +- Dependencies are declared in the registry (e.g. `"dependencies": ["chat", "utils", "extraction", "billing"]`). +- Circular dependencies are detected and raise `RuntimeError`. +- Resolution is cached per `(user, mandate_id, feature_instance_id)` + `key`. + +**Legacy fallback:** If a service fails to load from the service center, the resolver falls back to the legacy `Services` hub when `legacy_hub` is provided. + +--- + +### 1.4 Main Entry Points and Usage Patterns + +| Entry Point | Purpose | +|-------------|---------| +| `getService(key, context, legacy_hub=None)` | Resolve a service by key for the given context | +| `preWarm(service_keys=None)` | Pre-load service modules at startup (called in `app.py` lifespan) | +| `registerServiceObjects(catalogService)` | Register service RBAC objects (called via `registerAllFeaturesInCatalog`) | +| `can_access_service(user, rbac, service_key, ...)` | RBAC check for service access | +| `ServiceCenterContext(user, mandate_id, feature_instance_id, workflow)` | Context dataclass | + +**Typical usage (chatbot feature):** +```python +from modules.serviceCenter import getService +from modules.serviceCenter.context import ServiceCenterContext + +ctx = ServiceCenterContext(user=user, mandate_id=mandateId, feature_instance_id=featureInstanceId, workflow=workflow) +ai_service = getService("ai", ctx, legacy_hub=None) +chat_service = getService("chat", ctx, legacy_hub=None) +``` + +**Feature-level hub (e.g. chatbot):** +- `getChatbotServices()` builds a lightweight hub with only required services. +- Uses `REQUIRED_SERVICES` to resolve only `chat`, `ai`, `billing`, `streaming`. +- Returns a `_ChatbotServiceHub` object with `chat`, `ai`, `billing`, `streaming`, `interfaceDbComponent`, etc. + +--- + +### 1.5 Initialization and Bootstrapping + +1. **`app.py` lifespan:** + - `registerAllFeaturesInCatalog(catalogService)` → calls `registerServiceObjects(catalogService)` for service RBAC objects + - `preWarm()` — imports all service modules to avoid first-request latency + +2. **`registerAllFeaturesInCatalog` (modules/system/registry.py):** + - Registers system RBAC objects + - Registers service center RBAC objects via `registerServiceObjects` + - Registers feature RBAC objects + +3. **First request:** + - `getService(key, ctx)` → `resolve()` loads module, instantiates class with `(context, get_service)`, caches instance + +--- + +## 2. `modules/services` — Legacy Services Hub + +### 2.1 File/Folder Structure + +``` +modules/services/ +├── __init__.py # Services class, getInterface(), PublicService, _loadFeatureInterfaces, _loadFeatureServices +├── serviceAi/ +│ └── mainServiceAi.py +├── serviceBilling/ +│ └── mainServiceBilling.py +├── serviceChat/ +│ └── mainServiceChat.py +├── serviceExtraction/ +│ ├── extractors/ +│ ├── chunking/ +│ ├── merging/ +│ ├── subRegistry.py +│ ├── subPipeline.py +│ └── mainServiceExtraction.py +├── serviceGeneration/ +│ ├── paths/ +│ ├── renderers/ +│ └── mainServiceGeneration.py +├── serviceMessaging/ +│ └── mainServiceMessaging.py +├── serviceNormalization/ +│ └── mainServiceNormalization.py +├── serviceSharepoint/ +│ └── mainServiceSharepoint.py +├── serviceStreaming/ +│ ├── eventManager.py +│ ├── helpers.py +│ └── mainServiceStreaming.py +├── serviceTicket/ +│ └── mainServiceTicket.py +├── serviceUtils/ +│ └── mainServiceUtils.py +├── serviceWeb/ +│ └── mainServiceWeb.py +└── serviceSecurity/ + └── mainServiceSecurity.py +``` + +**No core vs. services split.** All services live in flat `serviceX/` directories. + +--- + +### 2.2 Service Registration and Discovery + +**Registration:** Services are **eagerly loaded** in `Services.__init__()` via hardcoded imports. No registry file. + +**Discovery:** +- **Shared services:** Loaded explicitly in `__init__` from `modules/services/serviceX/mainServiceX.py`. +- **Feature services:** Discovered dynamically via `_loadFeatureServices()` — scans `modules/features/*/service*/mainService*.py` and instantiates classes ending with `"Service"`. + +```python +# Shared services — hardcoded in Services.__init__ +from .serviceSharepoint.mainServiceSharepoint import SharepointService +self.sharepoint = PublicService(SharepointService(self)) +from .serviceChat.mainServiceChat import ChatService +self.chat = PublicService(ChatService(self)) +# ... etc. +``` + +--- + +### 2.3 Dependency Injection / Factory Patterns + +**Constructor pattern:** Services receive the entire `Services` hub as their single dependency. + +```python +# Legacy service constructor +def __init__(self, services): + self.services = services +``` + +**No explicit dependency graph.** Services access other services via `self.services.` (e.g. `self.services.interfaceDbComponent`, `self.services.extraction`). All services are loaded at construction time. + +**PublicService proxy:** Services are wrapped in `PublicService(target, functionsOnly=True)` to expose only callable, non-private attributes. Reduces accidental access to internal state. + +**BillingService:** Uses a separate factory `getService(currentUser, mandateId, featureInstanceId, featureCode)` and a module-level cache. Not integrated with the hub’s constructor pattern. + +--- + +### 2.4 Main Entry Points and Usage Patterns + +| Entry Point | Purpose | +|-------------|---------| +| `getInterface(user, workflow, mandateId, featureInstanceId)` | Returns a `Services` instance | +| `Services` | Central hub with all services and interfaces | + +**Typical usage:** +```python +from modules.services import getInterface as getServices + +services = getServices(user, workflow, mandateId=mandateId, featureInstanceId=featureInstanceId) +ai = services.ai +extraction = services.extraction +``` + +**Interfaces loaded at construction:** +- `interfaceDbApp`, `interfaceDbComponent`, `interfaceDbChat`, `rbac` +- Plus dynamically loaded `interfaceFeature*` from feature containers + +--- + +### 2.5 Initialization and Bootstrapping + +1. **No startup bootstrap** — services load on first `getInterface()` call. +2. **Construction flow:** + - `getInterface(user, ...)` → `Services(user, ...)` + - `Services.__init__`: + - Loads DB interfaces (`interfaceDbApp`, `interfaceDbComponent`, `interfaceDbChat`) + - Instantiates all shared services (sharepoint, ticket, chat, utils, security, streaming, ai, extraction, generation, web) + - Calls `_loadFeatureInterfaces()` — discovers `interfaceFeature*.py` in features + - Calls `_loadFeatureServices()` — discovers `service*/mainService*.py` in features, overrides hub attributes + +3. **Feature services:** If a feature defines `serviceAi/mainServiceAi.py`, it overrides `services.ai`. Shared `serviceAi` is only used when no feature override exists. + +--- + +## 3. Side-by-Side Comparison + +| Aspect | Service Center | Legacy Services | +|--------|----------------|-----------------| +| **Location** | `modules/serviceCenter/` | `modules/services/` | +| **Entry point** | `getService(key, context, legacy_hub)` | `getInterface(user, ...)` → `Services` | +| **Constructor** | `(context, get_service)` | `(services)` — full hub | +| **Context** | `ServiceCenterContext` (user, mandate_id, feature_instance_id, workflow) | Full `Services` with interfaces | +| **Dependencies** | Declared in registry, resolved lazily via `get_service("key")` | Via `self.services.` | +| **Loading** | Lazy (on first `getService`), cached per context | Eager (all at construction) | +| **RBAC** | Per-service `objectKey` in registry, `can_access_service()` | Shared via hub `.rbac` | +| **Feature services** | Not applicable — features use `getService(key, ctx)` | Discovered via `_loadFeatureServices()` | +| **Pre-warm** | `preWarm()` in app lifespan | None | +| **Bootstrap** | `registerServiceObjects()` via `registerAllFeaturesInCatalog` | None | + +--- + +## 4. Coexistence and Migration + +- **Service center** can fall back to **legacy hub** when `legacy_hub` is passed to `getService`. +- **Chatbot** uses service center via `getChatbotServices()` and does not use the legacy hub. +- **Billing, routes, teamsbot, commcoach, etc.** still use `modules.services` (e.g. `getInterface`, `getService` from `serviceBilling`). +- **`ServiceCenterContext`** is used when calling `getService`. Features that pass `workflow=None` use a placeholder workflow for billing/featureCode. +- Migration plan: `docs/SERVICE_CENTER_MIGRATION_PLAN.md`. + +--- + +## 5. Service Center Resolver Flow + +``` +getService("ai", ctx, legacy_hub) + → resolve("ai", ctx, cache, resolving, legacy_hub) + → Check cache (cache_key = user_mandate_feature_ai) + → If cache hit: return cached instance + → If miss: + → _load_service_class("modules.serviceCenter.services.serviceAi.mainServiceAi", "AiService") + → Resolve dependencies: chat, utils, extraction, billing (recursive resolve) + → instance = AiService(ctx, get_service) + → cache[cache_key] = instance + → return instance + → On ImportError/ModuleNotFoundError: _get_from_legacy(legacy_hub, "ai") if legacy_hub +``` + +--- + +## 6. Key Files Reference + +| File | Purpose | +|------|---------| +| `serviceCenter/registry.py` | Service definitions, dependency graph, RBAC keys | +| `serviceCenter/resolver.py` | Resolution logic, caching, legacy fallback | +| `serviceCenter/context.py` | `ServiceCenterContext` dataclass | +| `serviceCenter/__init__.py` | `getService`, `preWarm`, `registerServiceObjects`, `can_access_service` | +| `services/__init__.py` | `Services` class, `getInterface`, `PublicService`, feature discovery | +| `system/registry.py` | `registerAllFeaturesInCatalog` (calls `registerServiceObjects`) | +| `app.py` | Lifespan: `preWarm()`, `registerAllFeaturesInCatalog()` | +| `features/chatbot/mainChatbot.py` | Example: `getChatbotServices()` using service center | diff --git a/docs/SERVICE_CENTER_MIGRATION_PLAN.md b/docs/SERVICE_CENTER_MIGRATION_PLAN.md new file mode 100644 index 00000000..450965ea --- /dev/null +++ b/docs/SERVICE_CENTER_MIGRATION_PLAN.md @@ -0,0 +1,217 @@ +# Service Center Migration Plan + +## Overview + +This document describes a **step-by-step plan** to migrate from the old `modules/services` (Services hub) to the new `modules/serviceCenter`. The migration is **incremental**—one feature at a time—with UI-driven testing after each step. + +**Recommended first feature: Chatbot** — it has a clear UI, limited service dependencies, and is already partially using the service center (AI, generation, billing). + +--- + +## Architecture Summary + +### Current State + +| Component | Location | Notes | +|-----------|----------|-------| +| **Service Center** | `modules/serviceCenter/` | New: registry, resolver, context-based DI | +| **Services Hub** | `modules/services/` | Legacy: `getInterface()` → `Services` instance | +| **Chatbot** | `modules/features/chatbot/` | Uses `getServices()` → `.chat`, `.ai` | + +### Service Center vs Legacy Services + +| Aspect | Service Center | Legacy Services | +|--------|----------------|-----------------| +| **Constructor** | `(context: ServiceCenterContext, get_service)` | `(services: Services)` — receives hub | +| **Context** | Minimal: user, mandate_id, feature_instance_id, workflow | Full hub with all interfaces | +| **Dependencies** | Injected via `get_service("key")` | Via `self.services.` | +| **RBAC** | Per-service `objectKey` in registry | Shared via hub | +| **Pre-warm** | `preWarm()` at app startup | Loaded on first use | + +### Services Already Using Service Center (in Services class) + +The `Services` class in `modules/services/__init__.py` already uses `getService()` for: + +- `messaging` +- `ai` +- `generation` +- `billing` + +### Services Still Using Legacy Direct Imports + +- `chat` ← **Target for Phase 1** +- `sharepoint` +- `ticket` +- `utils` +- `security` +- `streaming` +- `extraction` +- `web` + +--- + +## Phase 1: Migrate Chatbot to Use Service Center for Chat + +**Goal:** Switch the Chatbot feature to get the Chat service from Service Center instead of the legacy hub. This validates the full flow with minimal risk. + +### Step 1.1: Switch Services Class to Use Service Center for Chat + +**File:** `modules/services/__init__.py` + +**Change:** Replace the direct ChatService import with `getService("chat", ...)`. + +```python +# BEFORE (line ~126-127): +from .serviceChat.mainServiceChat import ChatService +self.chat = PublicService(ChatService(self)) + +# AFTER: +self.chat = PublicService(getService("chat", _ctx, legacy_hub=self)) +``` + +The `_ctx` (ServiceCenterContext) is already created for messaging/ai/generation. Add `workflow=self.workflow` to the context if not already present (it should be—check the existing `_ctx` creation around line 109–116). + +**Verification:** +1. Ensure `ServiceCenterContext` includes `workflow` when Services has one (chatbot often passes `workflow=None` initially). +2. The service center ChatService gets `interfaceDbComponent` from `getInterface(context.user, mandateId=context.mandate_id)` — same as legacy. The chatbot calls `getFileInfo(fileId)` which only needs `interfaceDbComponent`, not workflow. + +### Step 1.2: Ensure Service Center Context Has Workflow + +**File:** `modules/services/__init__.py` + +Verify the existing context creation: + +```python +_ctx = ServiceCenterContext( + user=self.user, + mandate_id=self.mandateId, + feature_instance_id=self.featureInstanceId, + workflow=self.workflow, +) +``` + +If `workflow` is missing, add it. The ChatService uses `_context.workflow` for methods like `getChatDocumentsFromDocumentList`; for `getFileInfo` it is not needed. + +### Step 1.3: Run Unit Tests + +```powershell +cd c:\Users\IdaDittrich\Documents\01_Code\gateway +pytest tests/unit/serviceCenter/test_service_center_imports.py -v +python tests/scripts/smoke_test_service_center.py +``` + +### Step 1.4: Manual UI Test — Chatbot with File Upload + +1. **Start the gateway:** + ```powershell + cd c:\Users\IdaDittrich\Documents\01_Code\gateway + uvicorn app:app --reload --host 0.0.0.0 --port 8000 + ``` + +2. **Start the frontend** (if using frontend_nyla): + ```powershell + cd c:\Users\IdaDittrich\Documents\01_Code\frontend_nyla + npm run dev + ``` + +3. **Log in** as a user with access to the Chatbot feature. + +4. **Open a Chatbot instance** (navigate to the chatbot feature, select or create an instance). + +5. **Create a new conversation** — click "New conversation" or equivalent. + +6. **Attach a file** — upload a PDF or document before sending. + +7. **Send a message** — e.g. "Summarize this document." + +8. **Verify:** + - No 500 errors in gateway logs + - File is processed (chat service’s `getFileInfo` is used when creating `ChatbotDocument`s) + - AI response streams back correctly (AI service already from service center) + +### Step 1.5: Rollback if Needed + +If something breaks, revert the change in `modules/services/__init__.py`: + +```python +from .serviceChat.mainServiceChat import ChatService +self.chat = PublicService(ChatService(self)) +``` + +--- + +## Phase 2 (Future): Migrate Extraction for Chatbot + +The chatbot may use extraction when processing documents. After Phase 1 is stable: + +1. Switch `Services` to use `getService("extraction", _ctx, legacy_hub=self)` instead of direct import. +2. Ensure `ExtractionService` in service center has the same interface as the legacy one. +3. Re-test chatbot with document-heavy prompts. + +--- + +## Phase 3 (Future): Migrate Remaining Services + +| Service | Used By | Priority | +|---------|---------|----------| +| utils | Chat, Extraction, AI, Web, Generation | High (core) | +| security | Sharepoint | Medium | +| streaming | Workflows, Chatbot SSE | Medium | +| sharepoint | Sharepoint workflows | Medium | +| ticket | Ticket system | Low | +| web | Web research workflows | Medium | + +--- + +## Service Center Bootstrap (Already Done) + +The app already: +- Calls `preWarm()` at startup (`app.py` lifespan) +- Has `registerServiceObjects()` available for RBAC catalog (call from bootstrap if needed) + +### Optional: Register Service RBAC Objects + +If you want service-level RBAC (e.g. `can_access_service()`), call during bootstrap: + +```python +# In app.py lifespan or interfaceBootstrap +from modules.serviceCenter import registerServiceObjects +from modules.security.rbacCatalog import getCatalogService +catalogService = getCatalogService() +registerServiceObjects(catalogService) +``` + +--- + +## Testing Checklist (Chatbot Phase 1) + +- [ ] Unit tests pass: `pytest tests/unit/serviceCenter/ -v` +- [ ] Smoke test passes: `python tests/scripts/smoke_test_service_center.py` +- [ ] Gateway starts without import errors +- [ ] Chatbot UI loads +- [ ] New conversation creates successfully +- [ ] Message without file sends and gets AI response +- [ ] Message with file attachment sends and gets AI response +- [ ] No errors in gateway logs during the above flows + +--- + +## File Summary for Phase 1 + +| File | Action | +|------|--------| +| `modules/services/__init__.py` | Replace `ChatService` import with `getService("chat", _ctx, legacy_hub=self)` | +| (No other changes) | Service center ChatService and resolver already support legacy fallback | + +--- + +## FAQ + +**Q: Why start with Chat instead of Utils?** +A: Chat has a clear UI path (chatbot) and only a few call sites. Utils is used everywhere; migrating it later reduces risk. + +**Q: What if `getService("chat", ctx)` fails?** +A: The resolver passes `legacy_hub=self`, so it falls back to the legacy `Services.chat` if the service center module fails to load. You get graceful degradation. + +**Q: Can I test without the frontend?** +A: Yes. Use the API directly, e.g. `POST /api/chatbot/{instanceId}/start/stream` with a valid `UserInputRequest` (with `listFileId` for file upload). diff --git a/docs/SERVICE_CENTER_VS_LEGACY_COMPARISON.md b/docs/SERVICE_CENTER_VS_LEGACY_COMPARISON.md new file mode 100644 index 00000000..d017503e --- /dev/null +++ b/docs/SERVICE_CENTER_VS_LEGACY_COMPARISON.md @@ -0,0 +1,92 @@ +# Service Center vs Legacy Services Hub — Comparison & Assessment + +## Executive Summary + +The **Service Center** (`modules/serviceCenter`) is a superior architecture compared to the legacy **Services Hub** (`modules/services`). It was worthwhile to create it. The main benefits are: **explicit dependency graph**, **lazy loading**, **per-service RBAC**, and **context-scoped resolution** without carrying the entire hub. The legacy hub remains valid for incremental migration and backward compatibility. + +--- + +## 1. Architecture Comparison + +| Aspect | Service Center | Legacy Services Hub | +|--------|----------------|---------------------| +| **Location** | `modules/serviceCenter/` | `modules/services/` | +| **Entry point** | `getService(key, context, legacy_hub)` | `getInterface(user, ...)` → `Services` | +| **Constructor** | `(context, get_service)` | `(services)` — full hub | +| **Dependencies** | Declared in registry, resolved lazily via `get_service("key")` | Via `self.services.` — all services always present | +| **Loading** | **Lazy** — only requested services + deps | **Eager** — everything at construction | +| **RBAC** | Per-service `objectKey`, `can_access_service()` | Shared via hub `.rbac` | +| **Caching** | Per-context cache (user + mandate + featureInstance) | No instance cache — new `Services` each call | +| **Feature override** | N/A — features use `getService` directly | Feature services override hub attributes | +| **Pre-warm** | `preWarm()` at app startup | None | +| **Structure** | Core vs importable split; explicit registry | Flat `serviceX/` dirs; discovery via glob | + +--- + +## 2. Which Setup is Better? + +**Service Center is better** for these reasons: + +### 2.1 Explicit Dependency Graph +- Dependencies are declared in `registry.py` (e.g. `"ai": {"dependencies": ["chat", "utils", "extraction", "billing"]}`). +- Circular dependencies are detected and raise `RuntimeError`. +- Easier to reason about and refactor. + +### 2.2 Lazy Loading & Resource Efficiency +- Only requested services (and their transitive deps) are loaded. +- A feature like chatbot needs `chat`, `ai`, `billing`, `streaming` — not `sharepoint`, `ticket`, `neutralization`, etc. +- Legacy hub loads **everything** on first `getInterface()`. + +### 2.3 Context-Scoped Resolution +- Each request gets a `ServiceCenterContext` (user, mandate_id, feature_instance_id, workflow). +- Resolution is cached per context. Same user+mandate+feature → same instances. +- No need to pass or construct a full hub. + +### 2.4 Per-Service RBAC +- Services have `objectKey` (e.g. `service.ai`, `service.extraction`). +- `can_access_service(user, rbac, service_key)` checks before resolving. +- Finer-grained control than a single hub-level RBAC. + +### 2.5 Separation of Concerns +- **Core services** (utils, security, streaming): internal, no RBAC. +- **Importable services** (ai, billing, extraction, etc.): feature-facing, RBAC-protected. +- Clear distinction vs. flat structure in legacy. + +### 2.6 Pre-warm for Cold Start +- `preWarm()` imports all service modules at startup. +- First request avoids import latency. +- Legacy has no equivalent. + +--- + +## 3. When Legacy Still Makes Sense + +- **Migration**: Features that haven’t moved yet still use `getInterface()`. +- **Feature overrides**: Feature-specific services (e.g. `serviceAi/mainServiceAi.py` in a feature) that override hub attributes. +- **Backward compatibility**: `legacy_hub` fallback in Service Center allows gradual migration. + +--- + +## 4. Did It Make Sense to Create the Service Center? + +**Yes.** The legacy hub has inherent limitations: + +1. **Monolithic hub** — every `getInterface()` constructs a full `Services` object with all services, interfaces, and feature discovery. +2. **Implicit dependencies** — services grab what they need via `self.services.`, leading to hidden coupling. +3. **No explicit RBAC per service** — access control is at the hub level. +4. **Eager loading** — every request pays for all services even when only a few are used. + +Service Center addresses these while keeping a migration path via `legacy_hub` fallback. The Chatbot feature already uses it successfully. + +--- + +## 5. Benchmark Script + +Run the comparison script to measure runtime and memory: + +```bash +# From gateway root +python tests/benchmarks/benchmark_service_center_vs_legacy.py +``` + +See `tests/benchmarks/benchmark_service_center_vs_legacy.py` for details on metrics and methodology. diff --git a/modules/features/chatbot/chatbot.py b/modules/features/chatbot/chatbot.py index e91e4f99..431248ec 100644 --- a/modules/features/chatbot/chatbot.py +++ b/modules/features/chatbot/chatbot.py @@ -34,7 +34,6 @@ from modules.features.chatbot.bridges.tools import ( create_tavily_search_tool, create_send_streaming_message_tool, ) -from modules.services.serviceStreaming import ChatStreamingHelper from modules.datamodels.datamodelUam import User if TYPE_CHECKING: @@ -585,6 +584,7 @@ class Chatbot: workflow_id: str = "default" config: Optional["ChatbotConfig"] = None _event_manager: Any = None + _chat_streaming_helper: Any = None # From service center streaming service @classmethod async def create( @@ -596,6 +596,7 @@ class Chatbot: config: Optional["ChatbotConfig"] = None, event_manager=None, planner_model: Optional[AICenterChatModel] = None, + chat_streaming_helper=None, ) -> "Chatbot": """Factory method to create and configure a Chatbot instance. @@ -607,6 +608,7 @@ class Chatbot: config: Optional chatbot configuration for dynamic tool enablement. event_manager: Optional event manager for streaming (passed from route). planner_model: Optional fast model for planner/routing (default: same as model). + chat_streaming_helper: ChatStreamingHelper from service center streaming service. Returns: A configured Chatbot instance. @@ -619,6 +621,7 @@ class Chatbot: config=config, _event_manager=event_manager, planner_model=planner_model, + _chat_streaming_helper=chat_streaming_helper, ) configured_tools = await instance._configure_tools() instance._tools = configured_tools @@ -1244,10 +1247,11 @@ class Chatbot: if etype == "on_chain_end" and _is_root(event): output_obj = edata.get("output") - # Extract message list from the graph's final output - final_msgs = ChatStreamingHelper.extract_messages_from_output( - output_obj=output_obj - ) + # Extract message list from the graph's final output (ChatStreamingHelper from service center) + helper = self._chat_streaming_helper + if not helper: + raise RuntimeError("ChatStreamingHelper required; pass chat_streaming_helper to Chatbot.create()") + final_msgs = helper.extract_messages_from_output(output_obj=output_obj) # Normalize for the frontend (only user/assistant with text content) # Exclude planner-only and SQL-path intermediate messages from chat display @@ -1255,9 +1259,9 @@ class Chatbot: chat_history_payload: List[dict] = [] for m in final_msgs: if isinstance(m, BaseMessage): - d = ChatStreamingHelper.message_to_dict(msg=m) + d = helper.message_to_dict(msg=m) elif isinstance(m, dict): - d = ChatStreamingHelper.dict_message_to_dict(obj=m) + d = helper.dict_message_to_dict(obj=m) else: continue if d.get("role") not in ("user", "assistant") or not d.get("content"): diff --git a/modules/features/chatbot/mainChatbot.py b/modules/features/chatbot/mainChatbot.py index 766130b2..2ddb71c9 100644 --- a/modules/features/chatbot/mainChatbot.py +++ b/modules/features/chatbot/mainChatbot.py @@ -48,12 +48,25 @@ RESOURCE_OBJECTS = [ }, ] -# Service requirements for chatbot — resolved via service center +# Service requirements - services this feature needs from the service center +# Format: [{serviceKey, meta}]. Used by getChatbotServices() to resolve only needed services. REQUIRED_SERVICES = [ - {"serviceKey": "chat", "meta": {"usage": "File info, document handling"}}, - {"serviceKey": "ai", "meta": {"usage": "AI calls, conversation name generation"}}, - {"serviceKey": "billing", "meta": {"usage": "Usage tracking, balance checks"}}, - {"serviceKey": "streaming", "meta": {"usage": "Event manager, ChatStreamingHelper"}}, + { + "serviceKey": "chat", + "meta": {"usage": "File info, document handling"} + }, + { + "serviceKey": "ai", + "meta": {"usage": "AI calls, conversation name generation"} + }, + { + "serviceKey": "billing", + "meta": {"usage": "Usage tracking, balance checks"} + }, + { + "serviceKey": "streaming", + "meta": {"usage": "Event manager, ChatStreamingHelper"} + }, ] # Template roles for this feature @@ -123,6 +136,108 @@ def getFeatureDefinition() -> Dict[str, Any]: } +def getRequiredServiceKeys() -> List[str]: + """Return list of service keys this feature requires.""" + return [s["serviceKey"] for s in REQUIRED_SERVICES] + + +def getChatbotServices( + user, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, + workflow=None, +) -> Any: + """ + Get a service hub for the chatbot feature using the service center. + Resolves only the services declared in REQUIRED_SERVICES. + + Returns a hub-like object with: chat, ai, billing, streaming, + plus interfaceDbComponent, user, mandateId, featureInstanceId. + """ + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface + + # Provide workflow or placeholder so billing/etc get featureCode + _workflow = workflow + if _workflow is None: + _workflow = type("_Placeholder", (), {"featureCode": FEATURE_CODE})() + ctx = ServiceCenterContext( + user=user, + mandate_id=mandateId, + feature_instance_id=featureInstanceId, + workflow=_workflow, + ) + + hub = _ChatbotServiceHub() + hub.user = user + hub.mandateId = mandateId + hub.featureInstanceId = featureInstanceId + hub.workflow = workflow + hub.interfaceDbComponent = getComponentInterface(user, mandateId=mandateId, featureInstanceId=featureInstanceId) + + for spec in REQUIRED_SERVICES: + key = spec["serviceKey"] + try: + svc = getService(key, ctx, legacy_hub=None) + setattr(hub, key, svc) + except Exception as e: + logger.warning(f"Could not resolve service '{key}' for chatbot: {e}") + setattr(hub, key, None) + + return hub + + +def getChatStreamingHelper(): + """ + Get ChatStreamingHelper utility class (used by chatbot for message normalization). + Resolves via service center streaming service. + """ + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + # Minimal context - streaming service only needs it for resolver + ctx = ServiceCenterContext(user=__get_placeholder_user(), mandate_id=None, feature_instance_id=None) + streaming = getService("streaming", ctx, legacy_hub=None) + return streaming.getChatStreamingHelper() if streaming else None + + +def __get_placeholder_user(): + """Placeholder user for contexts that only need service resolution (e.g. ChatStreamingHelper).""" + from modules.datamodels.datamodelUam import User + return User(id="system", email="system@placeholder", firstName="System", lastName="Placeholder") + + +def getEventManager(user, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None): + """ + Get the global event manager for SSE streaming (used by chatbot routes). + """ + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + + ctx = ServiceCenterContext( + user=user, + mandate_id=mandateId, + feature_instance_id=featureInstanceId, + ) + streaming = getService("streaming", ctx, legacy_hub=None) + return streaming.getEventManager() + + +class _ChatbotServiceHub: + """Lightweight hub exposing only services required by the chatbot feature.""" + user = None + mandateId = None + featureInstanceId = None + workflow = None + interfaceDbComponent = None + chat = None + ai = None + billing = None + streaming = None + featureCode = "chatbot" + allowedProviders = None + + def getUiObjects() -> List[Dict[str, Any]]: """Return UI objects for RBAC catalog registration.""" return UI_OBJECTS diff --git a/modules/features/chatbot/routeFeatureChatbot.py b/modules/features/chatbot/routeFeatureChatbot.py index b85b45bc..1775a253 100644 --- a/modules/features/chatbot/routeFeatureChatbot.py +++ b/modules/features/chatbot/routeFeatureChatbot.py @@ -31,7 +31,7 @@ from modules.features.chatbot.interfaceFeatureChatbot import ChatbotConversation # Import chatbot feature from modules.features.chatbot import chatProcess -from modules.services.serviceStreaming import get_event_manager +from modules.features.chatbot.mainChatbot import getEventManager # Pre-warm AI connectors when this router loads (before first request). # Ensures connectors are ready; avoids 4–8 s delay on first chatbot message. @@ -250,7 +250,7 @@ async def stream_chatbot_start( # Validate instance access mandateId = _validateInstanceAccess(instanceId, context) - event_manager = get_event_manager() + event_manager = getEventManager(context.user, mandateId=mandateId, featureInstanceId=instanceId) try: # Use workflowId from query parameter if provided, otherwise from request body @@ -462,7 +462,7 @@ async def stop_chatbot( ) -> ChatbotConversation: """Stops a running chatbot workflow.""" # Validate instance access - _validateInstanceAccess(instanceId, context) + mandateId = _validateInstanceAccess(instanceId, context) try: # Get chatbot interface with instance context @@ -489,7 +489,7 @@ async def stop_chatbot( "lastActivity": getUtcTimestamp() }) - event_manager = get_event_manager() + event_manager = getEventManager(context.user, mandateId=mandateId, featureInstanceId=instanceId) # Store log entry (createLog emits when event_manager is provided) interfaceDbChat.createLog({ "id": f"log_{uuid.uuid4()}", diff --git a/modules/features/chatbot/service.py b/modules/features/chatbot/service.py index a1c2343c..194f9afa 100644 --- a/modules/features/chatbot/service.py +++ b/modules/features/chatbot/service.py @@ -91,15 +91,17 @@ async def chatProcess( ChatbotConversation instance """ try: - # Get services from service center (only chat, ai, billing, streaming — avoids ~90ms legacy hub) + # Get services from service center (only services declared in mainChatbot.REQUIRED_SERVICES) services = getChatbotServices(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) - services.featureCode = 'chatbot' - - # Config and model warm run in background task — return stream ~2–3 s faster for normal feel - chatbot_config = None - - # Reuse hub's interfaceDbChat (ChatObjects) - avoids duplicate DB init - interfaceDbChat = services.interfaceDbChat + + # Load instance config and apply allowedProviders for AI calls (conversation name + main chat) + chatbot_config = await _load_chatbot_config(featureInstanceId) + if chatbot_config.model.allowedProviders: + services.allowedProviders = chatbot_config.model.allowedProviders + logger.info(f"Chatbot instance {featureInstanceId}: restricting to providers {chatbot_config.model.allowedProviders}") + + from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface + interfaceDbChat = getChatbotInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) # Create or load workflow (event_manager passed from route) if workflowId: @@ -161,7 +163,11 @@ async def chatProcess( # Create event queue for new workflow (for streaming) event_manager.create_queue(workflow.id) - + + # Reload workflow to get current message count + workflow = interfaceDbChat.getWorkflow(workflow.id) + services.workflow = workflow # Required for chat service document resolution + # Process uploaded files and create ChatbotDocuments user_documents = [] if userInput.listFileId and len(userInput.listFileId) > 0: @@ -1204,49 +1210,45 @@ def _preflight_billing_check(services, mandateId: str, featureInstanceId: Option """ Pre-flight billing check before starting chatbot AI processing. Raises if mandate has insufficient balance or no providers allowed. + Uses services.billing from service center (REQUIRED_SERVICES). + Exception types from BillingService class (service center billing API). """ - from modules.services.serviceBilling.mainServiceBilling import ( - getService as getBillingService, - InsufficientBalanceException, - ProviderNotAllowedException, - BillingContextError, - ) - user = services.user - featureCode = "chatbot" + from modules.serviceCenter.services.serviceBilling import BillingService + + billingService = services.billing + if not billingService: + raise BillingService.BillingContextError("Billing service not available for chatbot") try: - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) balanceCheck = billingService.checkBalance(0.01) if not balanceCheck.allowed: - raise InsufficientBalanceException( + raise BillingService.InsufficientBalanceException( currentBalance=balanceCheck.currentBalance or 0.0, requiredAmount=0.01, message=f"Ungenuegendes Guthaben. Aktuell: CHF {balanceCheck.currentBalance:.2f}" ) rbacAllowedProviders = billingService.getallowedProviders() if not rbacAllowedProviders: - raise ProviderNotAllowedException( + raise BillingService.ProviderNotAllowedException( provider="any", message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator." ) - except (InsufficientBalanceException, ProviderNotAllowedException): + except (BillingService.InsufficientBalanceException, BillingService.ProviderNotAllowedException): raise except Exception as e: logger.error(f"Billing pre-flight failed: {e}") - raise BillingContextError(f"Billing check failed: {e}") + raise BillingService.BillingContextError(f"Billing check failed: {e}") def _create_chatbot_billing_callback(services, workflow_id: str): """ Create billing callback for AICenterChatModel. Records each AI call to poweron_billing. + Uses services.billing from service center (REQUIRED_SERVICES). """ - from modules.services.serviceBilling.mainServiceBilling import getService as getBillingService from modules.datamodels.datamodelAi import AiCallResponse - user = services.user - mandateId = services.mandateId - featureInstanceId = getattr(services, "featureInstanceId", None) - featureCode = "chatbot" - billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + billingService = services.billing + if not billingService: + return lambda _: None # No-op callback if billing unavailable def _billing_callback(response: AiCallResponse) -> None: if not response or getattr(response, "errorCount", 0) > 0: @@ -1389,6 +1391,11 @@ async def _processChatbotMessageLangGraph( ) # Create chatbot instance with config for dynamic tool configuration + chat_streaming_helper = None + if services.streaming: + chat_streaming_helper = services.streaming.getChatStreamingHelper() + if not chat_streaming_helper: + logger.warning("ChatStreamingHelper not available from streaming service; message normalization may fail") chatbot = await Chatbot.create( model=model, memory=memory, @@ -1397,6 +1404,7 @@ async def _processChatbotMessageLangGraph( config=config, event_manager=event_manager, planner_model=planner_model, + chat_streaming_helper=chat_streaming_helper, ) # Emit synthetic status for real-time UI feedback diff --git a/modules/serviceCenter/__init__.py b/modules/serviceCenter/__init__.py new file mode 100644 index 00000000..4f1a5be5 --- /dev/null +++ b/modules/serviceCenter/__init__.py @@ -0,0 +1,136 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Service Center. +Central registry for core and importable services with per-feature resolution. +""" + +import logging +from typing import Any, List, Optional + +from modules.serviceCenter.context import ServiceCenterContext +from modules.serviceCenter.registry import ( + CORE_SERVICES, + IMPORTABLE_SERVICES, + SERVICE_RBAC_OBJECTS, +) +from modules.serviceCenter.resolver import ( + resolve, + get_resolution_cache, + clear_cache, +) + +logger = logging.getLogger(__name__) + + +def getService( + key: str, + context: ServiceCenterContext, + legacy_hub: Optional[Any] = None, +) -> Any: + """ + Get a service instance by key for the given context. + + Args: + key: Service key (e.g., "web", "extraction", "utils") + context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow + legacy_hub: Optional legacy Services instance for fallback when service not yet migrated + + Returns: + Service instance + """ + cache = get_resolution_cache() + resolving = set() + return resolve(key, context, cache, resolving, legacy_hub=legacy_hub) + + +def preWarm(service_keys: Optional[List[str]] = None) -> None: + """ + Pre-load service modules at startup to avoid first-request latency. + + Args: + service_keys: Optional list of keys to preload. If None, preloads all registered services. + """ + import importlib + + keys = service_keys or list(CORE_SERVICES.keys()) + list(IMPORTABLE_SERVICES.keys()) + for key in keys: + spec = CORE_SERVICES.get(key) or IMPORTABLE_SERVICES.get(key) + if not spec: + continue + try: + importlib.import_module(spec["module"]) + logger.debug(f"Pre-warmed service module: {key}") + except (ImportError, ModuleNotFoundError) as e: + logger.debug(f"Could not pre-warm {key}: {e}") + + +def registerServiceObjects(catalogService) -> bool: + """Register service RBAC objects in the catalog. Call at startup.""" + try: + for obj in SERVICE_RBAC_OBJECTS: + catalogService.registerResourceObject( + featureCode="system", + objectKey=obj["objectKey"], + label=obj["label"], + meta=obj.get("meta"), + ) + logger.info(f"Registered {len(SERVICE_RBAC_OBJECTS)} service RBAC objects") + return True + except Exception as e: + logger.error(f"Failed to register service RBAC objects: {e}") + return False + + +def can_access_service( + user, + rbac, + service_key: str, + mandate_id: Optional[str] = None, + feature_instance_id: Optional[str] = None, + allow_when_no_rbac: bool = True, +) -> bool: + """ + Check if user has permission to access the given service. + + Args: + user: User object + rbac: RbacClass instance (e.g. from interfaceDbApp.rbac) + service_key: Service key (e.g., "web", "extraction") + mandate_id: Optional mandate context + feature_instance_id: Optional feature instance context + allow_when_no_rbac: If True, allow when rbac is None (migration/default) + + Returns: + True if user has view permission on the service + """ + if not rbac: + return allow_when_no_rbac + if service_key not in IMPORTABLE_SERVICES: + return False + obj = IMPORTABLE_SERVICES[service_key] + object_key = obj.get("objectKey") + if not object_key: + return False + from modules.datamodels.datamodelRbac import AccessRuleContext + permissions = rbac.getUserPermissions( + user, + AccessRuleContext.RESOURCE, + object_key, + mandateId=mandate_id, + featureInstanceId=feature_instance_id, + ) + return permissions.view if permissions else False + + +__all__ = [ + "ServiceCenterContext", + "getService", + "preWarm", + "clear_cache", + "registerServiceObjects", + "can_access_service", + "SERVICE_RBAC_OBJECTS", + "CORE_SERVICES", + "IMPORTABLE_SERVICES", +] diff --git a/modules/serviceCenter/context.py b/modules/serviceCenter/context.py new file mode 100644 index 00000000..f9ab0a44 --- /dev/null +++ b/modules/serviceCenter/context.py @@ -0,0 +1,32 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Service Center Context. +Minimal context passed to services: user, mandate, feature instance, workflow. +""" + +from dataclasses import dataclass +from typing import Any, Optional + +from modules.datamodels.datamodelUam import User + + +@dataclass +class ServiceCenterContext: + """Context for service resolution: user, mandate, feature instance, optional workflow.""" + + user: User + mandate_id: Optional[str] = None + feature_instance_id: Optional[str] = None + workflow_id: Optional[str] = None + workflow: Any = None + + @property + def mandateId(self) -> Optional[str]: + """Alias for mandate_id (backward compatibility).""" + return self.mandate_id + + @property + def featureInstanceId(self) -> Optional[str]: + """Alias for feature_instance_id (backward compatibility).""" + return self.feature_instance_id diff --git a/modules/serviceCenter/core/__init__.py b/modules/serviceCenter/core/__init__.py new file mode 100644 index 00000000..752c63b8 --- /dev/null +++ b/modules/serviceCenter/core/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Core services - internal building blocks, not requested by features.""" diff --git a/modules/serviceCenter/core/serviceSecurity/__init__.py b/modules/serviceCenter/core/serviceSecurity/__init__.py new file mode 100644 index 00000000..78f84b42 --- /dev/null +++ b/modules/serviceCenter/core/serviceSecurity/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Security core service.""" + +from .mainServiceSecurity import SecurityService + +__all__ = ["SecurityService"] diff --git a/modules/serviceCenter/core/serviceSecurity/mainServiceSecurity.py b/modules/serviceCenter/core/serviceSecurity/mainServiceSecurity.py new file mode 100644 index 00000000..6495d5d9 --- /dev/null +++ b/modules/serviceCenter/core/serviceSecurity/mainServiceSecurity.py @@ -0,0 +1,81 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Security service for token management operations. +Core service - not requested by features directly. +""" + +import logging +from typing import Optional, Callable, Any + +from modules.datamodels.datamodelSecurity import Token +from modules.auth import TokenManager + +logger = logging.getLogger(__name__) + + +class SecurityService: + """Security service providing token management operations.""" + + def __init__(self, context: Any, get_service: Callable[[str], Any]): + """Initialize with service center context and resolver.""" + self._context = context + self._get_service = get_service + self._tokenManager = TokenManager() + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + self._interfaceDbApp = getAppInterface( + context.user, + mandateId=context.mandate_id, + ) + + def getFreshToken(self, connectionId: str, secondsBeforeExpiry: int = 30 * 60) -> Optional[Token]: + """Get a fresh token for a connection, refreshing when expiring soon.""" + try: + token = self._interfaceDbApp.getConnectionToken(connectionId) + if not token: + return None + return self._tokenManager.ensureFreshToken( + token, + secondsBeforeExpiry=secondsBeforeExpiry, + saveCallback=lambda t: self._interfaceDbApp.saveConnectionToken(t) + ) + except Exception as e: + logger.error(f"getFreshToken: Error fetching or refreshing token for connection {connectionId}: {e}") + return None + + def refreshToken(self, oldToken: Token) -> Optional[Token]: + """Refresh an expired token using the appropriate OAuth service.""" + try: + return self._tokenManager.refreshToken(oldToken) + except Exception as e: + logger.error(f"refreshToken: Error refreshing token: {e}") + return None + + def ensureFreshToken(self, token: Token, *, secondsBeforeExpiry: int = 30 * 60, + saveCallback: Optional[Callable[[Token], None]] = None) -> Optional[Token]: + """Ensure a token is fresh; refresh if expiring within threshold.""" + try: + return self._tokenManager.ensureFreshToken( + token, + secondsBeforeExpiry=secondsBeforeExpiry, + saveCallback=saveCallback + ) + except Exception as e: + logger.error(f"ensureFreshToken: Error ensuring fresh token: {e}") + return None + + def refreshMicrosoftToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]: + """Refresh Microsoft OAuth token using refresh token.""" + try: + return self._tokenManager.refreshMicrosoftToken(refreshToken, userId, oldToken) + except Exception as e: + logger.error(f"refreshMicrosoftToken: Error refreshing Microsoft token: {e}") + return None + + def refreshGoogleToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]: + """Refresh Google OAuth token using refresh token.""" + try: + return self._tokenManager.refreshGoogleToken(refreshToken, userId, oldToken) + except Exception as e: + logger.error(f"refreshGoogleToken: Error refreshing Google token: {e}") + return None diff --git a/modules/serviceCenter/core/serviceStreaming/__init__.py b/modules/serviceCenter/core/serviceStreaming/__init__.py new file mode 100644 index 00000000..7f93d475 --- /dev/null +++ b/modules/serviceCenter/core/serviceStreaming/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Streaming core service for SSE event management.""" + +from .eventManager import EventManager, get_event_manager +from .helpers import ChatStreamingHelper +from .mainServiceStreaming import StreamingService + +__all__ = ["EventManager", "get_event_manager", "ChatStreamingHelper", "StreamingService"] diff --git a/modules/serviceCenter/core/serviceStreaming/eventManager.py b/modules/serviceCenter/core/serviceStreaming/eventManager.py new file mode 100644 index 00000000..ae603fad --- /dev/null +++ b/modules/serviceCenter/core/serviceStreaming/eventManager.py @@ -0,0 +1,158 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Event manager for SSE streaming. +Manages event queues for Server-Sent Events (SSE) streaming across features. +""" + +import logging +import asyncio +from typing import Dict, Optional, Any + +logger = logging.getLogger(__name__) + + +class EventManager: + """ + Manages event queues for SSE streaming. + Each workflow has its own async queue for events. + """ + + def __init__(self): + """Initialize the event manager.""" + self._queues: Dict[str, asyncio.Queue] = {} + self._cleanup_tasks: Dict[str, asyncio.Task] = {} + + def create_queue(self, workflow_id: str) -> asyncio.Queue: + """ + Create an event queue for a workflow. + + Args: + workflow_id: Workflow ID + + Returns: + Async queue for events + """ + if workflow_id not in self._queues: + self._queues[workflow_id] = asyncio.Queue() + logger.debug(f"Created event queue for workflow {workflow_id}") + return self._queues[workflow_id] + + def get_queue(self, workflow_id: str) -> Optional[asyncio.Queue]: + """ + Get the event queue for a workflow. + + Args: + workflow_id: Workflow ID + + Returns: + Async queue if exists, None otherwise + """ + return self._queues.get(workflow_id) + + def has_queue(self, workflow_id: str) -> bool: + """ + Check if a queue exists for a workflow. + + Args: + workflow_id: Workflow ID + + Returns: + True if queue exists, False otherwise + """ + return workflow_id in self._queues + + async def emit_event( + self, + context_id: str, + event_type: str, + data: Dict[str, Any], + event_category: str = "chat", + message: Optional[str] = None, + step: Optional[str] = None + ) -> None: + """ + Emit an event to the queue for a workflow. + + Args: + context_id: Workflow ID (context) + event_type: Type of event (e.g., "chatdata", "complete", "error") + data: Event data dictionary + event_category: Category of event (e.g., "chat", "workflow") + message: Optional message string + step: Optional step identifier + """ + queue = self._queues.get(context_id) + if not queue: + # DEBUG level: This is normal for background workflows without active SSE listener + return + + event = { + "type": event_type, + "data": data, + "category": event_category, + "message": message, + "step": step + } + + try: + await queue.put(event) + logger.debug(f"Emitted {event_type} event for workflow {context_id}") + except Exception as e: + logger.error(f"Error emitting event for workflow {context_id}: {e}", exc_info=True) + + async def cleanup(self, workflow_id: str, delay: float = 60.0) -> None: + """ + Schedule cleanup of a queue after a delay. + + Args: + workflow_id: Workflow ID + delay: Delay in seconds before cleanup + """ + # Cancel existing cleanup task if any + if workflow_id in self._cleanup_tasks: + self._cleanup_tasks[workflow_id].cancel() + + async def _cleanup(): + try: + await asyncio.sleep(delay) + if workflow_id in self._queues: + # Drain remaining events + queue = self._queues[workflow_id] + while not queue.empty(): + try: + queue.get_nowait() + except asyncio.QueueEmpty: + break + + # Remove queue + del self._queues[workflow_id] + logger.info(f"Cleaned up event queue for workflow {workflow_id}") + except asyncio.CancelledError: + logger.debug(f"Cleanup cancelled for workflow {workflow_id}") + except Exception as e: + logger.error(f"Error during cleanup for workflow {workflow_id}: {e}", exc_info=True) + finally: + if workflow_id in self._cleanup_tasks: + del self._cleanup_tasks[workflow_id] + + # Schedule cleanup + task = asyncio.create_task(_cleanup()) + self._cleanup_tasks[workflow_id] = task + + +# Global event manager instance +_event_manager: Optional[EventManager] = None + + +def get_event_manager() -> EventManager: + """ + Get the global event manager instance. + + Returns: + EventManager instance + """ + global _event_manager + if _event_manager is None: + _event_manager = EventManager() + return _event_manager diff --git a/modules/serviceCenter/core/serviceStreaming/helpers.py b/modules/serviceCenter/core/serviceStreaming/helpers.py new file mode 100644 index 00000000..664130ec --- /dev/null +++ b/modules/serviceCenter/core/serviceStreaming/helpers.py @@ -0,0 +1,242 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Streaming helper utilities for chat message processing and normalization.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Mapping, Optional + +from langchain_core.messages import ( + AIMessage, + BaseMessage, + HumanMessage, + SystemMessage, + ToolMessage, +) + +Role = Literal["user", "assistant", "system", "tool"] + + +class ChatStreamingHelper: + """Pure helper methods for streaming and message normalization. + + This class provides static utility methods for converting between different + message formats, extracting content, and normalizing message structures + for streaming chat applications. + """ + + @staticmethod + def role_from_message(*, msg: BaseMessage) -> Role: + """Extract the role from a BaseMessage instance. + + Args: + msg: The BaseMessage instance to extract the role from. + + Returns: + The role as a string literal: "user", "assistant", "system", or "tool". + Defaults to "assistant" if the message type is not recognized. + + Examples: + >>> from langchain_core.messages import HumanMessage + >>> msg = HumanMessage(content="Hello") + >>> ChatStreamingHelper.role_from_message(msg=msg) + 'user' + """ + if isinstance(msg, HumanMessage): + return "user" + if isinstance(msg, AIMessage): + return "assistant" + if isinstance(msg, SystemMessage): + return "system" + if isinstance(msg, ToolMessage): + return "tool" + return getattr(msg, "role", "assistant") + + @staticmethod + def flatten_content(*, content: Any) -> str: + """Convert complex content structures to plain text. + + This method handles various content formats including strings, lists of + content parts, and dictionaries with text fields. It's designed to + normalize content from different message sources into a consistent + plain text format. + + Args: + content: The content to flatten. Can be: + - str: Returned as-is after stripping whitespace + - list: Each item processed and joined with newlines + - dict: Text extracted from "text" or "content" fields + - None: Returns empty string + - Any other type: Converted to string + + Returns: + The flattened content as a plain text string with whitespace stripped. + + Examples: + >>> content = [{"type": "text", "text": "Hello"}, {"type": "text", "text": "world"}] + >>> ChatStreamingHelper.flatten_content(content=content) + 'Hello\nworld' + + >>> content = {"text": "Simple message"} + >>> ChatStreamingHelper.flatten_content(content=content) + 'Simple message' + """ + if content is None: + return "" + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: List[str] = [] + for part in content: + if isinstance(part, dict): + if "text" in part and isinstance(part["text"], str): + parts.append(part["text"]) + elif part.get("type") == "text" and isinstance( + part.get("text"), str + ): + parts.append(part["text"]) + elif "content" in part and isinstance(part["content"], str): + parts.append(part["content"]) + else: + # Fallback for unknown dictionary structures + val = part.get("value") + if isinstance(val, str): + parts.append(val) + else: + parts.append(str(part)) + return "\n".join(p.strip() for p in parts if p is not None) + if isinstance(content, dict): + if "text" in content and isinstance(content["text"], str): + return content["text"].strip() + if "content" in content and isinstance(content["content"], str): + return content["content"].strip() + return str(content).strip() + + @staticmethod + def message_to_dict(*, msg: BaseMessage) -> Dict[str, Any]: + """Convert a BaseMessage instance to a dictionary for streaming output. + + This method normalizes BaseMessage instances into a consistent dictionary + format suitable for JSON serialization and streaming to clients. + + Args: + msg: The BaseMessage instance to convert. + + Returns: + A dictionary containing: + - "role": The message role (user, assistant, system, tool) + - "content": The flattened message content as plain text + - "tool_calls": Tool calls if present (optional) + - "name": Message name if present (optional) + + Examples: + >>> from langchain_core.messages import HumanMessage + >>> msg = HumanMessage(content="Hello there") + >>> result = ChatStreamingHelper.message_to_dict(msg=msg) + >>> result["role"] + 'user' + >>> result["content"] + 'Hello there' + """ + payload: Dict[str, Any] = { + "role": ChatStreamingHelper.role_from_message(msg=msg), + "content": ChatStreamingHelper.flatten_content( + content=getattr(msg, "content", "") + ), + } + tool_calls = getattr(msg, "tool_calls", None) + if tool_calls: + payload["tool_calls"] = tool_calls + name = getattr(msg, "name", None) + if name: + payload["name"] = name + return payload + + @staticmethod + def dict_message_to_dict(*, obj: Mapping[str, Any]) -> Dict[str, Any]: + """Convert a dictionary-shaped message to a normalized dictionary. + + This method handles messages that come from serialized state and are + represented as dictionaries rather than BaseMessage instances. It + normalizes various dictionary formats into a consistent structure. + + Args: + obj: The dictionary-shaped message to convert. Expected to contain + fields like "role", "type", "content", "text", etc. + + Returns: + A normalized dictionary containing: + - "role": The message role (user, assistant, system, tool) + - "content": The flattened message content as plain text + - "tool_calls": Tool calls if present (optional) + - "name": Message name if present (optional) + + Examples: + >>> obj = {"type": "human", "content": "Hello"} + >>> result = ChatStreamingHelper.dict_message_to_dict(obj=obj) + >>> result["role"] + 'user' + >>> result["content"] + 'Hello' + """ + role: Optional[str] = obj.get("role") + if not role: + # Handle alternative type field mappings + typ = obj.get("type") + if typ in ("human", "user"): + role = "user" + elif typ in ("ai", "assistant"): + role = "assistant" + elif typ in ("system",): + role = "system" + elif typ in ("tool", "function"): + role = "tool" + + content = obj.get("content") + if content is None and "text" in obj: + content = obj["text"] + + out: Dict[str, Any] = { + "role": role or "assistant", + "content": ChatStreamingHelper.flatten_content(content=content), + } + if "tool_calls" in obj: + out["tool_calls"] = obj["tool_calls"] + if obj.get("name"): + out["name"] = obj["name"] + return out + + @staticmethod + def extract_messages_from_output(*, output_obj: Any) -> List[Any]: + """Extract messages from LangGraph output objects. + + This method handles various output formats from LangGraph execution, + extracting the messages list from different possible structures. + + Args: + output_obj: The output object from LangGraph execution. Can be: + - An object with a "messages" attribute + - A dictionary with a "messages" key + - Any other object (returns empty list) + + Returns: + A list of extracted messages, or an empty list if no messages + are found or if the output object is None. + + Examples: + >>> output = {"messages": [{"role": "user", "content": "Hello"}]} + >>> messages = ChatStreamingHelper.extract_messages_from_output(output_obj=output) + >>> len(messages) + 1 + """ + if output_obj is None: + return [] + + # Try to parse dicts first + if isinstance(output_obj, dict): + msgs = output_obj.get("messages") + return msgs if isinstance(msgs, list) else [] + + # Then try to get messages attribute + msgs = getattr(output_obj, "messages", None) + return msgs if isinstance(msgs, list) else [] diff --git a/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py b/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py new file mode 100644 index 00000000..8594c0e6 --- /dev/null +++ b/modules/serviceCenter/core/serviceStreaming/mainServiceStreaming.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Streaming service for SSE event management. +Core service - not requested by features directly. +""" + +import logging +from typing import Any, Callable + +from modules.serviceCenter.core.serviceStreaming.eventManager import EventManager, get_event_manager +from modules.serviceCenter.core.serviceStreaming.helpers import ChatStreamingHelper + +logger = logging.getLogger(__name__) + + +class StreamingService: + """Streaming service providing access to SSE event infrastructure.""" + + def __init__(self, context: Any, get_service: Callable[[str], Any]): + """Initialize with service center context and resolver.""" + self._context = context + self._get_service = get_service + + def getEventManager(self) -> EventManager: + """Get the global event manager instance for SSE streaming.""" + return get_event_manager() + + def getChatStreamingHelper(self): + """Get ChatStreamingHelper utility for message normalization (no legacy import at call site).""" + return ChatStreamingHelper diff --git a/modules/serviceCenter/core/serviceUtils/__init__.py b/modules/serviceCenter/core/serviceUtils/__init__.py new file mode 100644 index 00000000..b3661f8d --- /dev/null +++ b/modules/serviceCenter/core/serviceUtils/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Utils core service.""" + +from .mainServiceUtils import UtilsService + +__all__ = ["UtilsService"] diff --git a/modules/serviceCenter/core/serviceUtils/mainServiceUtils.py b/modules/serviceCenter/core/serviceUtils/mainServiceUtils.py new file mode 100644 index 00000000..d22eab1b --- /dev/null +++ b/modules/serviceCenter/core/serviceUtils/mainServiceUtils.py @@ -0,0 +1,185 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Utility service for common operations across the gateway. +Provides centralized access to configuration, events, and other utilities. +Core service - not requested by features directly. +""" + +import logging +from typing import Any, Optional, Dict, Callable, List +from modules.shared.configuration import APP_CONFIG +from modules.shared.eventManagement import eventManager +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared import jsonUtils + +logger = logging.getLogger(__name__) + + +class UtilsService: + """Utility service providing common operations.""" + + def __init__(self, context, get_service: Callable[[str], Any]): + """Initialize with service center context and resolver.""" + self._context = context + self._get_service = get_service + + # ===== Event handling ===== + + def eventRegisterCron(self, job_id: str, func: Callable, cron_kwargs: Dict[str, Any], + replace_existing: bool = True, coalesce: bool = True, + max_instances: int = 1, misfire_grace_time: int = 1800): + """Register a cron job with the event manager.""" + try: + eventManager.registerCron( + jobId=job_id, + func=func, + cronKwargs=cron_kwargs, + replaceExisting=replace_existing, + coalesce=coalesce, + maxInstances=max_instances, + misfireGraceTime=misfire_grace_time + ) + logger.info(f"Registered cron job '{job_id}' with schedule: {cron_kwargs}") + except Exception as e: + logger.error(f"Error registering cron job '{job_id}': {str(e)}") + + def eventRegisterInterval(self, job_id: str, func: Callable, seconds: Optional[int] = None, + minutes: Optional[int] = None, hours: Optional[int] = None, + replace_existing: bool = True, coalesce: bool = True, + max_instances: int = 1, misfire_grace_time: int = 1800): + """Register an interval job with the event manager.""" + try: + eventManager.registerInterval( + jobId=job_id, + func=func, + seconds=seconds, + minutes=minutes, + hours=hours, + replaceExisting=replace_existing, + coalesce=coalesce, + maxInstances=max_instances, + misfireGraceTime=misfire_grace_time + ) + logger.info(f"Registered interval job '{job_id}' (h={hours}, m={minutes}, s={seconds})") + except Exception as e: + logger.error(f"Error registering interval job '{job_id}': {str(e)}") + + def eventRemove(self, job_id: str): + """Remove a scheduled job from the event manager.""" + try: + eventManager.remove(job_id) + logger.info(f"Removed job '{job_id}'") + except Exception as e: + logger.error(f"Error removing job '{job_id}': {str(e)}") + + def configGet(self, key: str, default: Any = None, user_id: str = "system") -> Any: + """Get a configuration value with optional default.""" + try: + return APP_CONFIG.get(key, default, user_id) + except Exception as e: + logger.error(f"Error getting config '{key}': {str(e)}") + return default + + def timestampGetUtc(self) -> float: + """Get current UTC timestamp.""" + try: + return getUtcTimestamp() + except Exception as e: + logger.error(f"Error getting UTC timestamp: {str(e)}") + return 0.0 + + # ===== Debug Tools ===== + + def writeDebugFile(self, content: str, fileType: str, documents: Optional[List] = None) -> None: + """Wrapper to write debug files via shared debugLogger.""" + try: + from modules.shared.debugLogger import writeDebugFile as _writeDebugFile + _writeDebugFile(content, fileType, documents) + except Exception: + pass + + def debugLogToFile(self, message: str, context: str = "DEBUG"): + """Wrapper to log debug messages via shared debugLogger.""" + try: + from modules.shared.debugLogger import debugLogToFile as _debugLogToFile + _debugLogToFile(message, context) + except Exception: + pass + + def storeDebugMessageAndDocuments(self, message, currentUser, mandateId=None, featureInstanceId=None): + """Wrapper to store debug messages and documents via interfaceDbChat.""" + try: + from modules.interfaces.interfaceDbChat import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + _storeDebugMessageAndDocuments(message, currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId) + except Exception: + pass + + def writeDebugArtifact(self, fileName: str, obj: Any): + """Backward-compatible wrapper that now writes via writeDebugFile.""" + try: + import json + if isinstance(obj, (dict, list)): + content = json.dumps(obj, ensure_ascii=False, indent=2) + else: + content = str(obj) + from modules.shared.debugLogger import writeDebugFile as _writeDebugFile + _writeDebugFile(content, fileName) + except Exception: + pass + + # ===== Prompt sanitization ===== + + def sanitizePromptContent(self, content: str, contentType: str = "text") -> str: + """Centralized prompt content sanitization.""" + if not content: + return "" + try: + import re + content_str = str(content) + sanitized = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]', '', content_str) + if contentType == "userinput": + sanitized = sanitized.replace('{', '{{').replace('}', '}}') + sanitized = sanitized.replace('"', '\\"').replace("'", "\\'") + return f"'{sanitized}'" + elif contentType == "json": + sanitized = sanitized.replace('\\', '\\\\').replace('"', '\\"') + sanitized = sanitized.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + elif contentType == "document": + sanitized = sanitized.replace('\\', '\\\\').replace('"', '\\"').replace("'", "\\'") + sanitized = sanitized.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + else: + sanitized = sanitized.replace('\\', '\\\\').replace('"', '\\"').replace("'", "\\'") + sanitized = sanitized.replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + return sanitized + except Exception as e: + logger.error(f"Error sanitizing prompt content: {str(e)}") + return "[ERROR: Content could not be safely sanitized]" + + # ===== JSON utility wrappers ===== + + def jsonStripCodeFences(self, text: str) -> str: + return jsonUtils.stripCodeFences(text) + + def jsonExtractFirstBalanced(self, text: str) -> str: + return jsonUtils.extractFirstBalancedJson(text) + + def jsonNormalizeText(self, text: str) -> str: + return jsonUtils.normalizeJsonText(text) + + def jsonExtractString(self, text: str) -> str: + return jsonUtils.extractJsonString(text) + + def jsonTryParse(self, text) -> tuple: + return jsonUtils.tryParseJson(text) + + # ===== Enum utility functions ===== + + def mapToEnum(self, enum_class, value_str, default_value): + """Map string value to enum.""" + if not value_str: + return default_value + for enum_item in enum_class: + if enum_item.value.lower() == value_str.lower(): + return enum_item + return default_value diff --git a/modules/serviceCenter/registry.py b/modules/serviceCenter/registry.py new file mode 100644 index 00000000..409c72fd --- /dev/null +++ b/modules/serviceCenter/registry.py @@ -0,0 +1,108 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Service Center Registry. +Service definitions, dependency graph, and RBAC object keys. +""" + +from typing import Dict, List, Any + +# Core services: internal building blocks, no RBAC, never requested by features +CORE_SERVICES: Dict[str, Dict[str, Any]] = { + "utils": { + "module": "modules.serviceCenter.core.serviceUtils.mainServiceUtils", + "class": "UtilsService", + "dependencies": [], + }, + "security": { + "module": "modules.serviceCenter.core.serviceSecurity.mainServiceSecurity", + "class": "SecurityService", + "dependencies": [], + }, + "streaming": { + "module": "modules.serviceCenter.core.serviceStreaming.mainServiceStreaming", + "class": "StreamingService", + "dependencies": [], + }, +} + +# Importable services: feature-facing, RBAC-protected +IMPORTABLE_SERVICES: Dict[str, Dict[str, Any]] = { + "ticket": { + "module": "modules.serviceCenter.services.serviceTicket.mainServiceTicket", + "class": "TicketService", + "dependencies": [], + "objectKey": "service.ticket", + "label": {"en": "Ticket System", "de": "Ticket-System", "fr": "Système de tickets"}, + }, + "messaging": { + "module": "modules.serviceCenter.services.serviceMessaging.mainServiceMessaging", + "class": "MessagingService", + "dependencies": [], + "objectKey": "service.messaging", + "label": {"en": "Messaging", "de": "Nachrichten", "fr": "Messagerie"}, + }, + "billing": { + "module": "modules.serviceCenter.services.serviceBilling.mainServiceBilling", + "class": "BillingService", + "dependencies": [], + "objectKey": "service.billing", + "label": {"en": "Billing", "de": "Abrechnung", "fr": "Facturation"}, + }, + "sharepoint": { + "module": "modules.serviceCenter.services.serviceSharepoint.mainServiceSharepoint", + "class": "SharepointService", + "dependencies": ["security"], + "objectKey": "service.sharepoint", + "label": {"en": "SharePoint", "de": "SharePoint", "fr": "SharePoint"}, + }, + "chat": { + "module": "modules.serviceCenter.services.serviceChat.mainServiceChat", + "class": "ChatService", + "dependencies": ["utils"], + "objectKey": "service.chat", + "label": {"en": "Chat", "de": "Chat", "fr": "Chat"}, + }, + "extraction": { + "module": "modules.serviceCenter.services.serviceExtraction.mainServiceExtraction", + "class": "ExtractionService", + "dependencies": ["chat", "utils"], + "objectKey": "service.extraction", + "label": {"en": "Extraction", "de": "Extraktion", "fr": "Extraction"}, + }, + "generation": { + "module": "modules.serviceCenter.services.serviceGeneration.mainServiceGeneration", + "class": "GenerationService", + "dependencies": ["utils", "chat"], + "objectKey": "service.generation", + "label": {"en": "Generation", "de": "Generierung", "fr": "Génération"}, + }, + "ai": { + "module": "modules.serviceCenter.services.serviceAi.mainServiceAi", + "class": "AiService", + "dependencies": ["chat", "utils", "extraction", "billing"], + "objectKey": "service.ai", + "label": {"en": "AI", "de": "KI", "fr": "IA"}, + }, + "web": { + "module": "modules.serviceCenter.services.serviceWeb.mainServiceWeb", + "class": "WebService", + "dependencies": ["ai", "chat", "utils"], + "objectKey": "service.web", + "label": {"en": "Web Research", "de": "Web-Recherche", "fr": "Recherche Web"}, + }, + "neutralization": { + "module": "modules.serviceCenter.services.serviceNeutralization.mainServiceNeutralization", + "class": "NeutralizationService", + "dependencies": ["extraction", "generation"], + "objectKey": "service.neutralization", + "label": {"en": "Neutralization", "de": "Neutralisierung", "fr": "Neutralisation"}, + }, +} + +# RBAC objects for service-level access control (for catalog registration) +SERVICE_RBAC_OBJECTS: List[Dict[str, Any]] = [ + {"objectKey": s["objectKey"], "label": s["label"]} + for s in IMPORTABLE_SERVICES.values() + if "objectKey" in s +] diff --git a/modules/serviceCenter/resolver.py b/modules/serviceCenter/resolver.py new file mode 100644 index 00000000..dd525491 --- /dev/null +++ b/modules/serviceCenter/resolver.py @@ -0,0 +1,170 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Service Center Resolver. +Resolution logic, dependency injection, and optional legacy fallback. +""" + +import importlib +import logging +from typing import Any, Callable, Dict, Optional, Set + +from modules.serviceCenter.context import ServiceCenterContext +from modules.serviceCenter.registry import CORE_SERVICES, IMPORTABLE_SERVICES + +logger = logging.getLogger(__name__) + +# Type for get_service callable passed to services +GetServiceFunc = Callable[[str], Any] + + +def _make_context_id(ctx: ServiceCenterContext) -> str: + """Create a stable cache key from context.""" + return f"{id(ctx.user)}_{ctx.mandate_id or ''}_{ctx.feature_instance_id or ''}" + + +def _load_service_class(module_path: str, class_name: str): + """Load service class from module.""" + module = importlib.import_module(module_path) + return getattr(module, class_name) + + +def _create_legacy_hub(ctx: ServiceCenterContext) -> Any: + """Create legacy Services instance for fallback when service not yet migrated.""" + from modules.services import getInterface + return getInterface( + ctx.user, + workflow=ctx.workflow, + mandateId=ctx.mandate_id, + featureInstanceId=ctx.feature_instance_id, + ) + + +def _get_from_legacy(legacy_hub: Any, key: str) -> Any: + """Map service key to legacy hub attribute (for fallback when service center module fails).""" + key_to_attr = { + "utils": "utils", + "security": "security", + "streaming": "streaming", + "ticket": "ticket", + "messaging": "messaging", + "billing": "billing", + "sharepoint": "sharepoint", + "chat": "chat", + "extraction": "extraction", + "generation": "generation", + "ai": "ai", + "web": "web", + "neutralization": "neutralization", + } + attr = key_to_attr.get(key) + if attr and hasattr(legacy_hub, attr): + return getattr(legacy_hub, attr) + return None + + +def resolve( + key: str, + context: ServiceCenterContext, + cache: Dict[str, Any], + resolving: Set[str], + legacy_hub: Optional[Any] = None, +) -> Any: + """ + Resolve a service by key. Uses cache, resolves dependencies recursively. + Falls back to legacy_hub if service module cannot be loaded. + """ + cache_key = f"{_make_context_id(context)}_{key}" + if cache_key in cache: + return cache[cache_key] + + if key in resolving: + raise RuntimeError(f"Circular dependency detected for service: {key}") + + def get_service(dep_key: str) -> Any: + return resolve(dep_key, context, cache, resolving, legacy_hub) + + # Try core first + if key in CORE_SERVICES: + spec = CORE_SERVICES[key] + try: + cls = _load_service_class(spec["module"], spec["class"]) + resolving.add(key) + try: + for dep in spec.get("dependencies", []): + get_service(dep) + finally: + resolving.discard(key) + instance = cls(context, get_service) + cache[cache_key] = instance + return instance + except (ImportError, ModuleNotFoundError, AttributeError) as e: + logger.debug(f"Could not load core service '{key}' from service center: {e}") + if legacy_hub: + fallback = _get_from_legacy(legacy_hub, key) + if fallback is not None: + cache[cache_key] = fallback + return fallback + raise + + # Try importable + if key in IMPORTABLE_SERVICES: + spec = IMPORTABLE_SERVICES[key] + try: + cls = _load_service_class(spec["module"], spec["class"]) + resolving.add(key) + try: + for dep in spec.get("dependencies", []): + get_service(dep) + finally: + resolving.discard(key) + instance = cls(context, get_service) + cache[cache_key] = instance + return instance + except (ImportError, ModuleNotFoundError, AttributeError) as e: + logger.debug(f"Could not load importable service '{key}' from service center: {e}") + if legacy_hub: + fallback = _get_from_legacy(legacy_hub, key) + if fallback is not None: + cache[cache_key] = fallback + return fallback + raise + + if legacy_hub: + fallback = _get_from_legacy(legacy_hub, key) + if fallback is not None: + cache[cache_key] = fallback + return fallback + + raise KeyError(f"Unknown service: {key}") + + +# Module-level cache for service instances (per context) +_resolution_cache: Dict[str, Any] = {} +_cache_lock: Optional[Any] = None + +try: + from threading import Lock + _cache_lock = Lock() +except ImportError: + pass + + +def get_resolution_cache() -> Dict[str, Any]: + """Get the module-level resolution cache (for preWarm/clear).""" + return _resolution_cache + + +def clear_cache() -> None: + """Clear the resolution cache.""" + lock = _cache_lock if _cache_lock is not None else _DummyLock() + with lock: + _resolution_cache.clear() + + +class _DummyLock: + def __enter__(self): + return self + + def __exit__(self, *args): + pass diff --git a/modules/serviceCenter/services/__init__.py b/modules/serviceCenter/services/__init__.py new file mode 100644 index 00000000..3f161a0f --- /dev/null +++ b/modules/serviceCenter/services/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Importable services - feature-facing, RBAC-protected.""" diff --git a/modules/serviceCenter/services/serviceAi/__init__.py b/modules/serviceCenter/services/serviceAi/__init__.py new file mode 100644 index 00000000..c7f7d39c --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""AI service.""" + +from .mainServiceAi import AiService + +__all__ = ["AiService"] diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py new file mode 100644 index 00000000..08f3ccb3 --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -0,0 +1,1573 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +import json +import logging +import re +import time +import base64 +from typing import Dict, Any, List, Optional, Tuple, Callable +from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument, WorkflowModeEnum +from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent +from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData +from modules.datamodels.datamodelDocument import RenderedDocument +from modules.interfaces.interfaceAiObjects import AiObjects +from modules.shared.jsonUtils import ( + parseJsonWithModel +) +from .subJsonResponseHandling import JsonResponseHandler +from modules.datamodels.datamodelAi import JsonAccumulationState +from modules.serviceCenter.services.serviceBilling.mainServiceBilling import ( + getService as getBillingService, + InsufficientBalanceException, + ProviderNotAllowedException, + BillingContextError +) + +logger = logging.getLogger(__name__) + +# Rebuild the model to resolve forward references +AiCallRequest.model_rebuild() + + +class _ServicesAdapter: + """Adapter providing Services-like interface from (context, get_service).""" + + def __init__(self, context, get_service: Callable[[str], Any]): + self._context = context + self._get_service = get_service + self.user = context.user + self.mandateId = context.mandate_id + self.featureInstanceId = context.feature_instance_id + self.workflow = context.workflow + + @property + def chat(self): + return self._get_service("chat") + + @property + def extraction(self): + return self._get_service("extraction") + + @property + def utils(self): + return self._get_service("utils") + + @property + def ai(self): + return self._get_service("ai") + + @property + def interfaceDbChat(self): + return self._get_service("chat").interfaceDbChat + + @property + def featureCode(self) -> Optional[str]: + w = self.workflow + if w and hasattr(w, "feature") and w.feature: + return getattr(w.feature, "code", None) + return getattr(w, "featureCode", None) if w else None + + def __getattr__(self, name: str): + if name in ("allowedProviders", "preferredProviders", "currentUserLanguage"): + return getattr(self.workflow, name, None) if self.workflow else None + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") + + +class AiService: + """AI service with core operations integrated.""" + + def __init__(self, context, get_service: Callable[[str], Any]) -> None: + """Initialize with ServiceCenterContext and service resolver. + + Args: + context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow + get_service: Callable to resolve dependency services by key + """ + self.services = _ServicesAdapter(context, get_service) + self._get_service = get_service + self.aiObjects = None + self.extractionService = None + + def _initializeSubmodules(self): + """Initialize all submodules after aiObjects is ready.""" + if self.aiObjects is None: + raise RuntimeError("aiObjects must be initialized before initializing submodules") + + if self.extractionService is None: + logger.info("Initializing ExtractionService via service center...") + self.extractionService = self._get_service("extraction") + + # Initialize new submodules + from .subResponseParsing import ResponseParser + from .subDocumentIntents import DocumentIntentAnalyzer + from .subContentExtraction import ContentExtractor + from .subStructureGeneration import StructureGenerator + from .subStructureFilling import StructureFiller + from .subAiCallLooping import AiCallLooper + + if not hasattr(self, 'responseParser'): + logger.info("Initializing ResponseParser...") + self.responseParser = ResponseParser(self.services) + + if not hasattr(self, 'intentAnalyzer'): + logger.info("Initializing DocumentIntentAnalyzer...") + self.intentAnalyzer = DocumentIntentAnalyzer(self.services, self) + + if not hasattr(self, 'contentExtractor'): + logger.info("Initializing ContentExtractor...") + self.contentExtractor = ContentExtractor(self.services, self, self.intentAnalyzer) + + if not hasattr(self, 'structureGenerator'): + logger.info("Initializing StructureGenerator...") + self.structureGenerator = StructureGenerator(self.services, self) + + if not hasattr(self, 'structureFiller'): + logger.info("Initializing StructureFiller...") + self.structureFiller = StructureFiller(self.services, self) + + if not hasattr(self, 'aiCallLooper'): + logger.info("Initializing AiCallLooper...") + self.aiCallLooper = AiCallLooper(self.services, self, self.responseParser) + + async def callAi(self, request: AiCallRequest, progressCallback=None): + """Router: handles content parts via extractionService, text context via interface. + + FAIL-SAFE BILLING at the source: + 1. Pre-flight check: validates billing context is complete (RAISES if not) + 2. Balance & provider check before AI call + 3. billingCallback on aiObjects: records one billing transaction per model call + with exact provider + model name (set before AI call, invoked by _callWithModel) + """ + # SPEECH_TEAMS: Dedicated pipeline, bypasses standard model selection + if request.options and request.options.operationType == OperationTypeEnum.SPEECH_TEAMS: + return await self._handleSpeechTeams(request) + + # FAIL-SAFE: Pre-flight billing validation (like 0 CHF credit card check) + self._preflightBillingCheck() + + # Balance & provider permission checks + await self._checkBillingBeforeAiCall() + + # Calculate effective allowedProviders: RBAC ∩ Workflow + effectiveProviders = self._calculateEffectiveProviders() + if effectiveProviders and request.options: + request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders}) + logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}") + + # Set billing callback on aiObjects BEFORE the AI call + # This callback is invoked by _callWithModel() after EVERY individual model call + # For parallel content parts (e.g., 200 MB doc), each model call creates its own transaction + self.aiObjects.billingCallback = self._createBillingCallback() + + try: + if hasattr(request, 'contentParts') and request.contentParts: + response = await self.extractionService.processContentPartsWithAi( + request, self.aiObjects, progressCallback + ) + else: + response = await self.aiObjects.callWithTextContext(request) + finally: + # Clear callback after call completes + self.aiObjects.billingCallback = None + + # Store workflow stats for analytics + self._storeAiCallStats(response, request) + + return response + + # ========================================================================= + # SPEECH_TEAMS: Dedicated handler for Teams Meeting AI analysis + # Bypasses standard model selection. Uses a fixed fast model. + # ========================================================================= + + async def _handleSpeechTeams(self, request: AiCallRequest): + """ + Dedicated handler for SPEECH_TEAMS operation type. + Bypasses standard model selection and uses a fixed fast model optimized + for low-latency meeting transcript analysis. + + The handler: + 1. Selects a fixed fast model (no model selector) + 2. Builds a specialized system prompt for meeting analysis + 3. Calls the model with structured JSON output + 4. Returns a SpeechTeamsResponse wrapped in AiCallResponse + + Args: + request: AiCallRequest with: + - prompt: User-configured system prompt (from FeatureInstance.config.aiSystemPrompt) + - context: Accumulated transcript segments to analyze + - options.metadata: Optional dict with "botName" key + + Returns: + AiCallResponse with content as JSON string (SpeechTeamsResponse format) + """ + from modules.datamodels.datamodelAi import AiCallResponse, AiModelCall, AiCallOptions, PriorityEnum + + startTime = time.time() + + # Billing pre-flight (SPEECH_TEAMS also needs billing) + self._preflightBillingCheck() + await self._checkBillingBeforeAiCall() + + # 1. Select a fixed fast model (bypass model selector) + model = self._getSpeechTeamsModel() + if not model: + return AiCallResponse( + content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": "No suitable model available for SPEECH_TEAMS", "detectedIntent": "none"}), + modelName="error", + provider="unknown", + priceCHF=0.0, + processingTime=0.0, + bytesSent=0, + bytesReceived=0, + errorCount=1 + ) + + # 2. Build specialized system prompt + metadata = {} + if hasattr(request.options, 'allowedProviders') and request.options.allowedProviders: + # Reuse allowedProviders field as metadata carrier if set (backward compat) + pass + botName = metadata.get("botName", "AI Assistant") + + # Extract botName from context if embedded as header + contextText = request.context or "" + if contextText.startswith("BOT_NAME:"): + lines = contextText.split("\n", 1) + botName = lines[0].replace("BOT_NAME:", "").strip() + contextText = lines[1] if len(lines) > 1 else "" + + userSystemPrompt = request.prompt or "" + systemPrompt = self._buildSpeechTeamsSystemPrompt(userSystemPrompt, botName) + + # 3. Build messages + messages = [ + {"role": "system", "content": systemPrompt}, + {"role": "user", "content": contextText} + ] + + # 4. Call model directly (no failover loop -- single fast model) + modelCall = AiModelCall( + messages=messages, + model=model, + options=AiCallOptions( + operationType=OperationTypeEnum.SPEECH_TEAMS, + priority=PriorityEnum.SPEED, + temperature=0.3, + resultFormat="json" + ) + ) + + # Set billing callback + self.aiObjects.billingCallback = self._createBillingCallback() + + try: + inputBytes = len((systemPrompt + contextText).encode('utf-8')) + modelResponse = await model.functionCall(modelCall) + + if not modelResponse.success: + raise ValueError(f"SPEECH_TEAMS model call failed: {modelResponse.error}") + + content = modelResponse.content + outputBytes = len(content.encode('utf-8')) + processingTime = time.time() - startTime + priceCHF = model.calculatepriceCHF(processingTime, inputBytes, outputBytes) + + response = AiCallResponse( + content=content, + modelName=model.name, + provider=model.connectorType, + priceCHF=priceCHF, + processingTime=processingTime, + bytesSent=inputBytes, + bytesReceived=outputBytes, + errorCount=0 + ) + + # Record billing + if self.aiObjects.billingCallback: + try: + self.aiObjects.billingCallback(response) + except Exception as e: + logger.error(f"BILLING: Failed to record billing for SPEECH_TEAMS: {e}") + + # Store stats + self._storeAiCallStats(response, request) + + logger.info(f"SPEECH_TEAMS call completed: model={model.name}, time={processingTime:.2f}s, cost={priceCHF:.4f} CHF") + return response + + except Exception as e: + processingTime = time.time() - startTime + logger.error(f"SPEECH_TEAMS call failed: {e}") + return AiCallResponse( + content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": f"Error: {str(e)}", "detectedIntent": "none"}), + modelName=model.name if model else "error", + provider=model.connectorType if model else "unknown", + priceCHF=0.0, + processingTime=processingTime, + bytesSent=0, + bytesReceived=0, + errorCount=1 + ) + finally: + self.aiObjects.billingCallback = None + + def _getSpeechTeamsModel(self): + """ + Get the fixed fast model for SPEECH_TEAMS. + Prioritizes: GPT-4o-mini > Claude Haiku > any fast model with DATA_ANALYSE capability. + Returns the AiModel instance or None. + """ + from modules.aicore.aicoreModelRegistry import modelRegistry + + availableModels = modelRegistry.getAvailableModels() + if not availableModels: + logger.error("SPEECH_TEAMS: No models available in registry") + return None + + # Priority list of preferred models for SPEECH_TEAMS (fast + cheap) + _preferredModelNames = [ + "gpt-4o-mini", # OpenAI: fast, cheap, good at JSON + "claude-3-5-haiku", # Anthropic: fast, cheap + "gpt-4o", # OpenAI: fallback to quality model + "claude-sonnet-4-5", # Anthropic: fallback + ] + + # Try preferred models in order + for preferredName in _preferredModelNames: + for model in availableModels: + if preferredName in model.name.lower() and model.functionCall and model.isAvailable: + logger.info(f"SPEECH_TEAMS: Selected preferred model '{model.name}' ({model.displayName})") + return model + + # Fallback: pick fastest available model with DATA_ANALYSE capability + _dataAnalyseModels = [] + for model in availableModels: + if not model.functionCall or not model.isAvailable: + continue + for opRating in model.operationTypes: + if opRating.operationType == OperationTypeEnum.DATA_ANALYSE: + _dataAnalyseModels.append((model, opRating.rating)) + break + + if _dataAnalyseModels: + # Sort by speed rating (descending) then cost (ascending) + _dataAnalyseModels.sort(key=lambda x: (-x[0].speedRating, x[0].costPer1kTokensInput)) + bestModel = _dataAnalyseModels[0][0] + logger.info(f"SPEECH_TEAMS: Selected fallback model '{bestModel.name}' (speed={bestModel.speedRating})") + return bestModel + + # Last resort: first available model + for model in availableModels: + if model.functionCall and model.isAvailable: + logger.warning(f"SPEECH_TEAMS: Using last-resort model '{model.name}'") + return model + + return None + + def _buildSpeechTeamsSystemPrompt(self, userSystemPrompt: str, botName: str) -> str: + """ + Build the specialized system prompt for SPEECH_TEAMS meeting analysis. + Combines a fixed base prompt with user-configurable instructions. + """ + # Extract first name for examples (e.g. "Nyla" from "Nyla Larsson") + botFirstName = botName.split()[0] if " " in botName else botName + + basePrompt = f"""Du bist "{botName}", ein AI-Teilnehmer in einem Microsoft Teams Meeting. +Analysiere das folgende Transkript und entscheide, ob du antworten sollst. + +SPRACHE: Das Transkript kann in verschiedenen Sprachen sein. Antworte immer in der Sprache des letzten Sprechers der dich angesprochen hat. Wenn jemand sagt "let's talk German" oder "sprich deutsch", wechsle die Sprache entsprechend. + +WICHTIG - SPRACHERKENNUNG: Das Transkript stammt aus einer automatischen Spracherkennung (Live Captions). +Dein Name "{botFirstName}" kann VERZERRT transkribiert werden, z.B. als aehnlich klingende Varianten +(z.B. "{botFirstName}" koennte als "Naila", "Neela", "Nila", "Sheila" etc. erscheinen). +Wenn ein Wort im Transkript PHONETISCH AEHNLICH zu "{botFirstName}" klingt und im Kontext einer Anrede steht, bist du gemeint. + +WANN ANTWORTEN: + +REGEL 1 (HOECHSTE PRIORITAET - NUR wenn direkt angesprochen): +Antworte NUR wenn dein Name "{botFirstName}" (oder phonetisch aehnliche Varianten durch Spracherkennung) DIREKT im aktuellsten Transkript-Segment vorkommt. +Beispiele wo du antworten MUSST: "{botFirstName}, was denkst du?", "Hey {botFirstName}", "{botFirstName} please introduce yourself" +Beispiele wo du NICHT antworten darfst: Jemand spricht ueber ein Thema ohne dich zu adressieren. + +REGEL 2 (NUR bei direkter Frage an dich): +Wenn jemand eine Frage DIREKT AN DICH stellt (mit deinem Namen), beantworte sie. +Antworte NICHT auf allgemeine Fragen in der Runde, die nicht an dich gerichtet sind. + +REGEL 3 (NICHT ANTWORTEN - sehr wichtig): +- Wenn Teilnehmer miteinander sprechen ohne dich zu adressieren: NICHT antworten +- Wenn die Konversation nicht an dich gerichtet ist: NICHT antworten +- Wenn du bereits auf dieselbe Frage geantwortet hast: NICHT nochmal antworten +- Wenn du nicht sicher bist ob du gemeint bist: NICHT antworten +- Im Zweifel: shouldRespond = false + +ANTWORT-STIL (wenn du antwortest): +- Direkt und konkret antworten, KEINE Floskeln +- NICHT mit "Hallo [Name]" anfangen wenn du bereits begruessst hast +- NICHT "Ich bin {botName} und ich bin hier um zu helfen" wiederholen +- NICHT frueheres wiederholen das du schon gesagt hast +- Max 1-2 Saetze, praezise auf den Punkt +- Sieh dir an was du (markiert als [YOU]) bereits gesagt hast und wiederhole es NICHT +- KEINE reinen Absichtssaetze wie "Ich werde ...", "Ich kann ...", "Gerne ...". + Liefere direkt den eigentlichen Inhalt in der gleichen Antwort. + +WENN DER USER DICH BITTET ETWAS VORZULESEN / ZUSAMMENZUFASSEN: +- Gib IMMER sofort die Zusammenfassung aus (nicht nur ankündigen). +- Falls Vorlesen gewuenscht ist, setze zusaetzlich ein "readAloud"-Kommando mit dem Text. + +KANAL-AUSWAHL (Voice vs Chat) - Je nach Anfrage unterschiedlich antworten: +- Du kannst pro Anfrage festlegen, ob deine Antwort per Voice (TTS), per Chat, oder beides erfolgt. +- Wenn jemand sagt "schreib das in den Chat", "schreib die Zusammenfassung in den Chat", "poste das im Chat": + - responseChannels: ["voice", "chat"] + - responseTextForVoice: Kurze Bestaetigung (z.B. "Ich schreibe die Zusammenfassung jetzt in den Chat") + - responseTextForChat: Der eigentliche Inhalt (z.B. die vollstaendige Zusammenfassung) +- Wenn jemand sagt "sag mir das", "lies das vor", "sprich das aus": + - responseChannels: ["voice"] oder ["voice","chat"] je nach Kontext + - responseTextForVoice: Der zu sprechende Text +- Wenn jemand sagt "nur im Chat", "schreib nur": responseChannels: ["chat"] +- Wenn keine Kanal-Praeferenz erkennbar: responseChannels weglassen (Config entscheidet), responseText verwenden. + +STOP-ERKENNUNG: +Wenn jemand dich bittet aufzuhoeren, still zu sein, zu stoppen, oder nicht mehr zu reden +(in JEDER Sprache, z.B. "{botFirstName} stop", "{botFirstName} sei still", "{botFirstName} halt", "{botFirstName} be quiet", +"{botFirstName} shut up", "{botFirstName} arrete", etc.), dann setze detectedIntent auf "stop" und +shouldRespond auf false. Du musst NICHT antworten wenn jemand dich stoppt.""" + + # Append user-configured instructions if provided + if userSystemPrompt and userSystemPrompt.strip(): + basePrompt += f"\n\nZUSAETZLICHE ANWEISUNGEN:\n{userSystemPrompt.strip()}" + + basePrompt += f""" + +KOMMANDOS: Du kannst optionale Aktions-Kommandos ausfuehren lassen. +Verfuegbare Kommandos (im "commands" Array): +- {{"action": "toggleTranscript", "params": {{"enable": true/false}}}} — Transkription ein-/ausschalten +- {{"action": "sendChat", "params": {{"text": "Nachricht"}}}} — Zusaetzliche Nachricht in den Chat schreiben (unabhaengig von responseText) +- {{"action": "readAloud", "params": {{"text": "Text zum Vorlesen"}}}} — Einen bestimmten Text vorlesen (unabhaengig von responseText) +- {{"action": "changeLanguage", "params": {{"language": "en-US"}}}} — Kommunikationssprache aendern (z.B. "de-DE", "en-US", "fr-FR") + +Verwende Kommandos NUR wenn explizit darum gebeten wird (z.B. "schalte die Transkription ein", "schreib das in den Chat", "lies das vor", "sprich Englisch"). + +WICHTIG: Antworte IMMER als valides JSON in exakt diesem Format: +{{ + "shouldRespond": true/false, + "responseText": "Deine Antwort hier" oder null (Standard fuer beide Kanäle), + "responseTextForVoice": optional - Text nur fuer TTS/Voice (z.B. kurze Bestaetigung), + "responseTextForChat": optional - Text nur fuer Chat (z.B. lange Zusammenfassung), + "responseChannels": optional - ["voice"], ["chat"] oder ["voice","chat"] je nach User-Anfrage, + "reasoning": "Kurze Begruendung deiner Entscheidung", + "detectedIntent": "addressed" | "question" | "proactive" | "stop" | "none", + "commands": [] oder null +}} + +detectedIntent-Werte: +- "addressed": {botName} wurde direkt angesprochen +- "question": Eine allgemeine Frage wurde gestellt +- "proactive": Du hast einen wertvollen proaktiven Beitrag +- "stop": Der User bittet {botName} aufzuhoeren/still zu sein (in jeder Sprache) +- "none": Kein Handlungsbedarf""" + + return basePrompt + + def _preflightBillingCheck(self) -> None: + """ + Pre-flight billing validation - like a 0 CHF credit card authorization check. + + Validates that ALL required billing context is present and that a billing + transaction CAN be recorded. This dry-run check catches missing context + BEFORE an expensive AI call starts. + + FAIL-SAFE: This method RAISES if billing context is incomplete. + An AI call without billing context MUST NOT proceed. + + Raises: + BillingContextError: If billing context is incomplete or invalid + """ + if not self.services: + raise BillingContextError("No service context available - cannot bill AI call") + + user = getattr(self.services, 'user', None) + if not user: + raise BillingContextError("No user context - cannot bill AI call") + + mandateId = getattr(self.services, 'mandateId', None) + if not mandateId: + raise BillingContextError( + f"No mandateId in service context for user {user.id} - cannot bill AI call. " + "Every AI call MUST have a mandate context for billing." + ) + + # Validate billing service can be created + featureInstanceId = getattr(self.services, 'featureInstanceId', None) + featureCode = getattr(self.services, 'featureCode', None) + + try: + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + except Exception as e: + raise BillingContextError( + f"Cannot create billing service for user {user.id}, mandate {mandateId}: {e}" + ) + + # Dry-run: verify billing service can check balance (DB accessible) + try: + billingService.checkBalance(0.0) + except Exception as e: + raise BillingContextError( + f"Billing system not accessible for mandate {mandateId}: {e}" + ) + + logger.debug( + f"Pre-flight billing check PASSED: user={user.id}, mandate={mandateId}, " + f"feature={featureCode or 'none'}, instance={featureInstanceId or 'none'}" + ) + + async def _checkBillingBeforeAiCall(self) -> None: + """ + Check billing status before making an AI call. + + FAIL-SAFE: Context validation is done in _preflightBillingCheck() which is + called first. This method handles balance and provider permission checks. + + Verifies: + 1. User has sufficient balance (for prepay models) + 2. Provider is allowed for the user (via RBAC) + + Raises: + InsufficientBalanceException: If balance is insufficient + ProviderNotAllowedException: If provider is not allowed + BillingContextError: If billing check fails unexpectedly + """ + # Context is already validated by _preflightBillingCheck() + user = self.services.user + mandateId = self.services.mandateId + featureInstanceId = getattr(self.services, 'featureInstanceId', None) + featureCode = getattr(self.services, 'featureCode', None) + + try: + # Get billing service + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + + # Check balance (estimate typical AI call cost) + estimatedCost = 0.01 # ~1 cent CHF minimum + balanceCheck = billingService.checkBalance(estimatedCost) + + if not balanceCheck.allowed: + balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" + logger.warning( + f"Billing check failed for user {user.id}: " + f"Balance {balance_str} CHF, " + f"Reason: {balanceCheck.reason}" + ) + raise InsufficientBalanceException( + currentBalance=balanceCheck.currentBalance or 0.0, + requiredAmount=estimatedCost, + message=f"Ungenugendes Guthaben. Aktuell: CHF {balance_str}" + ) + + balance_str = f"{(balanceCheck.currentBalance or 0):.2f}" + logger.debug(f"Billing check passed: Balance {balance_str} CHF") + + # Check if at least one provider is allowed (RBAC check) + rbacAllowedProviders = billingService.getallowedProviders() + if not rbacAllowedProviders: + logger.warning(f"No AI providers allowed for user {user.id} in mandate {mandateId}") + raise ProviderNotAllowedException( + provider="any", + message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator." + ) + + # Check automation-level allowedProviders restriction + automationAllowedProviders = getattr(self.services, 'allowedProviders', None) + if automationAllowedProviders: + effectiveProviders = [p for p in automationAllowedProviders if p in rbacAllowedProviders] + if not effectiveProviders: + logger.warning(f"No providers available after automation restriction. " + f"Automation allows: {automationAllowedProviders}, " + f"RBAC allows: {rbacAllowedProviders}") + raise ProviderNotAllowedException( + provider="any", + message="Die konfigurierten AI-Provider dieser Automation sind fuer Ihre Rolle nicht freigegeben." + ) + logger.debug(f"Automation provider check passed: {effectiveProviders}") + + # Check if preferred providers (from UI multiselect) are allowed + preferredProviders = getattr(self.services, 'preferredProviders', None) + if preferredProviders: + for provider in preferredProviders: + if provider not in rbacAllowedProviders: + logger.warning(f"Preferred provider {provider} not allowed for user {user.id}") + raise ProviderNotAllowedException( + provider=provider, + message=f"Der gewaehlte Provider '{provider}' ist fuer Ihre Rolle nicht freigegeben." + ) + logger.debug(f"All preferred providers are allowed: {preferredProviders}") + + logger.debug(f"Provider check passed: {len(rbacAllowedProviders)} providers allowed") + + except InsufficientBalanceException: + raise + except ProviderNotAllowedException: + raise + except BillingContextError: + raise + except Exception as e: + # FAIL-SAFE: Don't silently swallow errors - log at ERROR level + logger.error(f"BILLING FAIL-SAFE: Billing check failed with unexpected error: {e}") + raise BillingContextError(f"Billing check failed: {e}") + + def _createBillingCallback(self): + """ + Create a billing callback for interfaceAiObjects._callWithModel(). + + Returns a function that records one billing transaction per individual model call. + Each transaction contains the exact provider name AND model name. + + For a 200 MB document processed with N parallel AI calls (possibly different models), + this creates N separate billing transactions - one per model call. + """ + user = self.services.user + mandateId = self.services.mandateId + featureInstanceId = getattr(self.services, 'featureInstanceId', None) + featureCode = getattr(self.services, 'featureCode', None) + + # Get workflow ID if available + workflowId = None + workflow = getattr(self.services, 'workflow', None) + if workflow and hasattr(workflow, 'id'): + workflowId = workflow.id + + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + + def _billingCallback(response) -> None: + """Record billing for a single AI model call.""" + if not response or getattr(response, 'errorCount', 0) > 0: + return + + priceCHF = getattr(response, 'priceCHF', 0.0) + if not priceCHF or priceCHF <= 0: + return + + provider = getattr(response, 'provider', None) or 'unknown' + modelName = getattr(response, 'modelName', None) or 'unknown' + + try: + billingService.recordUsage( + priceCHF=priceCHF, + workflowId=workflowId, + aicoreProvider=provider, + aicoreModel=modelName, + description=f"AI: {modelName}" + ) + logger.debug( + f"Billed model call: {priceCHF:.4f} CHF, " + f"provider={provider}, model={modelName}, mandate={mandateId}" + ) + except Exception as e: + logger.error( + f"BILLING: Failed to record transaction! " + f"Cost={priceCHF:.4f} CHF, user={user.id}, mandate={mandateId}, " + f"provider={provider}, model={modelName}, error={e}" + ) + + return _billingCallback + + def _calculateEffectiveProviders(self) -> Optional[List[str]]: + """ + Calculate effective allowed providers: RBAC ∩ Workflow. + + RBAC is master - only RBAC-permitted providers can ever be used. + If workflow specifies allowedProviders, intersect with RBAC. + If no workflow providers, use all RBAC-permitted providers. + + Returns: + List of effective allowed providers, or None if no filtering needed + """ + try: + user = getattr(self.services, 'user', None) + mandateId = getattr(self.services, 'mandateId', None) + + if not user or not mandateId: + return None + + # Get RBAC-permitted providers (master list) + # Note: getBillingService is imported at module level from mainServiceBilling + featureInstanceId = getattr(self.services, 'featureInstanceId', None) + featureCode = getattr(self.services, 'featureCode', None) + billingService = getBillingService(user, mandateId, featureInstanceId, featureCode) + rbacProviders = billingService.getallowedProviders() + + if not rbacProviders: + logger.warning("No RBAC-permitted providers found") + return None + + # Get workflow-specified providers (optional filter) + workflowProviders = getattr(self.services, 'allowedProviders', None) + + if workflowProviders: + # Intersect: only providers that are both RBAC-permitted AND workflow-allowed + effectiveProviders = [p for p in workflowProviders if p in rbacProviders] + logger.debug(f"Provider filter: RBAC={rbacProviders}, Workflow={workflowProviders}, Effective={effectiveProviders}") + else: + # No workflow filter - use all RBAC-permitted providers + effectiveProviders = rbacProviders + logger.debug(f"Provider filter: RBAC={rbacProviders} (no workflow filter)") + + return effectiveProviders if effectiveProviders else None + + except Exception as e: + logger.warning(f"Error calculating effective providers: {e}") + return None + + def _storeAiCallStats(self, response, request: AiCallRequest) -> None: + """Store workflow stats after an AI call. + + This method stores the AI call statistics (cost, processing time, bytes) + to the workflow stats collection for tracking and billing purposes. + + Args: + response: AiCallResponse with cost/timing data + request: Original AiCallRequest for context + """ + try: + # Skip if no workflow context + workflow = getattr(self.services, 'workflow', None) + if not workflow or not hasattr(workflow, 'id') or not workflow.id: + logger.debug("No workflow context - skipping stats storage") + return + + # Skip if response is an error + if not response or getattr(response, 'errorCount', 0) > 0: + logger.debug("Error response - skipping stats storage") + return + + # Determine process name from operation type + opType = getattr(request.options, 'operationType', 'unknown') if request.options else 'unknown' + process = f"ai.call.{opType}" + + # Store the stat + self.services.chat.storeWorkflowStat(workflow, response, process) + logger.debug(f"Stored AI call stat: {process}, cost={getattr(response, 'priceCHF', 0):.4f} CHF") + + except Exception as e: + # Log but don't fail - stats storage is not critical + logger.debug(f"Could not store AI call stat: {str(e)}") + + async def ensureAiObjectsInitialized(self): + """Ensure aiObjects is initialized and submodules are ready.""" + if self.aiObjects is None: + logger.info("Lazy initializing AiObjects...") + self.aiObjects = await AiObjects.create() + logger.info("AiObjects initialization completed") + self._initializeSubmodules() + + @classmethod + async def create(cls, legacy_services) -> "AiService": + """Create AiService from legacy Services hub. For backward compatibility with tests.""" + from modules.serviceCenter import getService + from modules.serviceCenter.context import ServiceCenterContext + ctx = ServiceCenterContext( + user=legacy_services.user, + mandate_id=legacy_services.mandateId, + feature_instance_id=legacy_services.featureInstanceId, + workflow=getattr(legacy_services, "workflow", None), + ) + return getService("ai", ctx, legacy_hub=legacy_services) + + # Helper methods + + def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str: + """ + Build full prompt by replacing placeholders with their content. + Uses the new {{KEY:placeholder}} format. + + Args: + prompt: The base prompt template + placeholders: Dictionary of placeholder key-value pairs + + Returns: + Prompt with placeholders replaced + """ + if not placeholders: + return prompt + + full_prompt = prompt + for placeholder, content in placeholders.items(): + # Skip if content is None or empty + if content is None: + continue + # Replace {{KEY:placeholder}} + full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", str(content)) + + return full_prompt + + async def _analyzePromptAndCreateOptions(self, prompt: str) -> AiCallOptions: + """Analyze prompt to determine appropriate AiCallOptions parameters.""" + try: + # Get dynamic enum values from Pydantic models + operationTypes = [e.value for e in OperationTypeEnum] + priorities = [e.value for e in PriorityEnum] + processingModes = [e.value for e in ProcessingModeEnum] + + # Create analysis prompt for AI to determine operation type and parameters + analysisPrompt = f""" +You are an AI operation analyzer. Analyze the following prompt and determine the most appropriate operation type and parameters. + +PROMPT TO ANALYZE: +{self.services.utils.sanitizePromptContent(prompt, 'userinput')} + +Based on the prompt content, determine: +1. operationType: Choose the most appropriate from: {', '.join(operationTypes)} +2. priority: Choose from: {', '.join(priorities)} +3. processingMode: Choose from: {', '.join(processingModes)} +4. compressPrompt: true/false (true for story-like prompts, false for structured prompts with JSON/schemas) +5. compressContext: true/false (true to summarize context, false to process fully) + +Respond with ONLY a JSON object in this exact format: +{{ + "operationType": "dataAnalyse", + "priority": "balanced", + "processingMode": "basic", + "compressPrompt": true, + "compressContext": true +}} +""" + + # Use AI to analyze the prompt + request = AiCallRequest( + prompt=analysisPrompt, + options=AiCallOptions( + operationType=OperationTypeEnum.DATA_ANALYSE, + priority=PriorityEnum.SPEED, + processingMode=ProcessingModeEnum.BASIC, + compressPrompt=True, + compressContext=False + ) + ) + + response = await self.callAi(request) + + # Parse AI response using structured parsing with AiCallOptions model + try: + # Use parseJsonWithModel to parse response into AiCallOptions (handles enum conversion automatically) + analysis = parseJsonWithModel(response.content, AiCallOptions) + return analysis + except Exception as e: + logger.warning(f"Failed to parse AI analysis response: {e}") + + except Exception as e: + logger.warning(f"Prompt analysis failed: {e}") + + # Fallback to default options + return AiCallOptions( + operationType=OperationTypeEnum.DATA_ANALYSE, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.BASIC + ) + + async def callAiWithLooping( + self, + prompt: str, + options: AiCallOptions, + debugPrefix: str = "ai_call", + promptBuilder: Optional[callable] = None, + promptArgs: Optional[Dict[str, Any]] = None, + operationId: Optional[str] = None, + userPrompt: Optional[str] = None, + contentParts: Optional[List[ContentPart]] = None, # ARCHITECTURE: Support ContentParts for large content + useCaseId: Optional[str] = None # REQUIRED: Explicit use case ID for generic looping system + ) -> str: + """Public method: Delegate to AiCallLooper for AI calls with looping support.""" + return await self.aiCallLooper.callAiWithLooping( + prompt, options, debugPrefix, promptBuilder, promptArgs, operationId, userPrompt, contentParts, useCaseId + ) + + # JSON merging logic moved to subJsonResponseHandling.py + + def _extractSectionsFromResponse( + self, + result: str, + iteration: int, + debugPrefix: str, + allSections: List[Dict[str, Any]] = None, + accumulationState: Optional[JsonAccumulationState] = None + ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]: + """Delegate to ResponseParser.""" + return self.responseParser.extractSectionsFromResponse( + result, iteration, debugPrefix, allSections, accumulationState + ) + + def _shouldContinueGeneration( + self, + allSections: List[Dict[str, Any]], + iteration: int, + wasJsonComplete: bool, + rawResponse: str = None + ) -> bool: + """Delegate to ResponseParser.""" + return self.responseParser.shouldContinueGeneration( + allSections, iteration, wasJsonComplete, rawResponse + ) + + def _extractDocumentMetadata( + self, + parsedResult: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + """Delegate to ResponseParser.""" + return self.responseParser.extractDocumentMetadata(parsedResult) + + def _buildFinalResultFromSections( + self, + allSections: List[Dict[str, Any]], + documentMetadata: Optional[Dict[str, Any]] = None + ) -> str: + """Delegate to ResponseParser.""" + return self.responseParser.buildFinalResultFromSections(allSections, documentMetadata) + + # Public API Methods + + # Planning AI Call + async def callAiPlanning( + self, + prompt: str, + placeholders: Optional[List[PromptPlaceholder]] = None, + debugType: Optional[str] = None + ) -> str: + """ + Planning AI call for task planning, action planning, action selection, etc. + Always uses static parameters optimized for planning tasks. + + Args: + prompt: The planning prompt + placeholders: Optional list of placeholder replacements + debugType: Optional debug file type identifier (e.g., 'taskplan', 'dynamic', 'intentanalysis') + If not provided, defaults to 'plan' + + Returns: + Planning JSON response + """ + await self.ensureAiObjectsInitialized() + + # Planning calls always use static parameters + options = AiCallOptions( + operationType=OperationTypeEnum.PLAN, + priority=PriorityEnum.QUALITY, + processingMode=ProcessingModeEnum.DETAILED, + compressPrompt=False, + compressContext=False + ) + + # Build full prompt with placeholders + if placeholders: + placeholdersDict = {p.label: p.content for p in placeholders} + fullPrompt = self._buildPromptWithPlaceholders(prompt, placeholdersDict) + else: + fullPrompt = prompt + + # Root-cause fix: planning must return raw single-shot JSON, not section-based output + request = AiCallRequest( + prompt=fullPrompt, + context="", + options=options + ) + + # Debug: persist prompt/response for analysis with context-specific naming + debugPrefix = debugType if debugType else "plan" + self.services.utils.writeDebugFile(fullPrompt, f"{debugPrefix}_prompt") + response = await self.callAi(request) # Use callAi to ensure stats are stored + result = response.content or "" + self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") + return result + + # Helper methods for callAiContent refactoring + + async def _handleImageGeneration( + self, + prompt: str, + options: AiCallOptions, + title: Optional[str], + parentOperationId: Optional[str] + ) -> AiResponse: + """Handle IMAGE_GENERATE operation type using image generation path.""" + from modules.serviceCenter.services.serviceGeneration.paths.imagePath import ImageGenerationPath + + imagePath = ImageGenerationPath(self.services) + + # Extract format from options + format = options.resultFormat or "png" + + return await imagePath.generateImages( + userPrompt=prompt, + format=format, + title=title, + parentOperationId=parentOperationId + ) + + async def _handleWebOperation( + self, + prompt: str, + options: AiCallOptions, + opType: OperationTypeEnum, + aiOperationId: str + ) -> AiResponse: + """Handle WEB_SEARCH_DATA and WEB_CRAWL operation types.""" + self.services.chat.progressLogUpdate(aiOperationId, 0.4, f"Calling AI for {opType.name}") + + request = AiCallRequest( + prompt=prompt, # Raw JSON prompt - connector will parse it + context="", + options=options + ) + + response = await self.callAi(request) + + if not response.content: + errorMsg = f"No content returned from {opType.name}: {response.content}" + logger.error(f"Error in {opType.name}: {errorMsg}") + self.services.chat.progressLogFinish(aiOperationId, False) + raise ValueError(errorMsg) + + metadata = AiResponseMetadata( + operationType=opType.value + ) + + # Note: Stats are now stored centrally in callAi() - no need to duplicate here + + self.services.chat.progressLogUpdate(aiOperationId, 0.9, f"{opType.name} completed") + self.services.chat.progressLogFinish(aiOperationId, True) + + # Preserve metadata from response if available (e.g., results_with_content from Tavily) + # Check if response has metadata attribute (AiCallResponse from callAi) + if hasattr(response, 'metadata') and response.metadata: + # If metadata is a dict, store it in additionalData + if isinstance(response.metadata, dict): + if not metadata.additionalData: + metadata.additionalData = {} + metadata.additionalData.update(response.metadata) + # If metadata is an object with attributes, extract them + elif hasattr(response.metadata, '__dict__'): + if not metadata.additionalData: + metadata.additionalData = {} + for key, value in response.metadata.__dict__.items(): + if not key.startswith('_'): + metadata.additionalData[key] = value + + return AiResponse( + content=response.content, + metadata=metadata + ) + + def _getIntentForDocument( + self, + docId: str, + intents: Optional[List[DocumentIntent]] + ) -> Optional[DocumentIntent]: + """Find DocumentIntent for given documentId.""" + if not intents: + return None + for intent in intents: + if intent.documentId == docId: + return intent + return None + + async def clarifyDocumentIntents( + self, + documents: List[ChatDocument], + userPrompt: str, + actionParameters: Dict[str, Any], + parentOperationId: str + ) -> List[DocumentIntent]: + """Public method: Delegate to DocumentIntentAnalyzer.""" + return await self.intentAnalyzer.clarifyDocumentIntents( + documents, userPrompt, actionParameters, parentOperationId + ) + + async def extractAndPrepareContent( + self, + documents: List[ChatDocument], + documentIntents: List[DocumentIntent], + parentOperationId: str + ) -> List[ContentPart]: + """Public method: Delegate to ContentExtractor.""" + return await self.contentExtractor.extractAndPrepareContent( + documents, documentIntents, parentOperationId, self._getIntentForDocument + ) + + async def generateStructure( + self, + userPrompt: str, + contentParts: List[ContentPart], + outputFormat: Optional[str] = None, + parentOperationId: str = None + ) -> Dict[str, Any]: + """Public method: Delegate to StructureGenerator.""" + return await self.structureGenerator.generateStructure( + userPrompt, contentParts, outputFormat, parentOperationId + ) + + async def fillStructure( + self, + structure: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + parentOperationId: str + ) -> Dict[str, Any]: + """Public method: Delegate to StructureFiller.""" + return await self.structureFiller.fillStructure( + structure, contentParts, userPrompt, parentOperationId + ) + + async def renderResult( + self, + filledStructure: Dict[str, Any], + outputFormat: str, + language: str, + title: str, + userPrompt: str, + parentOperationId: str + ) -> List[RenderedDocument]: + """ + Phase 5E: Rendert gefüllte Struktur zum Ziel-Format. + Jedes Dokument wird einzeln gerendert, jeder Renderer kann 1..n Dokumente zurückgeben. + + Render filled structure to documents. + Per-document format and language are extracted from structure (validated in State 3). + The outputFormat and language parameters are only used as global fallbacks. + Multiple documents can have different formats and languages. + + Args: + filledStructure: Gefüllte Struktur mit elements + outputFormat: Ziel-Format (pdf, docx, html, etc.) - Global fallback + language: Language (global fallback) - Per-document language extracted from structure + title: Dokument-Titel + userPrompt: User-Anfrage + parentOperationId: Parent Operation-ID für ChatLog-Hierarchie + + Returns: + List of RenderedDocument objects. + Jedes RenderedDocument repräsentiert ein gerendertes Dokument (Hauptdokument oder unterstützende Datei) + """ + # Language comes from structure (per-document), validated in State 3 + # This parameter is only used as global fallback if structure validation fails + # Use validated currentUserLanguage as fallback (always valid) + if not language: + language = self._getUserLanguage() if hasattr(self, '_getUserLanguage') else (self.services.currentUserLanguage if hasattr(self.services, 'currentUserLanguage') else 'en') + # Erstelle Operation-ID für Rendering + renderOperationId = f"{parentOperationId}_rendering" + + # Starte ChatLog mit Parent-Referenz + self.services.chat.progressLogStart( + renderOperationId, + "Content Rendering", + "Rendering", + f"Rendering to {outputFormat} format", + parentOperationId=parentOperationId + ) + + try: + generationService = self._get_service("generation") + + # renderReport verarbeitet jetzt jedes Dokument einzeln + # und gibt Liste von (documentData, mimeType, filename) zurück + renderedDocuments = await generationService.renderReport( + filledStructure, + outputFormat, + language, # Pass language (global fallback, per-document extracted in renderReport) + title, + userPrompt, + self, + parentOperationId=renderOperationId # Parent-Referenz für ChatLog-Hierarchie + ) + + # ChatLog abschließen + self.services.chat.progressLogFinish(renderOperationId, True) + + return renderedDocuments + + except Exception as e: + self.services.chat.progressLogFinish(renderOperationId, False) + logger.error(f"Error in _renderResult: {str(e)}") + raise + + def _shouldSkipContentPart( + self, + part: ContentPart + ) -> bool: + """Check if ContentPart should be skipped (already structured JSON).""" + if part.typeGroup == "structure" and part.mimeType == "application/json": + if part.metadata.get("skipExtraction", False): + logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (skipExtraction=True)") + return True + try: + if isinstance(part.data, str): + jsonData = json.loads(part.data) + if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): + logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (contains documents/sections)") + return True + except Exception: + pass # Not JSON, continue processing + return False + + async def callAiContent( + self, + prompt: str, + options: AiCallOptions, + contentParts: Optional[List[ContentPart]] = None, + documentList: Optional[Any] = None, # DocumentReferenceList + documentIntents: Optional[List[DocumentIntent]] = None, + outputFormat: Optional[str] = None, + title: Optional[str] = None, + parentOperationId: Optional[str] = None, + generationIntent: Optional[str] = None # NEW: Explicit intent from action (skips detection) + ) -> AiResponse: + """ + Unified AI content generation with explicit intent requirement. + + All AI-Actions (ai.process, ai.generateDocument, etc.) route through here. + They differ only in parameters, not in logic. + + Args: + prompt: The main prompt for the AI call + options: AI call configuration options (REQUIRED - operationType must be set) + contentParts: Optional list of already-extracted content parts (preferred) + documentList: Optional DocumentReferenceList (wird zu ChatDocuments konvertiert) + documentIntents: Optional list of DocumentIntent objects (wird erstellt wenn nicht vorhanden) + outputFormat: Optional output format for document generation (e.g., 'pdf', 'docx', 'xlsx') + title: Optional title for generated documents + parentOperationId: Optional parent operation ID for hierarchical logging + generationIntent: REQUIRED explicit intent ("document" | "code" | "image") from action. + NO auto-detection - actions must explicitly specify intent. + + Returns: + AiResponse with content, metadata, and optional documents + """ + await self.ensureAiObjectsInitialized() + + # Erstelle Operation-ID + workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" + aiOperationId = f"ai_content_{workflowId}_{int(time.time())}" + + # Starte Progress-Tracking mit Parent-Referenz + formatDisplay = outputFormat if outputFormat else "auto-determined" + self.services.chat.progressLogStart( + aiOperationId, + "AI content processing", + "Content Processing", + f"Format: {formatDisplay}", + parentOperationId=parentOperationId + ) + + try: + # outputFormat is optional - if None, formats determined from prompt by AI + # No default fallback here - let AI service handle it + + opType = getattr(options, "operationType", None) + if not opType: + options.operationType = OperationTypeEnum.DATA_GENERATE + opType = OperationTypeEnum.DATA_GENERATE + + # Route zu Operation-spezifischen Handlern + if opType == OperationTypeEnum.IMAGE_GENERATE: + # Image generation - route to image path + return await self._handleImageGeneration(prompt, options, title, parentOperationId) + + if opType == OperationTypeEnum.WEB_SEARCH_DATA or opType == OperationTypeEnum.WEB_CRAWL: + return await self._handleWebOperation(prompt, options, opType, aiOperationId) + + # Data generation - REQUIRES explicit generationIntent + if opType == OperationTypeEnum.DATA_GENERATE: + if not generationIntent: + errorMsg = ( + "generationIntent is required for DATA_GENERATE operation. " + "Actions must explicitly specify 'document' or 'code' intent. " + "No auto-detection - use qualified actions (ai.generateDocument, ai.generateCode)." + ) + logger.error(errorMsg) + self.services.chat.progressLogFinish(aiOperationId, False) + raise ValueError(errorMsg) + + # Route based on explicit intent (no auto-detection, no fallback) + if generationIntent == "code": + # Route to code generation path + return await self._handleCodeGeneration( + prompt=prompt, + options=options, + contentParts=contentParts, + outputFormat=outputFormat, + title=title, + parentOperationId=parentOperationId + ) + else: + # Route to document generation path (existing behavior) + return await self._handleDocumentGeneration( + prompt=prompt, + options=options, + documentList=documentList, + documentIntents=documentIntents, + contentParts=contentParts, + outputFormat=outputFormat, + title=title, + parentOperationId=parentOperationId + ) + + # DATA_EXTRACT: Extract content from documents and process with AI (no structure generation) + if opType == OperationTypeEnum.DATA_EXTRACT: + return await self._handleDataExtraction( + prompt=prompt, + options=options, + documentList=documentList, + documentIntents=documentIntents, + contentParts=contentParts, + outputFormat=outputFormat, + title=title, + parentOperationId=parentOperationId + ) + + # Other operation types (DATA_ANALYSE, etc.) - not supported + errorMsg = f"Unsupported operation type: {opType}. Supported types: IMAGE_GENERATE, DATA_GENERATE, DATA_EXTRACT" + logger.error(errorMsg) + self.services.chat.progressLogFinish(aiOperationId, False) + raise ValueError(errorMsg) + + except Exception as e: + logger.error(f"Error in callAiContent: {str(e)}") + self.services.chat.progressLogFinish(aiOperationId, False) + raise + + async def _handleDataExtraction( + self, + prompt: str, + options: AiCallOptions, + documentList: Optional[Any], + documentIntents: Optional[List[DocumentIntent]], + contentParts: Optional[List[ContentPart]], + outputFormat: str, + title: str, + parentOperationId: Optional[str] + ) -> AiResponse: + """ + Handle DATA_EXTRACT: Extract content from documents, then process with AI. + + - AUTOMATION mode: No intent analysis. The passed prompt is used as extractionPrompt + for every document and for the final AI call (exact prompt preserved). + - DYNAMIC mode: Intent analysis (clarifyDocumentIntents) runs first; extraction and + processing use the intents and AI-derived extractionPrompt. + """ + import time + + # Create operation ID + workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" + extractOperationId = f"data_extract_{workflowId}_{int(time.time())}" + + # Start progress tracking + self.services.chat.progressLogStart( + extractOperationId, + "Data Extraction", + "Extraction", + f"Format: {outputFormat}", + parentOperationId=parentOperationId + ) + + try: + # Step 1: Get documents from documentList + documents = [] + if documentList: + documents = self.services.chat.getChatDocumentsFromDocumentList(documentList) + + # Filter: Remove original documents if already covered by pre-extracted JSONs + # (to prevent duplicate ContentParts - pre-extracted JSONs contain already extracted ContentParts) + if documents: + # Step 1: Identify all original document IDs covered by pre-extracted JSONs + originalDocIdsCoveredByPreExtracted = set() + for doc in documents: + preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc) + if preExtracted: + originalDocId = preExtracted["originalDocument"]["id"] + originalDocIdsCoveredByPreExtracted.add(originalDocId) + logger.debug(f"Found pre-extracted JSON {doc.id} covering original document {originalDocId}") + + # Step 2: Filter documents - remove originals covered by pre-extracted JSONs + filteredDocuments = [] + for doc in documents: + preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc) + if preExtracted: + filteredDocuments.append(doc) # Keep pre-extracted JSON + elif doc.id in originalDocIdsCoveredByPreExtracted: + logger.info(f"Skipping original document {doc.id} ({doc.fileName}) - already covered by pre-extracted JSON") + else: + filteredDocuments.append(doc) # Keep regular document + + documents = filteredDocuments # Use filtered list + + # Step 2: Document intents – AUTOMATION uses exact prompt; DYNAMIC uses intent analysis + if not documentIntents and documents: + workflowMode = getattr(self.services.workflow, "workflowMode", None) if self.services.workflow else None + if workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION: + # Automation: no intent AI call – use the given prompt as extractionPrompt for every document + documentIntents = [ + DocumentIntent( + documentId=doc.id, + intents=["extract"], + extractionPrompt=prompt, + reasoning="Automation mode: use exact prompt from action", + ) + for doc in documents + ] + logger.debug("DATA_EXTRACT in AUTOMATION mode: using exact prompt for all documents (no intent analysis)") + else: + documentIntents = await self.clarifyDocumentIntents( + documents, + prompt, + {"outputFormat": outputFormat}, + extractOperationId + ) + + # Step 3: Extract and prepare content (NO AI - pure extraction) - REQUIRED for all documents + if documents: + preparedContentParts = await self.extractAndPrepareContent( + documents, + documentIntents or [], + extractOperationId + ) + + # Merge with provided contentParts (if any) + if contentParts: + for part in contentParts: + if part.metadata.get("skipExtraction", False): + part.metadata.setdefault("contentFormat", "extracted") + part.metadata.setdefault("isPreExtracted", True) + preparedContentParts.extend(contentParts) + + contentParts = preparedContentParts + + # Step 4: Process contentParts with AI via ExtractionService + # Always use processContentPartsWithAi – it handles text vs image parts correctly: + # - Text parts → text models (with chunking if needed) + # - Image parts → Vision AI (proper image_url content blocks) + # No manual contentText concatenation or token estimation needed. + if not contentParts: + raise ValueError("No content extracted from documents") + + # Filter out empty content parts (e.g. PDF container with 0 bytes) that would + # produce garbage AI responses and pollute the merged result. + nonEmptyParts = [p for p in contentParts if p.data and len(p.data.strip()) > 0] + if not nonEmptyParts: + raise ValueError("No non-empty content parts to process") + + self.services.utils.writeDebugFile(prompt, "data_extract_prompt") + extractionService = self.services.extraction + aiRequest = AiCallRequest( + prompt=prompt, + context="", + options=options, + contentParts=nonEmptyParts, + ) + aiResponse = await extractionService.processContentPartsWithAi( + aiRequest, self.aiObjects + ) + _respText = aiResponse.content if isinstance(aiResponse.content, str) else (aiResponse.content.decode("utf-8", errors="replace") if aiResponse.content else "") + self.services.utils.writeDebugFile(_respText, "data_extract_response") + + # Create response document + resultDocument = DocumentData( + documentName=f"{title or 'extracted_data'}.{outputFormat}", + documentData=aiResponse.content.encode('utf-8') if isinstance(aiResponse.content, str) else aiResponse.content, + mimeType=f"text/{outputFormat}" if outputFormat in ["txt", "json", "csv"] else "application/octet-stream" + ) + + metadata = AiResponseMetadata( + title=title or "Extracted Data", + operationType=OperationTypeEnum.DATA_EXTRACT.value + ) + + self.services.chat.progressLogFinish(extractOperationId, True) + + return AiResponse( + content=aiResponse.content if isinstance(aiResponse.content, str) else aiResponse.content.decode('utf-8', errors='replace'), + metadata=metadata, + documents=[resultDocument] + ) + + except Exception as e: + logger.error(f"Error in data extraction: {str(e)}") + self.services.chat.progressLogFinish(extractOperationId, False) + raise + + async def _handleCodeGeneration( + self, + prompt: str, + options: AiCallOptions, + contentParts: Optional[List[ContentPart]], + outputFormat: str, + title: str, + parentOperationId: Optional[str] + ) -> AiResponse: + """Handle code generation using code generation path.""" + from modules.serviceCenter.services.serviceGeneration.paths.codePath import CodeGenerationPath + + codePath = CodeGenerationPath(self.services) + return await codePath.generateCode( + userPrompt=prompt, + outputFormat=outputFormat, + contentParts=contentParts, + title=title or "Generated Code", + parentOperationId=parentOperationId + ) + + async def _handleDocumentGeneration( + self, + prompt: str, + options: AiCallOptions, + documentList: Optional[Any], + documentIntents: Optional[List[DocumentIntent]], + contentParts: Optional[List[ContentPart]], + outputFormat: str, + title: str, + parentOperationId: Optional[str] + ) -> AiResponse: + """Handle document generation using document generation path.""" + from modules.serviceCenter.services.serviceGeneration.paths.documentPath import DocumentGenerationPath + + # Set compression options for document generation + options.compressPrompt = False + options.compressContext = False + + documentPath = DocumentGenerationPath(self.services) + return await documentPath.generateDocument( + userPrompt=prompt, + documentList=documentList, + documentIntents=documentIntents, + contentParts=contentParts, + outputFormat=outputFormat, + title=title or "Generated Document", + parentOperationId=parentOperationId + ) + + + def _determineDocumentName( + self, + filledStructure: Dict[str, Any], + outputFormat: str, + title: Optional[str] + ) -> str: + """Bestimme Dokument-Namen aus Struktur oder Titel.""" + # Versuche aus Struktur zu extrahieren + if isinstance(filledStructure, dict) and "documents" in filledStructure: + docs = filledStructure["documents"] + if isinstance(docs, list) and len(docs) > 0: + firstDoc = docs[0] + if isinstance(firstDoc, dict) and firstDoc.get("filename"): + return firstDoc["filename"] + + # Fallback zu Titel + if title: + sanitized = re.sub(r"[^a-zA-Z0-9._-]", "_", title) + sanitized = re.sub(r"_+", "_", sanitized).strip("_") + if sanitized: + if not sanitized.lower().endswith(f".{outputFormat}"): + return f"{sanitized}.{outputFormat}" + return sanitized + + return f"generated.{outputFormat}" + diff --git a/modules/serviceCenter/services/serviceAi/subAiCallLooping.py b/modules/serviceCenter/services/serviceAi/subAiCallLooping.py new file mode 100644 index 00000000..2e4edc3e --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subAiCallLooping.py @@ -0,0 +1,665 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +AI Call Looping Module + +Handles AI calls with looping and repair logic, including: +- Looping with JSON repair and continuation +- KPI definition and tracking +- Progress tracking and iteration management + +FLOW LOGIC + +VARIABLES: +- jsonBase: str (merged JSON so far, starts empty) +- lastValidCompletePart: str (fallback for failures) +- mergeFailCount: int = 0 (max 3) + +FLOW: +┌─────────────────────────────────────────────────────────────────┐ +│ 1. BUILD PROMPT │ +│ - First: original prompt │ +│ - Next: buildContinuationContext(lastRawResponse) │ +├─────────────────────────────────────────────────────────────────┤ +│ 2. CALL AI → response fragment │ +├─────────────────────────────────────────────────────────────────┤ +│ 4. MERGE jsonBase + response │ +│ ├─ FAILS: repeat prompt, fails++ (if >=3 return fallback) │ +│ └─ SUCCEEDS: try parse │ +│ ├─ SUCCEEDS: FINISHED │ +│ └─ FAILS: → step 5 │ +├─────────────────────────────────────────────────────────────────┤ +│ 5. GET CONTEXTS (merge OK, parse failed) │ +│ getContexts(mergedJson) → │ +│ - If no cut point: overlapContext = "" │ +│ - Store contexts for next iteration │ +├─────────────────────────────────────────────────────────────────┤ +│ 6. DECIDE │ +│ ├─ jsonParsingSuccess=true AND overlapContext="": │ +│ │ FINISHED. return completePart │ +│ ├─ jsonParsingSuccess=true AND overlapContext!="": │ +│ │ CONTINUE, fails=0 │ +│ └─ ELSE: repeat prompt, fails++ │ +└─────────────────────────────────────────────────────────────────┘ + + +""" + +import json +import logging +from typing import Dict, Any, List, Optional, Callable + +from modules.datamodels.datamodelAi import ( + AiCallRequest, AiCallOptions +) +from modules.datamodels.datamodelExtraction import ContentPart +from .subJsonResponseHandling import JsonResponseHandler +from .subLoopingUseCases import LoopingUseCaseRegistry +from modules.workflows.processing.shared.stateTools import checkWorkflowStopped +from modules.shared.jsonContinuation import getContexts +from modules.shared.jsonUtils import buildContinuationContext, extractJsonString, tryParseJson +from modules.shared.jsonUtils import tryParseJson +from modules.shared.jsonUtils import closeJsonStructures +from modules.shared.jsonUtils import stripCodeFences, normalizeJsonText + +logger = logging.getLogger(__name__) + + +class AiCallLooper: + """Handles AI calls with looping and repair logic.""" + + def __init__(self, services, aiService, responseParser): + """Initialize AiCallLooper with service center, AI service, and response parser access.""" + self.services = services + self.aiService = aiService + self.responseParser = responseParser + self.useCaseRegistry = LoopingUseCaseRegistry() # Initialize use case registry + + async def callAiWithLooping( + self, + prompt: str, + options: AiCallOptions, + debugPrefix: str = "ai_call", + promptBuilder: Optional[Callable] = None, + promptArgs: Optional[Dict[str, Any]] = None, + operationId: Optional[str] = None, + userPrompt: Optional[str] = None, + contentParts: Optional[List[ContentPart]] = None, # ARCHITECTURE: Support ContentParts for large content + useCaseId: str = None # REQUIRED: Explicit use case ID - no auto-detection, no fallback + ) -> str: + """ + Shared core function for AI calls with repair-based looping system. + Automatically repairs broken JSON and continues generation seamlessly. + + Args: + prompt: The prompt to send to AI + options: AI call configuration options + debugPrefix: Prefix for debug file names + promptBuilder: Optional function to rebuild prompts for continuation + promptArgs: Optional arguments for prompt builder + operationId: Optional operation ID for progress tracking + userPrompt: Optional user prompt for KPI definition + contentParts: Optional content parts for first iteration + useCaseId: REQUIRED: Explicit use case ID - no auto-detection, no fallback + + Returns: + Complete AI response after all iterations + """ + # REQUIRED: useCaseId must be provided - no auto-detection, no fallback + if not useCaseId: + errorMsg = ( + "useCaseId is REQUIRED for callAiWithLooping. " + "No auto-detection - must explicitly specify use case ID. " + f"Available use cases: {list(self.useCaseRegistry.useCases.keys())}" + ) + logger.error(errorMsg) + raise ValueError(errorMsg) + + # Validate use case exists + useCase = self.useCaseRegistry.get(useCaseId) + if not useCase: + errorMsg = ( + f"Use case '{useCaseId}' not found in registry. " + f"Available use cases: {list(self.useCaseRegistry.useCases.keys())}" + ) + logger.error(errorMsg) + raise ValueError(errorMsg) + + maxIterations = 50 # Prevent infinite loops + iteration = 0 + allSections = [] # Accumulate all sections across iterations + lastRawResponse = None # Store last raw JSON response for continuation + + # JSON Base Iteration System: + # - jsonBase: the merged JSON string (replaces accumulatedDirectJson array) + # - After each iteration, new response is merged with jsonBase + # - On merge success: check if complete, store contexts for next iteration + # - On merge fail: retry with same prompt, increment fails + jsonBase = None # Merged JSON string (starts None, set on first response) + + # Merge fail tracking - stop after 3 consecutive merge failures + MAX_MERGE_FAILS = 3 + mergeFailCount = 0 # Global counter for merge failures across entire loop + lastValidCompletePart = None # Store last successfully parsed completePart for fallback + + # Get parent operation ID for iteration operations (parentId should be operationId, not log entry ID) + parentOperationId = operationId # Use the parent's operationId directly + + while iteration < maxIterations: + iteration += 1 + + # Create separate operation for each iteration with parent reference + iterationOperationId = None + if operationId: + iterationOperationId = f"{operationId}_iter_{iteration}" + self.services.chat.progressLogStart( + iterationOperationId, + "AI Call", + f"Iteration {iteration}", + "", + parentOperationId=parentOperationId + ) + + # Build iteration prompt + # CRITICAL: Build continuation prompt if we have sections OR if we have a previous response (even if broken) + # This ensures continuation prompts are built even when JSON is so broken that no sections can be extracted + if (len(allSections) > 0 or lastRawResponse) and promptBuilder and promptArgs: + # Extract templateStructure and basePrompt from promptArgs (REQUIRED) + templateStructure = promptArgs.get("templateStructure") + if not templateStructure: + raise ValueError( + f"templateStructure is REQUIRED in promptArgs for use case '{useCaseId}'. " + "Prompt creation functions must return (prompt, templateStructure) tuple." + ) + + basePrompt = promptArgs.get("basePrompt") + if not basePrompt: + # Fallback: use prompt parameter (should be the same) + basePrompt = prompt + logger.warning( + f"basePrompt not found in promptArgs for use case '{useCaseId}', " + "using prompt parameter instead. This may indicate a bug." + ) + + # This is a continuation - build continuation context with raw JSON and rebuild prompt + continuationContext = buildContinuationContext( + allSections, lastRawResponse, useCaseId, templateStructure + ) + if not lastRawResponse: + logger.warning(f"Iteration {iteration}: No previous response available for continuation!") + + # Store valid completePart from continuation context for fallback on merge failures + # Use getContexts to check if completePart is parseable and store it + if lastRawResponse and not lastValidCompletePart: + try: + contexts = getContexts(lastRawResponse) + if contexts.jsonParsingSuccess and contexts.completePart: + lastValidCompletePart = contexts.completePart + logger.debug(f"Iteration {iteration}: Stored initial valid completePart ({len(lastValidCompletePart)} chars)") + except Exception as e: + logger.debug(f"Iteration {iteration}: Failed to extract completePart: {e}") + + # Unified prompt builder call: Continuation builders only need continuationContext, templateStructure, and basePrompt + # All initial context (section, userPrompt, etc.) is already in basePrompt, so promptArgs is not needed + # Extract templateStructure and basePrompt from promptArgs (they're explicit parameters) + iterationPrompt = await promptBuilder( + continuationContext=continuationContext, + templateStructure=templateStructure, + basePrompt=basePrompt + ) + else: + # First iteration - use original prompt + iterationPrompt = prompt + + # Make AI call + try: + checkWorkflowStopped(self.services) + if iterationOperationId: + self.services.chat.progressLogUpdate(iterationOperationId, 0.3, "Calling AI model") + # ARCHITECTURE: Pass ContentParts directly to AiCallRequest + # This allows model-aware chunking to handle large content properly + # ContentParts are only passed in first iteration (continuations don't need them) + request = AiCallRequest( + prompt=iterationPrompt, + context="", + options=options, + contentParts=contentParts if iteration == 1 else None # Only pass ContentParts in first iteration + ) + + # Write the ACTUAL prompt sent to AI + # For section content generation: write prompt for first iteration and continuation iterations + # For document generation: write prompt for each iteration + isSectionContent = "_section_" in debugPrefix + if iteration == 1: + self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt") + elif isSectionContent: + # Save continuation prompts for section_content debugging + self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") + else: + # Document generation - save all iteration prompts + self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") + + response = await self.aiService.callAi(request) + result = response.content + + # Track bytes for progress reporting + bytesReceived = len(result.encode('utf-8')) if result else 0 + totalBytesSoFar = sum(len(section.get('content', '').encode('utf-8')) if isinstance(section.get('content'), str) else 0 for section in allSections) + bytesReceived + + # Update progress after AI call with byte information + if iterationOperationId: + # Format bytes for display (kB or MB) + if totalBytesSoFar < 1024: + bytesDisplay = f"{totalBytesSoFar}B" + elif totalBytesSoFar < 1024 * 1024: + bytesDisplay = f"{totalBytesSoFar / 1024:.1f}kB" + else: + bytesDisplay = f"{totalBytesSoFar / (1024 * 1024):.1f}MB" + self.services.chat.progressLogUpdate(iterationOperationId, 0.6, f"AI response received ({bytesDisplay})") + + # Write raw AI response to debug file + # For section content generation: write response for first iteration and continuation iterations + # For document generation: write response for each iteration + if iteration == 1: + self.services.utils.writeDebugFile(result, f"{debugPrefix}_response") + elif isSectionContent: + # Save continuation responses for section_content debugging + self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}") + else: + # Document generation - save all iteration responses + self.services.utils.writeDebugFile(result, f"{debugPrefix}_response_iteration_{iteration}") + + # Note: Stats are now stored centrally in callAi() - no need to duplicate here + + # Check for error response using generic error detection (errorCount > 0 or modelName == "error") + if hasattr(response, 'errorCount') and response.errorCount > 0: + errorMsg = f"Iteration {iteration}: Error response detected (errorCount={response.errorCount}), stopping loop: {result[:200] if result else 'empty'}" + logger.error(errorMsg) + break + + if hasattr(response, 'modelName') and response.modelName == "error": + errorMsg = f"Iteration {iteration}: Error response detected (modelName=error), stopping loop: {result[:200] if result else 'empty'}" + logger.error(errorMsg) + break + + if not result or not result.strip(): + logger.warning(f"Iteration {iteration}: Empty response, stopping") + break + + # Check if this is a text response (not document generation) + # Text responses don't need JSON parsing - return immediately after first successful response + isTextResponse = (promptBuilder is None and promptArgs is None) or debugPrefix == "text" + + if isTextResponse: + # For text responses, return the text immediately - no JSON parsing needed + logger.info(f"Iteration {iteration}: Text response received, returning immediately") + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, True) + return result + + # NOTE: Do NOT update lastRawResponse here! + # lastRawResponse should only be updated after successful merge + # This ensures retry iterations use the correct base context + + # Handle use cases that return JSON directly (no section extraction needed) + # Check if use case supports direct return (all registered use cases do) + if useCase and not useCase.requiresExtraction: + # ===================================================================== + # ITERATION FLOW (Simplified) + # ===================================================================== + # Step 4: MERGE jsonBase + new response + # - FAILS: repeat prompt, increment fails cont (if >=3 return fallback) + # - SUCCEEDS: try parse + # - SUCCEEDS: FINISHED + # - FAILS: proceed to Step 5 + # Step 5: GET CONTEXTS (merge OK, parse failed) + # - getContexts() with repair + # - If no cut point: overlapContext = "" + # Step 6: DECIDE + # - jsonParsingSuccess=true AND overlapContext="": FINISHED + # - jsonParsingSuccess=true AND overlapContext!="": continue, fails=0 + # - ELSE: repeat prompt, increment fails count + # ===================================================================== + + # STEP 4: MERGE jsonBase + new response + # Use candidateJson to hold merged result until we confirm it's valid + candidateJson = None + + if jsonBase is None: + # First iteration - candidate is the current result + candidateJson = result + logger.debug(f"Iteration {iteration}: First response, candidateJson ({len(candidateJson)} chars)") + else: + # Merge jsonBase with new response + logger.info(f"Iteration {iteration}: Merging jsonBase ({len(jsonBase)} chars) with new response ({len(result)} chars)") + mergedJsonString, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap(jsonBase, result) + + if not hasOverlap: + # MERGE FAILED - repeat prompt with unchanged jsonBase + mergeFailCount += 1 + logger.warning( + f"Iteration {iteration}: Merge failed, no overlap found " + f"(fail {mergeFailCount}/{MAX_MERGE_FAILS})" + ) + + if mergeFailCount >= MAX_MERGE_FAILS: + # Max failures reached - return last valid completePart + logger.error( + f"Iteration {iteration}: Max merge failures ({MAX_MERGE_FAILS}) reached, " + "returning last valid completePart" + ) + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, False) + + if lastValidCompletePart: + try: + extracted = extractJsonString(lastValidCompletePart) + parsed, parseErr, _ = tryParseJson(extracted) + if parseErr is None and parsed: + normalized = self._normalizeJsonStructure(parsed, useCase) + return json.dumps(normalized, indent=2, ensure_ascii=False) + except Exception: + pass + return lastValidCompletePart + else: + # No valid fallback - return whatever we have + return jsonBase if jsonBase else "" + + # Not at max failures - retry with same prompt (jsonBase unchanged) + if iterationOperationId: + self.services.chat.progressLogUpdate( + iterationOperationId, 0.7, + f"Merge failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying" + ) + self.services.chat.progressLogFinish(iterationOperationId, True) + continue + + # MERGE SUCCEEDED - set candidate (don't update jsonBase yet!) + candidateJson = mergedJsonString + logger.debug(f"Iteration {iteration}: Merge succeeded, candidateJson ({len(candidateJson)} chars)") + + # Update lastRawResponse ONLY after we have a valid candidateJson + # (first iteration or successful merge - NOT on merge failure!) + # This ensures retry iterations use the correct base context + lastRawResponse = candidateJson + + # Try direct parse of candidate + try: + extracted = extractJsonString(candidateJson) + parsed, parseErr, _ = tryParseJson(extracted) + if parseErr is None and parsed: + # Direct parse succeeded - FINISHED + # Commit candidate to jsonBase + jsonBase = candidateJson + logger.info(f"Iteration {iteration}: Direct parse succeeded, JSON is complete") + normalized = self._normalizeJsonStructure(parsed, useCase) + result = json.dumps(normalized, indent=2, ensure_ascii=False) + + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, True) + + if not useCase.finalResultHandler: + raise ValueError( + f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback." + ) + return useCase.finalResultHandler( + result, normalized, extracted, debugPrefix, self.services + ) + except Exception as e: + logger.debug(f"Iteration {iteration}: Direct parse failed: {e}") + + # STEP 5: GET CONTEXTS (merge OK, parse failed = cut JSON) + # Use candidateJson for context extraction + contexts = getContexts(candidateJson) + overlapInfo = "(empty=complete)" if contexts.overlapContext == "" else f"({len(contexts.overlapContext)} chars)" + logger.debug( + f"Iteration {iteration}: getContexts() -> " + f"jsonParsingSuccess={contexts.jsonParsingSuccess}, " + f"overlapContext={overlapInfo}" + ) + + # STEP 6: DECIDE based on jsonParsingSuccess and overlapContext + if contexts.jsonParsingSuccess and contexts.overlapContext == "": + # JSON is complete (no cut point) - FINISHED + # Use completePart for final result (closed, repaired JSON) + # No more merging needed, so we don't need the cut version + jsonBase = contexts.completePart + logger.info(f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext='', JSON complete") + + # Store and parse completePart + lastValidCompletePart = contexts.completePart + + try: + extracted = extractJsonString(contexts.completePart) + parsed, parseErr, _ = tryParseJson(extracted) + if parseErr is None and parsed: + normalized = self._normalizeJsonStructure(parsed, useCase) + result = json.dumps(normalized, indent=2, ensure_ascii=False) + + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, True) + + if not useCase.finalResultHandler: + raise ValueError( + f"Use case '{useCaseId}' is missing required 'finalResultHandler' callback." + ) + return useCase.finalResultHandler( + result, normalized, extracted, debugPrefix, self.services + ) + except Exception as e: + logger.warning(f"Iteration {iteration}: Failed to parse completePart: {e}") + + # Fallback: return completePart as-is + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, True) + return contexts.completePart + + elif contexts.jsonParsingSuccess and contexts.overlapContext != "": + # JSON parseable but has cut point - CONTINUE to next iteration + # CRITICAL: Use hierarchyContext (CUT json) as jsonBase for next merge! + # - hierarchyContext = the truncated JSON at cut point (needed for overlap matching) + # - completePart = closed JSON (for validation/fallback only) + # The next AI fragment's overlap must match the CUT point, not closed structures + jsonBase = contexts.hierarchyContext + logger.info( + f"Iteration {iteration}: jsonParsingSuccess=true, overlapContext not empty, " + f"continuing iteration (jsonBase updated to hierarchyContext: {len(jsonBase)} chars)" + ) + + # Store valid completePart as fallback (different from jsonBase!) + lastValidCompletePart = contexts.completePart + + # Reset fail counter on successful progress + mergeFailCount = 0 + + # Update lastRawResponse for continuation prompt building + # Use the CUT version for prompt context as well + lastRawResponse = jsonBase + + if iterationOperationId: + self.services.chat.progressLogUpdate(iterationOperationId, 0.7, "JSON incomplete, requesting continuation") + self.services.chat.progressLogFinish(iterationOperationId, True) + continue + + else: + # JSON not parseable after repair - repeat prompt, increment fails + # Do NOT update jsonBase - keep previous valid state + mergeFailCount += 1 + logger.warning( + f"Iteration {iteration}: jsonParsingSuccess=false, " + f"repeat prompt (fail {mergeFailCount}/{MAX_MERGE_FAILS})" + ) + + if mergeFailCount >= MAX_MERGE_FAILS: + # Max failures reached - return last valid completePart + logger.error( + f"Iteration {iteration}: Max failures ({MAX_MERGE_FAILS}) reached, " + "returning last valid completePart" + ) + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, False) + + if lastValidCompletePart: + try: + extracted = extractJsonString(lastValidCompletePart) + parsed, parseErr, _ = tryParseJson(extracted) + if parseErr is None and parsed: + normalized = self._normalizeJsonStructure(parsed, useCase) + return json.dumps(normalized, indent=2, ensure_ascii=False) + except Exception: + pass + return lastValidCompletePart + else: + return jsonBase if jsonBase else "" + + # Not at max - retry with same prompt + # Do NOT update jsonBase or lastRawResponse - keep previous for retry + if iterationOperationId: + self.services.chat.progressLogUpdate( + iterationOperationId, 0.7, + f"Parse failed ({mergeFailCount}/{MAX_MERGE_FAILS}), retrying" + ) + self.services.chat.progressLogFinish(iterationOperationId, True) + continue + + except Exception as e: + logger.error(f"Error in AI call iteration {iteration}: {str(e)}") + if iterationOperationId: + self.services.chat.progressLogFinish(iterationOperationId, False) + break + + if iteration >= maxIterations: + logger.warning(f"AI call stopped after maximum iterations ({maxIterations})") + + # This code path should never be reached because all registered use cases + # return early when JSON is complete. This would only execute for use cases that + # require section extraction, but no such use cases are currently registered. + logger.error(f"Unexpected code path: reached end of loop without return for use case '{useCaseId}'") + return result if result else "" + + def _isJsonStringIncomplete(self, jsonString: str) -> bool: + """ + Check if JSON string is incomplete (truncated) BEFORE closing/parsing. + + This is critical because if JSON is truncated, closing it makes it appear complete, + but we need to detect the truncation to continue iteration. + + Args: + jsonString: JSON string to check + + Returns: + True if JSON string appears incomplete/truncated, False otherwise + """ + if not jsonString or not jsonString.strip(): + return False + + # Normalize JSON string + normalized = stripCodeFences(normalizeJsonText(jsonString)).strip() + if not normalized: + return False + + # Find first '{' or '[' to start + startIdx = -1 + for i, char in enumerate(normalized): + if char in '{[': + startIdx = i + break + + if startIdx == -1: + return False + + jsonContent = normalized[startIdx:] + + # Check if structures are balanced (all opened structures are closed) + braceCount = 0 + bracketCount = 0 + inString = False + escapeNext = False + + for char in jsonContent: + if escapeNext: + escapeNext = False + continue + + if char == '\\': + escapeNext = True + continue + + if char == '"': + inString = not inString + continue + + if not inString: + if char == '{': + braceCount += 1 + elif char == '}': + braceCount -= 1 + elif char == '[': + bracketCount += 1 + elif char == ']': + bracketCount -= 1 + + # If structures are unbalanced, JSON is incomplete + if braceCount > 0 or bracketCount > 0: + return True + + # Check if JSON ends with incomplete value (e.g., unclosed string, incomplete number, trailing comma) + trimmed = jsonContent.rstrip() + if not trimmed: + return False + + # Check for trailing comma (might indicate incomplete) + if trimmed.endswith(','): + # Trailing comma might indicate incomplete, but could also be valid + # Check if there's a closing bracket/brace after the comma + return False # Trailing comma alone doesn't mean incomplete + + # Check if ends with incomplete string (odd number of quotes) + quoteCount = jsonContent.count('"') + if quoteCount % 2 == 1: + # Odd number of quotes - string is not closed + return True + + # Check if ends mid-value (e.g., ends with "417 instead of "4170. 41719"]) + # Look for patterns that suggest truncation: + # - Ends with incomplete number (e.g., "417) + # - Ends with incomplete array element (e.g., ["417) + # - Ends with incomplete object property (e.g., {"key": "val) + + # If JSON parses successfully without closing, it's complete + parsed, parseErr, _ = tryParseJson(jsonContent) + if parseErr is None: + # Parses successfully - it's complete + return False + + # If it doesn't parse, try closing it and see if that helps + closed = closeJsonStructures(jsonContent) + parsedClosed, parseErrClosed, _ = tryParseJson(closed) + + if parseErrClosed is None: + # Only parses after closing - it was incomplete + return True + + # Doesn't parse even after closing - might be malformed, but assume incomplete to be safe + return True + + def _normalizeJsonStructure(self, parsed: Any, useCase) -> Any: + """ + Normalize JSON structure to ensure consistent format before merging. + Handles different response formats and converts them to expected structure. + + Args: + parsed: Parsed JSON object (can be dict, list, or primitive) + useCase: LoopingUseCase instance with jsonNormalizer callback + + Returns: + Normalized JSON structure + """ + # Use callback to normalize JSON structure (REQUIRED - no fallback) + if not useCase or not useCase.jsonNormalizer: + raise ValueError( + f"Use case '{useCase.useCaseId if useCase else 'unknown'}' is missing required 'jsonNormalizer' callback. " + "All use cases must provide a jsonNormalizer function." + ) + return useCase.jsonNormalizer(parsed, useCase.useCaseId) + diff --git a/modules/serviceCenter/services/serviceAi/subContentExtraction.py b/modules/serviceCenter/services/serviceAi/subContentExtraction.py new file mode 100644 index 00000000..a7250a3a --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subContentExtraction.py @@ -0,0 +1,721 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Content Extraction Module + +Handles content extraction and preparation, including: +- Extracting content from documents based on intents +- Processing pre-extracted documents +- Vision AI for image text extraction +- AI processing of text content +""" +import json +import logging +import base64 +from typing import Dict, Any, List, Optional + +from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent, ExtractionOptions, MergeStrategy +from modules.workflows.processing.shared.stateTools import checkWorkflowStopped + +logger = logging.getLogger(__name__) + + +class ContentExtractor: + """Handles content extraction and preparation.""" + + def __init__(self, services, aiService, intentAnalyzer): + """Initialize ContentExtractor with service center, AI service, and intent analyzer access.""" + self.services = services + self.aiService = aiService + self.intentAnalyzer = intentAnalyzer + + async def extractAndPrepareContent( + self, + documents: List[ChatDocument], + documentIntents: List[DocumentIntent], + parentOperationId: str, + getIntentForDocument: callable + ) -> List[ContentPart]: + """ + Phase 5B: Extrahiert Content basierend auf Intents und bereitet ContentParts mit Metadaten vor. + Gibt Liste von ContentParts im passenden Format zurück. + + WICHTIG: Ein Dokument kann mehrere ContentParts erzeugen, wenn mehrere Intents vorhanden sind. + Beispiel: Bild mit intents=["extract", "render"] erzeugt: + - ContentPart(contentFormat="object", ...) für Rendering + - ContentPart(contentFormat="extracted", ...) für Text-Analyse + + Args: + documents: Liste der zu verarbeitenden Dokumente + documentIntents: Liste von DocumentIntent-Objekten + parentOperationId: Parent Operation-ID für ChatLog-Hierarchie + getIntentForDocument: Callable to get intent for document ID + + Returns: + Liste von ContentParts mit vollständigen Metadaten + """ + # Erstelle Operation-ID für Extraktion + extractionOperationId = f"{parentOperationId}_content_extraction" + + # Starte ChatLog mit Parent-Referenz + self.services.chat.progressLogStart( + extractionOperationId, + "Content Extraction", + "Extraction", + f"Extracting from {len(documents)} documents", + parentOperationId=parentOperationId + ) + + try: + allContentParts = [] + + for document in documents: + checkWorkflowStopped(self.services) + # Check if document is already a ContentExtracted document (pre-extracted JSON) + logger.debug(f"Checking document {document.id} ({document.fileName}, mimeType={document.mimeType}) for pre-extracted content") + preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(document) + + if preExtracted: + logger.info(f"✅ Found pre-extracted document: {document.fileName} -> Original: {preExtracted['originalDocument']['fileName']}") + logger.info(f" Pre-extracted document ID: {document.id}, Original document ID: {preExtracted['originalDocument']['id']}") + logger.info(f" ContentParts count: {len(preExtracted['contentExtracted'].parts) if preExtracted['contentExtracted'].parts else 0}") + + # Verwende bereits extrahierte ContentParts direkt + contentExtracted = preExtracted["contentExtracted"] + + # WICHTIG: Intent muss für das JSON-Dokument gefunden werden, nicht für das Original + # (Intent-Analyse mappt bereits zurück zu JSON-Dokument-ID) + intent = getIntentForDocument(document.id, documentIntents) + logger.info(f" Intent lookup for document {document.id}: found={intent is not None}") + if intent: + logger.info(f" Intent: {intent.intents}, extractionPrompt: {intent.extractionPrompt[:100] if intent.extractionPrompt else None}...") + else: + logger.warning(f" ⚠️ No intent found for pre-extracted document {document.id}! Available intent documentIds: {[i.documentId for i in documentIntents]}") + + if contentExtracted.parts: + # CRITICAL: Process pre-extracted parts - analyze structure parts for nested content + processedParts = [] + for part in contentExtracted.parts: + # Überspringe leere Parts (Container ohne Daten) + if not part.data or (isinstance(part.data, str) and len(part.data.strip()) == 0): + if part.typeGroup == "container": + continue # Überspringe leere Container + + # CRITICAL: Check if structure part contains nested parts (e.g., JSON with documentData.parts) + if part.typeGroup == "structure" and part.mimeType == "application/json" and part.data: + nestedParts = self._extractNestedPartsFromStructure(part, document, preExtracted, intent) + if nestedParts: + # Replace structure part with extracted nested parts + processedParts.extend(nestedParts) + logger.info(f"✅ Extracted {len(nestedParts)} nested parts from structure part {part.id}") + continue # Skip original structure part + + # Keep original part if no nested parts found + processedParts.append(part) + + # Use processed parts (with nested parts extracted) + for part in processedParts: + if not part.metadata: + part.metadata = {} + + # Ensure metadata is complete + if "documentId" not in part.metadata: + part.metadata["documentId"] = document.id + + # WICHTIG: Prüfe Intent für dieses Part + partIntent = intent.intents if intent else ["extract"] + + # Debug-Logging für Intent-Verarbeitung + logger.debug(f"Processing part {part.id}: typeGroup={part.typeGroup}, intents={partIntent}, hasData={bool(part.data)}, dataLength={len(str(part.data)) if part.data else 0}") + + # WICHTIG: Ein Part kann mehrere Intents haben - erstelle für jeden Intent einen ContentPart + # Generische Intent-Verarbeitung für ALLE Content-Typen + hasReferenceIntent = "reference" in partIntent + hasRenderIntent = "render" in partIntent + hasExtractIntent = "extract" in partIntent + hasPartData = bool(part.data) and (not isinstance(part.data, str) or len(part.data.strip()) > 0) + + logger.debug(f"Part {part.id}: reference={hasReferenceIntent}, render={hasRenderIntent}, extract={hasExtractIntent}, hasData={hasPartData}") + + # SAFETY: For images with any intent, always ensure render is included + # This ensures the image object part is always available for later rendering + isImage = part.typeGroup == "image" or (part.mimeType and part.mimeType.startswith("image/")) + if isImage and hasPartData and not hasRenderIntent: + logger.info(f"🖼️ Auto-adding render intent for image {part.id} (original intents: {partIntent})") + hasRenderIntent = True + + # Track ob der originale Part bereits hinzugefügt wurde + originalPartAdded = False + + # 1. Reference Intent: Erstelle Reference ContentPart + if hasReferenceIntent: + referencePart = ContentPart( + id=f"ref_{document.id}_{part.id}", + label=f"Reference: {part.label or 'Content'}", + typeGroup="reference", + mimeType=part.mimeType or "application/octet-stream", + data="", # Leer - nur Referenz + metadata={ + "contentFormat": "reference", + "documentId": document.id, + "documentReference": f"docItem:{document.id}:{preExtracted['originalDocument']['fileName']}", + "intent": "reference", + "usageHint": f"Reference: {preExtracted['originalDocument']['fileName']}", + "originalFileName": preExtracted["originalDocument"]["fileName"] + } + ) + allContentParts.append(referencePart) + logger.debug(f"✅ Created reference ContentPart for {part.id}") + + # 2. Render Intent: Erstelle Object ContentPart (für Binary/Image Rendering) + if hasRenderIntent and hasPartData: + # Prüfe ob es ein Binary/Image ist (kann gerendert werden) + isRenderable = ( + part.typeGroup == "image" or + part.typeGroup == "binary" or + (part.mimeType and ( + part.mimeType.startswith("image/") or + part.mimeType.startswith("video/") or + part.mimeType.startswith("audio/") or + self._isBinary(part.mimeType) + )) + ) + + if isRenderable: + objectPart = ContentPart( + id=f"obj_{document.id}_{part.id}", + label=f"Object: {part.label or 'Content'}", + typeGroup=part.typeGroup, + mimeType=part.mimeType or "application/octet-stream", + data=part.data, # Base64/Binary data ist bereits vorhanden + metadata={ + "contentFormat": "object", + "documentId": document.id, + "intent": "render", + "usageHint": f"Render as visual element: {preExtracted['originalDocument']['fileName']}", + "originalFileName": preExtracted["originalDocument"]["fileName"], + "relatedExtractedPartId": f"extracted_{document.id}_{part.id}" if hasExtractIntent else None + } + ) + allContentParts.append(objectPart) + logger.debug(f"✅ Created object ContentPart for {part.id} (render intent)") + else: + logger.warning(f"⚠️ Part {part.id} has render intent but is not renderable (typeGroup={part.typeGroup}, mimeType={part.mimeType})") + elif hasRenderIntent and not hasPartData: + logger.warning(f"⚠️ Part {part.id} has render intent but no data, skipping render part") + + # 3. Extract Intent: Erstelle Extracted ContentPart (NO AI processing here - happens during section generation) + if hasExtractIntent: + # For images: Keep as image part with extract intent - Vision AI extraction happens during section generation + if part.typeGroup == "image" and hasPartData: + logger.info(f"📷 Image {part.id} with extract intent - will be processed with Vision AI during section generation") + # Keep image part as-is, mark with extract intent + part.metadata.update({ + "contentFormat": "extracted", # Marked for extraction, but not yet extracted + "intent": "extract", + "originalFileName": preExtracted["originalDocument"]["fileName"], + "relatedObjectPartId": f"obj_{document.id}_{part.id}" if hasRenderIntent else None, + "extractionPrompt": intent.extractionPrompt if intent and intent.extractionPrompt else "Extract all text content from this image.", + "needsVisionExtraction": True # Flag to indicate Vision AI extraction needed + }) + allContentParts.append(part) + originalPartAdded = True + else: + # For text/table content: Use directly as extracted (no AI processing here) + # AI processing with extractionPrompt happens during section generation + if not originalPartAdded: + part.metadata.update({ + "contentFormat": "extracted", + "intent": "extract", + "fromExtractContent": True, + "skipExtraction": True, # Already extracted (raw extraction) + "originalFileName": preExtracted["originalDocument"]["fileName"], + "relatedObjectPartId": f"obj_{document.id}_{part.id}" if hasRenderIntent else None, + "extractionPrompt": intent.extractionPrompt if intent and intent.extractionPrompt else None + }) + # Stelle sicher dass contentFormat gesetzt ist + if "contentFormat" not in part.metadata: + part.metadata["contentFormat"] = "extracted" + allContentParts.append(part) + originalPartAdded = True + logger.debug(f"✅ Using pre-extracted ContentPart {part.id} as extracted (no AI processing needed)") + + # 4. Fallback: Wenn kein Intent vorhanden oder Part wurde noch nicht hinzugefügt + # (sollte normalerweise nicht vorkommen, da default "extract" ist) + if not hasReferenceIntent and not hasRenderIntent and not hasExtractIntent and not originalPartAdded: + logger.warning(f"⚠️ Part {part.id} has no recognized intents, adding as extracted by default") + part.metadata.update({ + "contentFormat": "extracted", + "intent": "extract", + "fromExtractContent": True, + "skipExtraction": True, + "originalFileName": preExtracted["originalDocument"]["fileName"] + }) + allContentParts.append(part) + originalPartAdded = True + + logger.info(f"✅ Using {len([p for p in contentExtracted.parts if p.data and len(str(p.data)) > 0])} pre-extracted ContentParts from ContentExtracted document {document.fileName}") + logger.info(f" Original document: {preExtracted['originalDocument']['fileName']}") + continue # Skip normal extraction for this document + + # Check if it's standardized JSON format (has "documents" or "sections") + if document.mimeType == "application/json": + try: + docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) + if docBytes: + docData = docBytes.decode('utf-8') + jsonData = json.loads(docData) + + if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData): + logger.info(f"Document is already in standardized JSON format, using as reference") + # Create reference ContentPart for structured JSON + contentPart = ContentPart( + id=f"ref_{document.id}", + label=f"Reference: {document.fileName}", + typeGroup="structure", + mimeType="application/json", + data=docData, + metadata={ + "contentFormat": "reference", + "documentId": document.id, + "documentReference": f"docItem:{document.id}:{document.fileName}", + "skipExtraction": True, + "intent": "reference" + } + ) + allContentParts.append(contentPart) + logger.info(f"✅ Using JSON document directly without extraction") + continue # Skip normal extraction for this document + except Exception as e: + logger.warning(f"Could not parse JSON document {document.fileName}, will extract normally: {str(e)}") + # Continue with normal extraction + + # Normal extraction path + intent = getIntentForDocument(document.id, documentIntents) + + if not intent: + # Try to find intent by similar UUID (fix for AI UUID hallucination) + correctedIntent = self._findIntentBySimilarId(document.id, documentIntents) + if correctedIntent: + logger.warning(f"Found intent for document {document.id} using UUID correction (original: {correctedIntent.documentId})") + # Create new intent with correct document ID + intent = DocumentIntent( + documentId=document.id, + intents=correctedIntent.intents, + extractionPrompt=correctedIntent.extractionPrompt, + reasoning=f"Intent matched by UUID similarity (original: {correctedIntent.documentId})" + ) + else: + # Default: extract für alle Dokumente ohne Intent + logger.warning(f"No intent found for document {document.id}, using default 'extract'") + intent = DocumentIntent( + documentId=document.id, + intents=["extract"], + extractionPrompt="Extract all content from the document", + reasoning="Default intent: no specific intent found" + ) + + # WICHTIG: Prüfe alle Intents - ein Dokument kann mehrere ContentParts erzeugen + + if "reference" in intent.intents: + # Erstelle Reference ContentPart + contentPart = ContentPart( + id=f"ref_{document.id}", + label=f"Reference: {document.fileName}", + typeGroup="reference", + mimeType=document.mimeType, + data="", + metadata={ + "contentFormat": "reference", + "documentId": document.id, + "documentReference": f"docItem:{document.id}:{document.fileName}", + "intent": "reference", + "usageHint": f"Reference document: {document.fileName}" + } + ) + allContentParts.append(contentPart) + + # WICHTIG: "render" und "extract" können beide vorhanden sein! + # In diesem Fall erzeugen wir BEIDE ContentParts + + # SAFETY: For images with any intent, always create object part for later rendering + isImageDocument = document.mimeType and document.mimeType.startswith("image/") + shouldAutoRender = isImageDocument and "render" not in intent.intents and ("extract" in intent.intents or "reference" in intent.intents) + if shouldAutoRender: + logger.info(f"🖼️ Auto-adding render for image document {document.id} (original intents: {intent.intents})") + + if "render" in intent.intents or shouldAutoRender: + # Für Images/Binary: extrahiere als Object + if document.mimeType.startswith("image/") or self._isBinary(document.mimeType): + try: + # Lade Binary-Daten (getFileData ist nicht async - keine await nötig) + binaryData = self.services.interfaceDbComponent.getFileData(document.fileId) + if not binaryData: + logger.warning(f"No binary data found for document {document.id}") + continue + base64Data = base64.b64encode(binaryData).decode('utf-8') + + contentPart = ContentPart( + id=f"obj_{document.id}", + label=f"Object: {document.fileName}", + typeGroup="image" if document.mimeType.startswith("image/") else "binary", + mimeType=document.mimeType, + data=base64Data, + metadata={ + "contentFormat": "object", + "documentId": document.id, + "intent": "render", + "usageHint": f"Render as visual element: {document.fileName}", + "originalFileName": document.fileName, + # Verknüpfung zu extracted Part (falls vorhanden) + "relatedExtractedPartId": f"ext_{document.id}" if "extract" in intent.intents else None + } + ) + allContentParts.append(contentPart) + except Exception as e: + logger.error(f"Failed to load binary data for document {document.id}: {str(e)}") + + if "extract" in intent.intents: + # Extrahiere Content mit Extraction Service + extractionPrompt = intent.extractionPrompt or "Extract all content from the document" + + # Debug-Log (harmonisiert) + self.services.utils.writeDebugFile( + extractionPrompt, + f"content_extraction_prompt_{document.id}" + ) + + # Führe Extraktion aus + + extractionOptions = ExtractionOptions( + prompt=extractionPrompt, + mergeStrategy=MergeStrategy() + ) + + # extractContent ist nicht async - keine await nötig + checkWorkflowStopped(self.services) + extractedResults = self.services.extraction.extractContent( + [document], + extractionOptions, + operationId=extractionOperationId, + parentOperationId=extractionOperationId + ) + + # Konvertiere extrahierte Ergebnisse zu ContentParts mit Metadaten + # Check if object part exists (either explicit render or auto-render for images) + hasObjectPart = "render" in intent.intents or shouldAutoRender + + for extracted in extractedResults: + for part in extracted.parts: + # Markiere als extracted Format + part.metadata.update({ + "contentFormat": "extracted", + "documentId": document.id, + "extractionPrompt": extractionPrompt, + "intent": "extract", + "usageHint": f"Use extracted content from {document.fileName}", + # Verknüpfung zu object Part (falls vorhanden - including auto-render for images) + "relatedObjectPartId": f"obj_{document.id}" if hasObjectPart else None + }) + + # For images: Mark that Vision AI extraction is needed during section generation + if part.typeGroup == "image": + part.metadata["needsVisionExtraction"] = True + logger.info(f"📷 Image part {part.id} marked for Vision AI extraction during section generation") + + # Stelle sicher, dass ID eindeutig ist (falls object Part existiert) + if hasObjectPart: + part.id = f"ext_{document.id}_{part.id}" + allContentParts.append(part) + + # Debug-Log (harmonisiert) + self.services.utils.writeDebugFile( + json.dumps([part.dict() for part in allContentParts], indent=2, default=str), + "content_extraction_result" + ) + + # State 2 Validation: Validate and auto-fix ContentParts + validatedParts = [] + for part in allContentParts: + # Validation 2.1: Skip ContentParts without documentId + if not part.metadata.get("documentId"): + logger.warning(f"Skipping ContentPart {part.id} - missing documentId in metadata") + continue + + # Validation 2.2: Skip ContentParts with invalid contentFormat + contentFormat = part.metadata.get("contentFormat") + if contentFormat not in ["extracted", "object", "reference"]: + logger.warning( + f"Skipping ContentPart {part.id} - invalid contentFormat: {contentFormat}" + ) + continue + + validatedParts.append(part) + + # ChatLog abschließen + self.services.chat.progressLogFinish(extractionOperationId, True) + + return validatedParts + + except Exception as e: + self.services.chat.progressLogFinish(extractionOperationId, False) + logger.error(f"Error in extractAndPrepareContent: {str(e)}") + raise + + async def extractTextFromImage(self, imagePart: ContentPart, extractionPrompt: str) -> Optional[str]: + """ + Extrahiere Text aus einem Image-Part mit Vision AI. + + Args: + imagePart: ContentPart mit typeGroup="image" + extractionPrompt: Prompt für die Text-Extraktion + + Returns: + Extrahierter Text oder None bei Fehler + """ + try: + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + + # Final extraction prompt + finalPrompt = extractionPrompt or "Extract all text content from this image. Return only the extracted text, no additional formatting." + + # Debug-Log (harmonisiert) + self.services.utils.writeDebugFile( + finalPrompt, + f"content_extraction_prompt_image_{imagePart.id}" + ) + + # Erstelle AI-Call-Request mit Image-Part + request = AiCallRequest( + prompt=finalPrompt, + context="", + options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), + contentParts=[imagePart] + ) + + # Verwende AI-Service für Vision AI-Verarbeitung + checkWorkflowStopped(self.services) + response = await self.aiService.callAi(request) + + # Debug-Log für Response (harmonisiert) + if response and response.content: + self.services.utils.writeDebugFile( + response.content, + f"content_extraction_response_image_{imagePart.id}" + ) + + if response and response.content: + return response.content.strip() + + # Kein Content zurückgegeben - return error message für Debugging + errorMsg = f"Vision AI extraction failed: No content returned for image {imagePart.id}" + logger.warning(errorMsg) + return f"[ERROR: {errorMsg}]" + except Exception as e: + errorMsg = f"Vision AI extraction failed for image {imagePart.id}: {str(e)}" + logger.error(errorMsg) + import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") + # Return error message statt None für Debugging + return f"[ERROR: {errorMsg}]" + + async def processTextContentWithAi(self, textPart: ContentPart, extractionPrompt: str) -> Optional[str]: + """ + Verarbeite Text-Content mit AI basierend auf extractionPrompt. + + WICHTIG: Pre-extracted ContentParts von context.extractContent enthalten RAW extrahierten Text + (z.B. aus PDF-Text-Layer). Wenn "extract" Intent vorhanden ist, muss dieser Text mit AI + verarbeitet werden (Transformation, Strukturierung, etc.) basierend auf extractionPrompt. + + Args: + textPart: ContentPart mit typeGroup="text" (oder anderer Text-basierter Typ) + extractionPrompt: Prompt für die AI-Verarbeitung des Textes + + Returns: + AI-verarbeiteter Text oder None bei Fehler + """ + try: + from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum + + # Final extraction prompt + finalPrompt = extractionPrompt or "Process and extract the key information from the following text content." + + # Debug-Log (harmonisiert) - log prompt with text preview + textPreview = textPart.data[:500] + "..." if textPart.data and len(textPart.data) > 500 else (textPart.data or "") + promptWithContext = f"{finalPrompt}\n\n--- Text Content (preview) ---\n{textPreview}" + self.services.utils.writeDebugFile( + promptWithContext, + f"content_extraction_prompt_text_{textPart.id}" + ) + + # Erstelle Text-ContentPart für AI-Verarbeitung + # Verwende den vorhandenen Text als Input + textContentPart = ContentPart( + id=textPart.id, + label=textPart.label, + typeGroup="text", + mimeType="text/plain", + data=textPart.data if textPart.data else "", + metadata=textPart.metadata.copy() if textPart.metadata else {} + ) + + # Erstelle AI-Call-Request mit Text-Part + request = AiCallRequest( + prompt=finalPrompt, + context="", + options=AiCallOptions(operationType=OperationTypeEnum.DATA_EXTRACT), + contentParts=[textContentPart] + ) + + # Verwende AI-Service für Text-Verarbeitung + checkWorkflowStopped(self.services) + response = await self.aiService.callAi(request) + + # Debug-Log für Response (harmonisiert) + if response and response.content: + self.services.utils.writeDebugFile( + response.content, + f"content_extraction_response_text_{textPart.id}" + ) + + if response and response.content: + return response.content.strip() + + # Kein Content zurückgegeben - return error message für Debugging + errorMsg = f"AI text processing failed: No content returned for text part {textPart.id}" + logger.warning(errorMsg) + return f"[ERROR: {errorMsg}]" + except Exception as e: + errorMsg = f"AI text processing failed for text part {textPart.id}: {str(e)}" + logger.error(errorMsg) + import traceback + logger.debug(f"Traceback: {traceback.format_exc()}") + # Return error message statt None für Debugging + return f"[ERROR: {errorMsg}]" + + def _isBinary(self, mimeType: str) -> bool: + """Prüfe ob MIME-Type binary ist.""" + binaryTypes = [ + "application/octet-stream", + "application/pdf", + "application/zip", + "application/x-zip-compressed" + ] + return mimeType in binaryTypes or mimeType.startswith("image/") or mimeType.startswith("video/") or mimeType.startswith("audio/") + + def _extractNestedPartsFromStructure( + self, + structurePart: ContentPart, + document: ChatDocument, + preExtracted: Dict[str, Any], + intent: Optional[Any] + ) -> List[ContentPart]: + """ + Extract nested parts from a structure ContentPart (e.g., JSON with documentData.parts). + + This is a generic function that analyzes pre-processed ContentParts and extracts + any nested parts that are embedded in structure data (typically JSON). + + Works with standard ContentExtracted format: documentData.parts array. + Each nested part is extracted as a separate ContentPart with proper metadata. + + Args: + structurePart: ContentPart with typeGroup="structure" containing nested parts + document: The document this part belongs to + preExtracted: Pre-extracted document metadata + intent: Document intent for nested parts + + Returns: + List of extracted ContentParts, empty if no nested parts found + """ + nestedParts = [] + + try: + # Parse JSON structure + jsonData = json.loads(structurePart.data) + + # Check for standard ContentExtracted format: documentData.parts + if isinstance(jsonData, dict): + documentData = jsonData.get("documentData") + if isinstance(documentData, dict): + parts = documentData.get("parts", []) + if isinstance(parts, list) and len(parts) > 0: + # Extract each nested part + for nestedPartData in parts: + if not isinstance(nestedPartData, dict): + continue + + nestedPartId = nestedPartData.get("id") or f"nested_{len(nestedParts)}" + nestedTypeGroup = nestedPartData.get("typeGroup", "text") + nestedMimeType = nestedPartData.get("mimeType", "text/plain") + nestedLabel = nestedPartData.get("label", structurePart.label) + nestedData = nestedPartData.get("data", "") + nestedMetadata = nestedPartData.get("metadata", {}) + + # Create ContentPart for nested part + nestedPart = ContentPart( + id=f"{structurePart.id}_{nestedPartId}", + parentId=structurePart.id, + label=nestedLabel, + typeGroup=nestedTypeGroup, + mimeType=nestedMimeType, + data=nestedData, + metadata={ + **nestedMetadata, + "documentId": document.id, + "fromNestedStructure": True, + "parentStructurePartId": structurePart.id, + "originalFileName": preExtracted["originalDocument"]["fileName"] + } + ) + + nestedParts.append(nestedPart) + logger.debug(f"✅ Extracted nested part: {nestedPart.id} (typeGroup={nestedTypeGroup}, mimeType={nestedMimeType})") + + # If no nested parts found, return empty list (original part will be kept) + if not nestedParts: + logger.debug(f"No nested parts found in structure part {structurePart.id}") + + except json.JSONDecodeError as e: + logger.warning(f"Could not parse structure part {structurePart.id} as JSON: {str(e)}") + except Exception as e: + logger.error(f"Error extracting nested parts from structure part {structurePart.id}: {str(e)}") + + return nestedParts + + def _findIntentBySimilarId(self, documentId: str, documentIntents: List[DocumentIntent]) -> Optional[DocumentIntent]: + """ + Versucht ein Intent zu finden, dessen UUID ähnlich zur angegebenen Dokument-ID ist. + Dies hilft bei AI UUID-Halluzinationen (z.B. 4451 -> 4551). + + Args: + documentId: Die Dokument-ID für die ein Intent gesucht wird + documentIntents: Liste aller verfügbaren DocumentIntents + + Returns: + DocumentIntent mit ähnlicher UUID falls gefunden, sonst None + """ + if not documentId or len(documentId) != 36: # UUID Format: 8-4-4-4-12 + return None + + # Prüfe ob es eine UUID ist (Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) + if documentId.count('-') != 4: + return None + + for intent in documentIntents: + intentId = intent.documentId + if len(intentId) != 36: + continue + + # Zähle unterschiedliche Zeichen + differences = sum(c1 != c2 for c1, c2 in zip(documentId, intentId)) + + # Wenn nur 1-2 Zeichen unterschiedlich sind, ist es wahrscheinlich ein Typo + if differences <= 2: + # Prüfe ob die Struktur ähnlich ist (gleiche Positionen der Bindestriche) + if documentId.count('-') == intentId.count('-'): + return intent + + return None + diff --git a/modules/serviceCenter/services/serviceAi/subDocumentIntents.py b/modules/serviceCenter/services/serviceAi/subDocumentIntents.py new file mode 100644 index 00000000..274a8a5a --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subDocumentIntents.py @@ -0,0 +1,369 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Document Intent Analysis Module + +Handles analysis of document intents, including: +- Clarifying which documents need extraction vs reference +- Resolving pre-extracted documents +- Building intent analysis prompts +""" +import json +import logging +from typing import Dict, Any, List, Optional + +from modules.datamodels.datamodelChat import ChatDocument +from modules.datamodels.datamodelExtraction import DocumentIntent +from modules.workflows.processing.shared.stateTools import checkWorkflowStopped + +logger = logging.getLogger(__name__) + + +class DocumentIntentAnalyzer: + """Handles document intent analysis and resolution.""" + + def __init__(self, services, aiService): + """Initialize DocumentIntentAnalyzer with service center and AI service access.""" + self.services = services + self.aiService = aiService + + async def clarifyDocumentIntents( + self, + documents: List[ChatDocument], + userPrompt: str, + actionParameters: Dict[str, Any], + parentOperationId: str + ) -> List[DocumentIntent]: + """ + Phase 5A: Analysiert, welche Dokumente Extraktion vs Referenz benötigen. + Gibt DocumentIntent für jedes Dokument zurück. + + Args: + documents: Liste der zu verarbeitenden Dokumente + userPrompt: User-Anfrage + actionParameters: Action-spezifische Parameter (z.B. resultType, outputFormat) + parentOperationId: Parent Operation-ID für ChatLog-Hierarchie + + Returns: + Liste von DocumentIntent-Objekten + """ + # Erstelle Operation-ID für Intent-Analyse + intentOperationId = f"{parentOperationId}_intent_analysis" + + # Starte ChatLog mit Parent-Referenz + self.services.chat.progressLogStart( + intentOperationId, + "Document Intent Analysis", + "Intent Analysis", + f"Analyzing {len(documents)} documents", + parentOperationId=parentOperationId + ) + + try: + # Mappe pre-extracted JSONs zu ursprünglichen Dokument-IDs für Intent-Analyse + documentMapping = {} # Maps original doc ID -> JSON doc ID + resolvedDocuments = [] + + for doc in documents: + preExtracted = self.resolvePreExtractedDocument(doc) + if preExtracted: + originalDocId = preExtracted["originalDocument"]["id"] + documentMapping[originalDocId] = doc.id + # Erstelle temporäres ChatDocument für ursprüngliches Dokument + originalDoc = ChatDocument( + id=originalDocId, + fileName=preExtracted["originalDocument"]["fileName"], + mimeType=preExtracted["originalDocument"]["mimeType"], + fileSize=preExtracted["originalDocument"].get("fileSize", doc.fileSize), + fileId=doc.fileId, # Behalte fileId vom JSON + messageId=doc.messageId if hasattr(doc, 'messageId') else None # Behalte messageId falls vorhanden + ) + resolvedDocuments.append(originalDoc) + else: + resolvedDocuments.append(doc) + + # Baue Intent-Analyse-Prompt mit ursprünglichen Dokumenten + intentPrompt = self._buildIntentAnalysisPrompt(userPrompt, resolvedDocuments, actionParameters) + + # AI-Call (verwende callAiPlanning für einfache JSON-Responses) + # Debug-Logs werden bereits von callAiPlanning geschrieben + checkWorkflowStopped(self.services) + aiResponse = await self.aiService.callAiPlanning( + prompt=intentPrompt, + debugType="document_intent_analysis" + ) + + # Parse Result und mappe zurück zu JSON-Dokument-IDs falls nötig + intentsData = json.loads(self.services.utils.jsonExtractString(aiResponse)) + documentIntents = [] + for intent in intentsData.get("intents", []): + docId = intent.get("documentId") + # Wenn Intent für ursprüngliches Dokument, mappe zurück zu JSON-Dokument-ID + if docId in documentMapping: + intent["documentId"] = documentMapping[docId] + documentIntents.append(DocumentIntent(**intent)) + + # Debug-Log (harmonisiert) + self.services.utils.writeDebugFile( + json.dumps([intent.dict() for intent in documentIntents], indent=2), + "document_intent_analysis_result" + ) + + # State 1 Validation: Validate and auto-fix document intents + documentIds = {d.id for d in documents} + validatedIntents = [] + + for intent in documentIntents: + # Validation 1.2: Skip intents for unknown documents + if intent.documentId not in documentIds: + # Try to find similar UUID (fix AI hallucination/typo) + correctedDocId = self._findSimilarDocumentId(intent.documentId, documentIds) + if correctedDocId: + logger.warning(f"Corrected UUID typo in AI response: {intent.documentId} -> {correctedDocId}") + intent.documentId = correctedDocId + else: + logger.warning(f"Skipping intent for unknown document: {intent.documentId}") + continue + validatedIntents.append(intent) + + # Validation 1.1: Documents without intents are OK (not needed) + # Intents for non-existing documents are already filtered above + documentIntents = validatedIntents + + # ChatLog abschließen + self.services.chat.progressLogFinish(intentOperationId, True) + + return documentIntents + + except Exception as e: + self.services.chat.progressLogFinish(intentOperationId, False) + logger.error(f"Error in clarifyDocumentIntents: {str(e)}") + raise + + def resolvePreExtractedDocument(self, document: ChatDocument) -> Optional[Dict[str, Any]]: + """ + Prüft ob ein JSON-Dokument bereits extrahierte ContentParts enthält. + Gibt Dict zurück mit: + - originalDocument: ChatDocument-Info des ursprünglichen Dokuments + - contentExtracted: ContentExtracted-Objekt mit Parts + - parts: Liste der ContentParts + + Returns None wenn kein pre-extracted Format erkannt wird. + """ + if document.mimeType != "application/json": + logger.debug(f"Document {document.id} is not JSON (mimeType={document.mimeType}), skipping pre-extracted check") + return None + + try: + docBytes = self.services.interfaceDbComponent.getFileData(document.fileId) + if not docBytes: + return None + + docData = docBytes.decode('utf-8') + jsonData = json.loads(docData) + + if not isinstance(jsonData, dict): + return None + + # Check for ContentExtracted format + # Nur Format 1 (ActionDocument-Format mit validationMetadata) wird unterstützt + documentData = None + + validationMetadata = jsonData.get("validationMetadata", {}) + actionType = validationMetadata.get("actionType") + logger.debug(f"JSON document {document.id}: validationMetadata.actionType={actionType}, keys={list(jsonData.keys())}") + + if actionType == "context.extractContent": + # Format: {"validationMetadata": {"actionType": "context.extractContent"}, "documentData": {...}} + documentData = jsonData.get("documentData") + logger.debug(f"Found ContentExtracted via validationMetadata for {document.fileName}, documentData keys: {list(documentData.keys()) if documentData else None}") + else: + logger.debug(f"JSON document {document.id} does not have actionType='context.extractContent' (got: {actionType})") + + if documentData: + + try: + # Stelle sicher, dass "id" vorhanden ist + if "id" not in documentData: + documentData["id"] = document.id + + contentExtracted = ContentExtracted(**documentData) + + if contentExtracted.parts: + # Extrahiere ursprüngliche Dokument-Info aus den Parts + originalDocId = None + originalFileName = None + originalMimeType = None + + for part in contentExtracted.parts: + if part.metadata: + # Versuche ursprüngliche Dokument-Info zu finden + if not originalDocId and part.metadata.get("documentId"): + originalDocId = part.metadata.get("documentId") + if not originalFileName and part.metadata.get("originalFileName"): + originalFileName = part.metadata.get("originalFileName") + if not originalMimeType and part.metadata.get("documentMimeType"): + originalMimeType = part.metadata.get("documentMimeType") + + # Falls nicht gefunden, versuche aus documentName zu extrahieren + if not originalFileName: + # Versuche aus documentName zu extrahieren (z.B. "B2025-02c_28_extracted_...json" -> "B2025-02c_28.pdf") + if document.fileName and "_extracted_" in document.fileName: + originalFileName = document.fileName.split("_extracted_")[0] + ".pdf" + + return { + "originalDocument": { + "id": originalDocId or document.id, + "fileName": originalFileName or document.fileName, + "mimeType": originalMimeType or "application/pdf", + "fileSize": document.fileSize + }, + "contentExtracted": contentExtracted, + "parts": contentExtracted.parts + } + except Exception as parseError: + logger.warning(f"Could not parse ContentExtracted format from {document.fileName}: {str(parseError)}") + logger.debug(f"JSON keys: {list(jsonData.keys())}, has parts: {'parts' in jsonData}") + import traceback + logger.debug(f"Parse error traceback: {traceback.format_exc()}") + return None + else: + logger.debug(f"JSON document {document.id} has no documentData (actionType={actionType})") + + return None + except Exception as e: + logger.debug(f"Error resolving pre-extracted document {document.fileName}: {str(e)}") + return None + + def _buildIntentAnalysisPrompt( + self, + userPrompt: str, + documents: List[ChatDocument], + actionParameters: Dict[str, Any] + ) -> str: + """Baue Prompt für Intent-Analyse.""" + # Baue Dokument-Liste - zeige ursprüngliche Dokumente für pre-extracted JSONs + docListText = "" + for i, doc in enumerate(documents, 1): + # Prüfe ob es ein pre-extracted JSON ist + preExtracted = self.resolvePreExtractedDocument(doc) + + if preExtracted: + # Zeige ursprüngliches Dokument statt JSON + originalDoc = preExtracted["originalDocument"] + partsInfo = f" (contains {len(preExtracted['parts'])} pre-extracted parts: {', '.join([p.typeGroup for p in preExtracted['parts'] if p.data and len(str(p.data)) > 0])})" + docListText += f"\n{i}. Document ID: {originalDoc['id']}\n" + docListText += f" File Name: {originalDoc['fileName']}{partsInfo}\n" + docListText += f" MIME Type: {originalDoc['mimeType']}\n" + docListText += f" File Size: {originalDoc.get('fileSize', doc.fileSize)} bytes\n" + else: + # Normales Dokument + docListText += f"\n{i}. Document ID: {doc.id}\n" + docListText += f" File Name: {doc.fileName}\n" + docListText += f" MIME Type: {doc.mimeType}\n" + docListText += f" File Size: {doc.fileSize} bytes\n" + + outputFormat = actionParameters.get("outputFormat", "txt") + + # FENCE user input to prevent prompt injection + fencedUserPrompt = f"""```user_request +{userPrompt} +```""" + + prompt = f"""USER REQUEST: +{fencedUserPrompt} + +DOCUMENTS TO ANALYZE: +{docListText} + +TASK: For each document, determine its intents (can be multiple): +- "extract": Content extraction needed (text, structure, OCR, etc.) +- "render": Image/binary should be rendered as-is (visual element) +- "reference": Document reference/attachment (no extraction, just reference) + +TASK: For each document, determine: +1. Intents (can be multiple): "extract", "render", "reference" +Note: Output format and language are NOT determined here - they will be + determined during structure generation (Phase 3) in the chapter structure JSON + +OUTPUT FORMAT: {outputFormat} (global fallback - for reference only) + +RETURN JSON: +{{ + "intents": [ + {{ + "documentId": "doc_1", + "intents": ["extract"], + "extractionPrompt": "Extract all text content, preserving structure", + "reasoning": "User needs text content for document generation" + }}, + {{ + "documentId": "doc_2", + "intents": ["extract", "render"], + "extractionPrompt": "Extract text content from image using vision AI", + "reasoning": "Image contains text that needs extraction, but also should be rendered visually" + }}, + {{ + "documentId": "doc_3", + "intents": ["reference"], + "extractionPrompt": null, + "reasoning": "Document is only used as reference, no extraction needed" + }} + ] +}} + +CRITICAL RULES: +1. For images (mimeType starts with "image/"): + - If user wants to "include" or "show" images → add "render" + - If user wants to "analyze", "read text", or "extract text" from images → add "extract" + - Can have BOTH "extract" and "render" if image needs both text extraction and visual rendering + +2. For text documents: + - If user mentions "template" or "structure" → "reference" or "extract" based on context + - If user mentions "reference" or "context" → "reference" + - Default → "extract" + +3. Consider output format: + - For formats like PDF, DOCX, PPTX: images usually need "render" + - For formats like CSV, JSON: usually "extract" only + - For HTML: can have both "extract" and "render" + +Return ONLY valid JSON following the structure above. +""" + return prompt + + def _findSimilarDocumentId(self, incorrectId: str, validIds: set) -> Optional[str]: + """ + Versucht eine ähnliche Dokument-ID zu finden, falls die AI die UUID geändert hat. + Prüft auf UUID-Typo (z.B. 4451 -> 4551). + + Args: + incorrectId: Die falsche UUID aus der AI-Response + validIds: Set von gültigen Dokument-IDs + + Returns: + Korrigierte UUID falls gefunden, sonst None + """ + if not incorrectId or len(incorrectId) != 36: # UUID Format: 8-4-4-4-12 + return None + + # Prüfe ob es eine UUID ist (Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) + if incorrectId.count('-') != 4: + return None + + # Versuche Levenshtein-ähnliche Suche: Prüfe ob nur 1-2 Zeichen unterschiedlich sind + for validId in validIds: + if len(validId) != 36: + continue + + # Zähle unterschiedliche Zeichen + differences = sum(c1 != c2 for c1, c2 in zip(incorrectId, validId)) + + # Wenn nur 1-2 Zeichen unterschiedlich sind, ist es wahrscheinlich ein Typo + if differences <= 2: + # Prüfe ob die Struktur ähnlich ist (gleiche Positionen der Bindestriche) + if incorrectId.count('-') == validId.count('-'): + return validId + + return None + diff --git a/modules/serviceCenter/services/serviceAi/subJsonMerger.py b/modules/serviceCenter/services/serviceAi/subJsonMerger.py new file mode 100644 index 00000000..c5a7b058 --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subJsonMerger.py @@ -0,0 +1,2081 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Modular JSON Merger - Intelligent JSON Fragment Merging + +A clean, modular approach to merging JSON fragments that may be cut randomly. +Designed to be simple, robust, and always return valid data. + +Architecture: +1. Data Extractor: Extracts all possible data from fragments (even incomplete) +2. Structure Detector: Detects JSON structure type (elements, documents, files, etc.) +3. Data Merger: Intelligently merges data with overlap detection +4. Result Builder: Always returns valid JSON structure +""" + +import json +import re +import logging +import os +from datetime import datetime +from typing import Dict, Any, List, Optional, Tuple, Union + +from modules.shared.jsonUtils import ( + normalizeJsonText, stripCodeFences, closeJsonStructures, tryParseJson +) + +logger = logging.getLogger(__name__) + + +class JsonMergeLogger: + """Consolidated logger for JSON merging process.""" + + _logBuffer: List[str] = [] + _mergeId: int = 0 + _currentLogFile: Optional[str] = None + _appendMode: bool = False + + @staticmethod + def initializeLogFile(logFileName: Optional[str] = None): + """Initialize a new log file for a test run.""" + JsonMergeLogger._logBuffer = [] + JsonMergeLogger._mergeId = 0 + + if logFileName: + JsonMergeLogger._currentLogFile = logFileName + JsonMergeLogger._appendMode = False + # Clear existing file + try: + currentFileDir = os.path.dirname(os.path.abspath(__file__)) + logFilePath = os.path.join(currentFileDir, logFileName) + with open(logFilePath, 'w', encoding='utf-8') as f: + f.write("") # Clear file + except Exception: + pass + else: + JsonMergeLogger._currentLogFile = None + JsonMergeLogger._appendMode = False + + @staticmethod + def startMerge(accumulated: str, newFragment: str) -> str: + """Start a new merge operation and return merge ID.""" + JsonMergeLogger._mergeId += 1 + mergeId = f"merge_{JsonMergeLogger._mergeId}" + + JsonMergeLogger._log(f"{'='*80}") + JsonMergeLogger._log(f"JSON MERGE OPERATION #{JsonMergeLogger._mergeId}") + JsonMergeLogger._log(f"{'='*80}") + JsonMergeLogger._log(f"Timestamp: {datetime.now().isoformat()}") + JsonMergeLogger._log("") + + JsonMergeLogger._log("INPUT:") + JsonMergeLogger._log(f" Accumulated length: {len(accumulated)} chars") + JsonMergeLogger._log(f" New Fragment length: {len(newFragment)} chars") + # Log only summary (first 5 and last 5 lines) to avoid log spam + accLines = accumulated.split('\n') + fragLines = newFragment.split('\n') + JsonMergeLogger._log(f" Accumulated: {len(accLines)} lines (showing first 5 and last 5)") + if len(accLines) > 10: + for line in accLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(accLines) - 10} lines omitted) ...") + for line in accLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in accLines: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" New Fragment: {len(fragLines)} lines (showing first 5 and last 5)") + if len(fragLines) > 10: + for line in fragLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(fragLines) - 10} lines omitted) ...") + for line in fragLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in fragLines: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log("") + + return mergeId + + @staticmethod + def logStep(stepName: str, description: str, result: Any = None, error: Optional[str] = None): + """Log a step with its result.""" + JsonMergeLogger._log(f"STEP: {stepName}") + JsonMergeLogger._log(f" Description: {description}") + + if error: + JsonMergeLogger._log(f" ❌ ERROR: {error}") + elif result is not None: + if isinstance(result, str): + resultLines = result.split('\n') + JsonMergeLogger._log(f" ✅ Result (string, {len(result)} chars, {len(resultLines)} lines)") + if len(resultLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in resultLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(resultLines) - 10} lines omitted) ...") + for line in resultLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in resultLines: + JsonMergeLogger._log(f" {line}") + elif isinstance(result, dict): + keys = list(result.keys()) + JsonMergeLogger._log(f" ✅ Result (dict): keys={keys}, size={len(str(result))} chars") + # Log full structure with JSON formatting - NO TRUNCATION + try: + jsonStr = json.dumps(result, indent=2, ensure_ascii=False) + JsonMergeLogger._log(f" Full data (COMPLETE, {len(jsonStr)} chars):") + JsonMergeLogger._log(" " + "="*76) + for line in jsonStr.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*76) + except Exception as e: + JsonMergeLogger._log(f" Could not serialize: {e}") + strRepr = str(result) + strLines = strRepr.split('\n') + JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") + if len(strLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in strLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") + for line in strLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in strLines: + JsonMergeLogger._log(f" {line}") + # Log structure details + if "elements" in result: + elemCount = len(result["elements"]) if isinstance(result["elements"], list) else 0 + JsonMergeLogger._log(f" - elements: {elemCount} items") + if isinstance(result["elements"], list) and elemCount > 0: + JsonMergeLogger._log(f" First element type: {result['elements'][0].get('type', 'unknown') if isinstance(result['elements'][0], dict) else 'not a dict'}") + if "documents" in result: + docCount = len(result["documents"]) if isinstance(result["documents"], list) else 0 + JsonMergeLogger._log(f" - documents: {docCount} items") + elif isinstance(result, list): + JsonMergeLogger._log(f" ✅ Result (list): {len(result)} items (COMPLETE)") + if len(result) > 0: + JsonMergeLogger._log(f" First item type: {type(result[0]).__name__}") + try: + jsonStr = json.dumps(result, indent=2, ensure_ascii=False) # ALL items + JsonMergeLogger._log(f" All items (COMPLETE, {len(jsonStr)} chars):") + JsonMergeLogger._log(" " + "="*76) + for line in jsonStr.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*76) + except Exception: + strRepr = str(result) + strLines = strRepr.split('\n') + JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") + if len(strLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in strLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") + for line in strLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in strLines: + JsonMergeLogger._log(f" {line}") + else: + JsonMergeLogger._log(f" ✅ Result: {type(result).__name__} = {str(result)[:200]}") + else: + JsonMergeLogger._log(f" ⏳ In progress...") + + JsonMergeLogger._log("") + + @staticmethod + def logExtraction(strategy: str, success: bool, data: Any = None, error: Optional[str] = None): + """Log extraction strategy result.""" + status = "✅ SUCCESS" if success else "❌ FAILED" + JsonMergeLogger._log(f" Extraction Strategy: {strategy} - {status}") + if error: + JsonMergeLogger._log(f" Error: {error}") + elif data is not None: + if isinstance(data, dict): + keys = list(data.keys()) + JsonMergeLogger._log(f" Extracted keys: {keys}") + # Log full extracted data - NO TRUNCATION + try: + jsonStr = json.dumps(data, indent=2, ensure_ascii=False) + JsonMergeLogger._log(f" Extracted data (COMPLETE, {len(jsonStr)} chars):") + JsonMergeLogger._log(" " + "="*76) + for line in jsonStr.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*76) + except Exception as e: + JsonMergeLogger._log(f" Could not serialize extracted data: {e}") + strRepr = str(data) + strLines = strRepr.split('\n') + JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") + if len(strLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in strLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") + for line in strLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in strLines: + JsonMergeLogger._log(f" {line}") + elif isinstance(data, list): + JsonMergeLogger._log(f" Extracted {len(data)} items (COMPLETE)") + if len(data) > 0: + try: + jsonStr = json.dumps(data, indent=2, ensure_ascii=False) # ALL items + JsonMergeLogger._log(f" All items (COMPLETE, {len(jsonStr)} chars):") + JsonMergeLogger._log(" " + "="*76) + for line in jsonStr.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*76) + except Exception as e: + JsonMergeLogger._log(f" Could not serialize list: {e}") + strRepr = str(data) + strLines = strRepr.split('\n') + JsonMergeLogger._log(f" String representation ({len(strRepr)} chars, {len(strLines)} lines)") + if len(strLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in strLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(strLines) - 10} lines omitted) ...") + for line in strLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in strLines: + JsonMergeLogger._log(f" {line}") + + @staticmethod + def logOverlap(overlapType: str, overlapLen: int, accSuffix: Any = None, fragPrefix: Any = None): + """Log overlap detection result.""" + JsonMergeLogger._log(f" Overlap Detection ({overlapType}):") + JsonMergeLogger._log(f" Overlap length: {overlapLen}") + if overlapLen > 0: + JsonMergeLogger._log(f" ✅ Found overlap of {overlapLen} chars") + if accSuffix is not None: + if isinstance(accSuffix, str): + JsonMergeLogger._log(f" Accumulated suffix (COMPLETE, {len(accSuffix)} chars):") + JsonMergeLogger._log(" " + "="*76) + for line in accSuffix.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*76) + else: + # For lists/arrays, only log summary to avoid log flooding + if isinstance(accSuffix, list): + JsonMergeLogger._log(f" Accumulated suffix: list with {len(accSuffix)} items") + else: + JsonMergeLogger._log(f" Accumulated suffix: {type(accSuffix).__name__}") + if fragPrefix is not None: + if isinstance(fragPrefix, str): + prefixLines = fragPrefix.split('\n') + JsonMergeLogger._log(f" Fragment prefix ({len(fragPrefix)} chars, {len(prefixLines)} lines)") + if len(prefixLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 lines)") + for line in prefixLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(prefixLines) - 10} lines omitted) ...") + for line in prefixLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in prefixLines: + JsonMergeLogger._log(f" {line}") + else: + # For lists/arrays, only log summary to avoid log flooding + if isinstance(fragPrefix, list): + JsonMergeLogger._log(f" Fragment prefix: list with {len(fragPrefix)} items") + else: + JsonMergeLogger._log(f" Fragment prefix: {type(fragPrefix).__name__}") + else: + JsonMergeLogger._log(f" ⚠️ No overlap detected - appending all") + + @staticmethod + def logValidation(validationType: str, success: bool, error: Optional[str] = None): + """Log validation result.""" + status = "✅ VALID" if success else "❌ INVALID" + JsonMergeLogger._log(f" Validation ({validationType}): {status}") + if error: + JsonMergeLogger._log(f" Error: {error}") + + @staticmethod + def finishMerge(mergeId: str, finalResult: str, success: bool): + """Finish merge operation and write log file.""" + JsonMergeLogger._log("") + JsonMergeLogger._log(f"{'='*80}") + JsonMergeLogger._log(f"MERGE RESULT: {'✅ SUCCESS' if success else '❌ FAILED'}") + JsonMergeLogger._log(f"{'='*80}") + JsonMergeLogger._log(f"Final result length: {len(finalResult)} chars") + JsonMergeLogger._log("Final result (COMPLETE):") + JsonMergeLogger._log("="*80) + for line in finalResult.split('\n'): + JsonMergeLogger._log(line) + JsonMergeLogger._log("="*80) + JsonMergeLogger._log("") + + # Write log content to buffer (will be written at end of test run) + logContent = "\n".join(JsonMergeLogger._logBuffer) + + # If we have a current log file, append to it + if JsonMergeLogger._currentLogFile: + try: + currentFileDir = os.path.dirname(os.path.abspath(__file__)) + logFilePath = os.path.join(currentFileDir, JsonMergeLogger._currentLogFile) + mode = 'a' if JsonMergeLogger._appendMode else 'w' + with open(logFilePath, mode, encoding='utf-8') as f: + f.write(logContent) + f.write("\n\n") # Add separator between merges + JsonMergeLogger._appendMode = True # Next writes will append + logger.debug(f"JSON merge log appended to: {logFilePath}") + except Exception as e: + logger.error(f"Failed to write merge log file: {e}") + else: + # No log file set - write individual file (fallback) + currentFileDir = os.path.dirname(os.path.abspath(__file__)) + logDir = currentFileDir + os.makedirs(logDir, exist_ok=True) + logFilePath = os.path.join(logDir, f"{mergeId}.txt") + try: + with open(logFilePath, 'w', encoding='utf-8') as f: + f.write(logContent) + logger.info(f"JSON merge log written to: {logFilePath}") + except Exception as e: + logger.error(f"Failed to write merge log file: {e}") + + # Clear buffer for next merge + JsonMergeLogger._logBuffer = [] + + @staticmethod + def _log(message: str): + """Internal log method.""" + JsonMergeLogger._logBuffer.append(message) + # Debug logging disabled to avoid log spam with large JSON data + # logger.debug(message) + + +class JsonDataExtractor: + """Extracts data from JSON fragments, even if incomplete.""" + + @staticmethod + def extract(jsonString: str, mergeId: Optional[str] = None, removeFromEnd: bool = True) -> Dict[str, Any]: + """ + Extract complete data from JSON fragment. + + For merging: We know exactly where to clean: + - accumulated: remove incomplete parts at the END + - newFragment: remove incomplete parts at the BEGINNING + + Simple approach: Remove incomplete parts at specified position, then parse. + """ + if mergeId: + position = "END" if removeFromEnd else "BEGINNING" + JsonMergeLogger.logStep("EXTRACTION", f"Extracting data from JSON fragment ({len(jsonString)} chars) - cleaning from {position}") + + if not jsonString or not jsonString.strip(): + if mergeId: + JsonMergeLogger.logExtraction("Empty input", False, error="Input is empty") + return {} + + normalized = stripCodeFences(normalizeJsonText(jsonString)).strip() + if not normalized: + if mergeId: + JsonMergeLogger.logExtraction("Normalization", False, error="Normalized string is empty") + return {} + + # Try to parse as complete JSON first + parsed, parseErr, _ = tryParseJson(normalized) + if parseErr is None and parsed is not None: + if isinstance(parsed, dict): + finalResult = parsed + elif isinstance(parsed, list): + finalResult = {"elements": parsed} + else: + finalResult = {"elements": [parsed]} if parsed else {} + + if mergeId: + JsonMergeLogger.logExtraction("Direct parsing", True, finalResult) + JsonMergeLogger.logStep("EXTRACTION", "Direct parsing successful", finalResult) + + return finalResult if finalResult else {} + + # Remove incomplete parts from specified position + if removeFromEnd: + cleaned = JsonDataExtractor._removeIncompleteFromEnd(normalized) + else: + cleaned = JsonDataExtractor._removeIncompleteFromBeginning(normalized) + + if cleaned: + # Close structures and try to parse + closed = closeJsonStructures(cleaned) + parsed, parseErr2, _ = tryParseJson(closed) + if parseErr2 is None and parsed is not None: + if isinstance(parsed, dict): + finalResult = parsed + elif isinstance(parsed, list): + finalResult = {"elements": parsed} + else: + finalResult = {"elements": [parsed]} if parsed else {} + + if mergeId: + JsonMergeLogger.logExtraction("Remove incomplete + close", True, finalResult) + JsonMergeLogger.logStep("EXTRACTION", "Remove incomplete + close successful", finalResult) + + return finalResult if finalResult else {} + + # Return empty dict if nothing worked + if mergeId: + JsonMergeLogger.logStep("EXTRACTION", "No data extracted", {}, error="All strategies failed") + return {} + + @staticmethod + def _removeIncompleteFromEnd(jsonString: str) -> str: + """ + Remove incomplete parts from the END of JSON string. + Goes through structure level by level, keeps complete elements, removes incomplete ones at the end. + """ + # Find first '{' or '[' to start + startIdx = -1 + for i, char in enumerate(jsonString): + if char in '{[': + startIdx = i + break + + if startIdx == -1: + return "" + + # Remove incomplete parts from end recursively + cleaned = JsonDataExtractor._cleanJsonFromEnd(jsonString[startIdx:]) + return cleaned + + @staticmethod + def _removeIncompleteFromBeginning(jsonString: str) -> str: + """ + Remove incomplete parts from the BEGINNING of JSON string. + Finds where valid JSON starts and removes everything before it. + """ + # Find first '{' or '[' to start + startIdx = -1 + for i, char in enumerate(jsonString): + if char in '{[': + startIdx = i + break + + if startIdx == -1: + return "" + + # Return from start position - beginning cleanup is just finding the start + return jsonString[startIdx:] + + @staticmethod + def _cleanJsonFromEnd(jsonStr: str) -> str: + """ + Recursively clean JSON from the END: keep complete elements, remove incomplete ones at the end. + Goes through structure level by level. + """ + # Try to parse as-is first + try: + parsed = json.loads(jsonStr) + return jsonStr + except Exception: + pass + + # If dict: go through each key-value pair, remove incomplete ones at the end + if jsonStr.strip().startswith('{'): + return JsonDataExtractor._cleanDictFromEnd(jsonStr) + + # If array: go through each element, remove incomplete ones at the end + if jsonStr.strip().startswith('['): + return JsonDataExtractor._cleanArrayFromEnd(jsonStr) + + return "" + + @staticmethod + def _cleanDictFromEnd(jsonStr: str) -> str: + """Clean dict from END: keep complete key-value pairs, remove incomplete ones at the end.""" + if not jsonStr.strip().startswith('{'): + return "" + + result = ['{'] + i = 1 # Skip opening '{' + first = True + + while i < len(jsonStr): + # Skip whitespace + while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': + i += 1 + + if i >= len(jsonStr): + break + + # Check if we hit closing brace + if jsonStr[i] == '}': + break + + # Skip comma + if jsonStr[i] == ',': + i += 1 + continue + + # Try to extract key-value pair + keyStart = i + # Find key (string) + if jsonStr[i] == '"': + i += 1 + while i < len(jsonStr) and jsonStr[i] != '"': + if jsonStr[i] == '\\': + i += 2 + else: + i += 1 + if i < len(jsonStr): + i += 1 # Skip closing quote + else: + # Invalid key - stop here (incomplete at end) + break + + # Skip whitespace and colon + while i < len(jsonStr) and jsonStr[i] in ' \n\r\t:': + i += 1 + + if i >= len(jsonStr): + break + + # Try to extract value + valueStart = i + valueEnd = JsonDataExtractor._findCompleteValue(jsonStr, i) + + if valueEnd > valueStart: + # Try to parse this key-value pair + pairStr = jsonStr[keyStart:valueEnd] + try: + # Test if it's valid JSON + testStr = '{' + pairStr + '}' + json.loads(testStr) + # Valid pair - add it + if not first: + result.append(',') + result.append(pairStr) + first = False + i = valueEnd + except Exception: + # Invalid pair - stop here (incomplete at end) + break + else: + # Incomplete value - stop here (incomplete at end) + break + + result.append('}') + return ''.join(result) + + @staticmethod + def _cleanArrayFromEnd(jsonStr: str) -> str: + """Clean array from END: keep complete elements, remove incomplete ones at the end.""" + if not jsonStr.strip().startswith('['): + return "" + + result = ['['] + i = 1 # Skip opening '[' + first = True + + while i < len(jsonStr): + # Skip whitespace + while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': + i += 1 + + if i >= len(jsonStr): + break + + # Check if we hit closing bracket + if jsonStr[i] == ']': + break + + # Skip comma + if jsonStr[i] == ',': + i += 1 + continue + + # Try to extract element + elemStart = i + elemEnd = JsonDataExtractor._findCompleteValue(jsonStr, i) + + if elemEnd > elemStart: + # Try to parse this element + elemStr = jsonStr[elemStart:elemEnd] + try: + # Test if it's valid JSON + json.loads(elemStr) + # Valid element - add it + if not first: + result.append(',') + result.append(elemStr) + first = False + i = elemEnd + except Exception: + # Invalid element - stop here (incomplete at end) + break + else: + # Incomplete element - stop here (incomplete at end) + break + + result.append(']') + return ''.join(result) + + @staticmethod + def _findCompleteValue(jsonStr: str, start: int) -> int: + """Find the end of a complete JSON value starting at start position.""" + if start >= len(jsonStr): + return start + + i = start + + # Skip whitespace + while i < len(jsonStr) and jsonStr[i] in ' \n\r\t': + i += 1 + + if i >= len(jsonStr): + return start + + char = jsonStr[i] + + # String + if char == '"': + i += 1 + while i < len(jsonStr): + if jsonStr[i] == '\\': + i += 2 + elif jsonStr[i] == '"': + return i + 1 + else: + i += 1 + return start # Incomplete string + + # Number, boolean, null + if char in '-0123456789tfn': + while i < len(jsonStr) and jsonStr[i] not in ',}]': + i += 1 + return i + + # Object + if char == '{': + braceCount = 1 + i += 1 + while i < len(jsonStr) and braceCount > 0: + if jsonStr[i] == '\\': + i += 2 + elif jsonStr[i] == '"': + # Skip string + i += 1 + while i < len(jsonStr): + if jsonStr[i] == '\\': + i += 2 + elif jsonStr[i] == '"': + i += 1 + break + else: + i += 1 + elif jsonStr[i] == '{': + braceCount += 1 + i += 1 + elif jsonStr[i] == '}': + braceCount -= 1 + i += 1 + else: + i += 1 + if braceCount == 0: + return i + return start # Incomplete object + + # Array + if char == '[': + bracketCount = 1 + i += 1 + while i < len(jsonStr) and bracketCount > 0: + if jsonStr[i] == '\\': + i += 2 + elif jsonStr[i] == '"': + # Skip string + i += 1 + while i < len(jsonStr): + if jsonStr[i] == '\\': + i += 2 + elif jsonStr[i] == '"': + i += 1 + break + else: + i += 1 + elif jsonStr[i] == '[': + bracketCount += 1 + i += 1 + elif jsonStr[i] == ']': + bracketCount -= 1 + i += 1 + else: + i += 1 + if bracketCount == 0: + return i + return start # Incomplete array + + return start + + @staticmethod + def _extractAllCompleteObjects(jsonString: str) -> List[Dict[str, Any]]: + """ + Extract ALL complete objects from JSON string using balanced brace matching. + Ignores incomplete objects at the end. + + Core principle: Every fragment can be cut anywhere - extract only complete objects. + """ + foundObjs = [] + braceCount = 0 + startPos = -1 + + for i, char in enumerate(jsonString): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + # Found a complete object + objStr = jsonString[startPos:i+1] + try: + obj = json.loads(objStr) + if isinstance(obj, dict) and obj: + foundObjs.append(obj) + except Exception: + # Not valid JSON - skip it + pass + startPos = -1 + elif braceCount < 0: + # Unbalanced - reset + braceCount = 0 + startPos = -1 + + # If we end with an incomplete object (startPos >= 0 and braceCount > 0), ignore it + # It will be in the next fragment + + return foundObjs + + @staticmethod + def _extractElements(jsonString: str) -> List[Dict[str, Any]]: + """Extract elements array from JSON string - extracts ALL complete elements.""" + elements = [] + + # Pattern 1: Look for "elements": [...] (including incomplete at end) + elementsPattern = r'"elements"\s*:\s*\[(.*)' + match = re.search(elementsPattern, jsonString, re.DOTALL) + if match: + elementsContent = match.group(1) + # Extract ALL complete element objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(elementsContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + elementStr = elementsContent[startPos:i+1] + try: + element = json.loads(elementStr) + if isinstance(element, dict): + elements.append(element) + except Exception: + # Try to extract table rows from incomplete element + rows = JsonDataExtractor._extractTableRowsFromElement(elementStr) + if rows: + elements.append({ + "type": "table", + "content": { + "rows": rows + } + }) + startPos = -1 + elif braceCount < 0: + break # Unbalanced - stop + + # Pattern 2: Look for table structure directly (even if incomplete) + if not elements: + # Look for "type": "table" pattern + tablePattern = r'"type"\s*:\s*"table"[^}]*"rows"\s*:\s*\[(.*?)(?:\]|$)' + tableMatch = re.search(tablePattern, jsonString, re.DOTALL) + if tableMatch: + rowsContent = tableMatch.group(1) + rows = JsonDataExtractor._extractRowsFromContent(rowsContent) + if rows: + elements.append({ + "type": "table", + "content": { + "rows": rows + } + }) + + # Pattern 3: Look for table rows directly (without structure) + if not elements: + rows = JsonDataExtractor._extractTableRows(jsonString) + if rows: + elements.append({ + "type": "table", + "content": { + "rows": rows + } + }) + + return elements + + @staticmethod + def _extractTableRowsFromElement(elementStr: str) -> List[List[str]]: + """Extract table rows from incomplete element string.""" + # Look for rows array in element + rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' + match = re.search(rowsPattern, elementStr, re.DOTALL) + if match: + return JsonDataExtractor._extractRowsFromContent(match.group(1)) + return [] + + @staticmethod + def _extractRowsFromContent(rowsContent: str) -> List[List[str]]: + """Extract rows from rows content string.""" + rows = [] + # Extract all array patterns: ["value1", "value2"] + # Use non-greedy matching but ensure we get complete arrays + arrayPattern = r'\[(.*?)\]' + arrayMatches = re.findall(arrayPattern, rowsContent) + for arrayContent in arrayMatches: + # Extract cells - handle both quoted strings and numbers + # First try to find quoted strings + cellPattern = r'"([^"]*)"' + cells = re.findall(cellPattern, arrayContent) + # If no quoted strings, try numbers or other values + if not cells: + # Try to find any values (numbers, booleans, etc.) + valuePattern = r'(-?\d+\.?\d*|true|false|null)' + cells = re.findall(valuePattern, arrayContent) + # Only add rows with at least 1 cell (allow single-column tables) + if len(cells) >= 1: + rows.append(cells) + return rows + + @staticmethod + def _extractTableRows(jsonString: str) -> List[List[str]]: + """Extract table rows from JSON string using multiple strategies.""" + rows = [] + + # Strategy 1: Look for "rows": [[...], [...]] + rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' + match = re.search(rowsPattern, jsonString, re.DOTALL) + if match: + rowsContent = match.group(1) + rows = JsonDataExtractor._extractRowsFromContent(rowsContent) + if rows: + return rows + + # Strategy 2: Look for standalone array patterns ["value1", "value2"] + # Pattern for complete arrays with 2 columns + completeArrayPattern = r'\["([^"]*)",\s*"([^"]*)"\]' + matches = re.findall(completeArrayPattern, jsonString) + if len(matches) >= 2: # Need at least 2 rows to be confident + return [[m[0], m[1]] for m in matches] + + # Strategy 3: Extract any array patterns (more lenient) + # Find all [ ... ] patterns that contain quoted strings + allArrays = re.findall(r'\[([^\]]*)\]', jsonString) + for arrayContent in allArrays: + # Extract quoted strings + cells = re.findall(r'"([^"]*)"', arrayContent) + if len(cells) >= 2: # At least 2 columns + rows.append(cells) + + # Only return if we have multiple rows (likely a table) + if len(rows) >= 2: + return rows + + return [] + + @staticmethod + def _extractDocuments(jsonString: str) -> List[Dict[str, Any]]: + """ + Extract documents structure from JSON string - extracts ALL complete documents/chapters/sections. + Ignores incomplete ones at the end. + + Core principle: Fragment can be cut anywhere - extract only complete objects. + """ + documents = [] + + # Pattern 1: Look for "documents": [...] structure (including incomplete at end) + documentsPattern = r'"documents"\s*:\s*\[(.*)' + match = re.search(documentsPattern, jsonString, re.DOTALL) + if match: + documentsContent = match.group(1) + # Extract ALL complete document objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(documentsContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + # Found a complete document object + docStr = documentsContent[startPos:i+1] + try: + doc = json.loads(docStr) + if isinstance(doc, dict): + # Extract chapters/sections from document + chapters = JsonDataExtractor._extractChaptersFromDocument(docStr) + sections = JsonDataExtractor._extractSectionsFromDocument(docStr) + if chapters: + doc["chapters"] = chapters + if sections: + doc["sections"] = sections + if doc: + documents.append(doc) + except Exception: + # Not valid JSON - try to extract chapters/sections directly + chapters = JsonDataExtractor._extractChaptersFromDocument(docStr) + sections = JsonDataExtractor._extractSectionsFromDocument(docStr) + if chapters or sections: + doc = {} + if chapters: + doc["chapters"] = chapters + if sections: + doc["sections"] = sections + if doc: + documents.append(doc) + startPos = -1 + elif braceCount < 0: + break + + # If we end with an incomplete document (startPos >= 0 and braceCount > 0), ignore it + # It will be in the next fragment + + if documents: + return documents + + # Pattern 2: Look for "chapters": [...] pattern directly (fragment might start mid-document) + chapters = JsonDataExtractor._extractChaptersFromString(jsonString) + if chapters: + documents.append({"chapters": chapters}) + + # Pattern 3: Look for "sections": [...] pattern directly + sections = JsonDataExtractor._extractSectionsFromString(jsonString) + if sections: + documents.append({"sections": sections}) + + return documents + + @staticmethod + def _extractChaptersFromDocument(docStr: str) -> List[Dict[str, Any]]: + """Extract chapters array from document string.""" + return JsonDataExtractor._extractChaptersFromString(docStr) + + @staticmethod + def _extractChaptersFromString(jsonString: str) -> List[Dict[str, Any]]: + """ + Extract chapters array from JSON string - extracts ALL complete chapters. + Ignores incomplete chapters at the end. + + Core principle: Fragment can be cut anywhere - extract only complete objects. + """ + chapters = [] + + # Look for "chapters": [...] pattern (including incomplete at end) + chaptersPattern = r'"chapters"\s*:\s*\[(.*)' + match = re.search(chaptersPattern, jsonString, re.DOTALL) + if match: + chaptersContent = match.group(1) + # Extract ALL complete chapter objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(chaptersContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + # Found a complete chapter object + chapterStr = chaptersContent[startPos:i+1] + try: + chapter = json.loads(chapterStr) + if isinstance(chapter, dict): + chapters.append(chapter) + except Exception: + # Not valid JSON - skip it (incomplete chapter) + pass + startPos = -1 + elif braceCount < 0: + # Unbalanced - stop here + break + + # If we end with an incomplete chapter (startPos >= 0 and braceCount > 0), ignore it + # It will be in the next fragment + + # Also try to extract chapters that might be standalone (fragment starts mid-array) + # Look for complete chapter objects anywhere in the string + if not chapters: + # Try to find complete chapter objects using balanced brace matching + allObjs = JsonDataExtractor._extractAllCompleteObjects(jsonString) + # Filter for objects that look like chapters (have id and title) + for obj in allObjs: + if isinstance(obj, dict) and "id" in obj and "title" in obj: + chapters.append(obj) + + return chapters + + @staticmethod + def _extractSectionsFromDocument(docStr: str) -> List[Dict[str, Any]]: + """Extract sections array from document string.""" + return JsonDataExtractor._extractSectionsFromString(docStr) + + @staticmethod + def _extractSectionsFromString(jsonString: str) -> List[Dict[str, Any]]: + """Extract sections array from JSON string, even if incomplete.""" + sections = [] + + # Look for "sections": [...] + sectionsPattern = r'"sections"\s*:\s*\[(.*?)(?:\]|$)' + match = re.search(sectionsPattern, jsonString, re.DOTALL) + if match: + sectionsContent = match.group(1) + # Extract section objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(sectionsContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + sectionStr = sectionsContent[startPos:i+1] + try: + section = json.loads(sectionStr) + if isinstance(section, dict): + sections.append(section) + except Exception: + # Incomplete section - try to extract what we can + idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', sectionStr) + contentTypeMatch = re.search(r'"content_type"\s*:\s*"([^"]*)"', sectionStr) + if idMatch or contentTypeMatch: + section = {} + if idMatch: + section["id"] = idMatch.group(1) + if contentTypeMatch: + section["content_type"] = contentTypeMatch.group(1) + if section: + sections.append(section) + startPos = -1 + + return sections + + @staticmethod + def _extractFiles(jsonString: str) -> List[Dict[str, Any]]: + """Extract files array from JSON string, even if incomplete.""" + files = [] + + # Look for "files": [...] + filesPattern = r'"files"\s*:\s*\[(.*?)(?:\]|$)' + match = re.search(filesPattern, jsonString, re.DOTALL) + if match: + filesContent = match.group(1) + # Extract file objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(filesContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + fileStr = filesContent[startPos:i+1] + try: + fileObj = json.loads(fileStr) + if isinstance(fileObj, dict): + files.append(fileObj) + except Exception: + # Incomplete file - try to extract what we can + idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', fileStr) + filenameMatch = re.search(r'"filename"\s*:\s*"([^"]*)"', fileStr) + if idMatch or filenameMatch: + fileObj = {} + if idMatch: + fileObj["id"] = idMatch.group(1) + if filenameMatch: + fileObj["filename"] = filenameMatch.group(1) + if fileObj: + files.append(fileObj) + startPos = -1 + + return files + + @staticmethod + def _extractImages(jsonString: str) -> List[Dict[str, Any]]: + """Extract images array from JSON string, even if incomplete.""" + images = [] + + # Look for "images": [...] + imagesPattern = r'"images"\s*:\s*\[(.*?)(?:\]|$)' + match = re.search(imagesPattern, jsonString, re.DOTALL) + if match: + imagesContent = match.group(1) + # Extract image objects using balanced brace matching + braceCount = 0 + startPos = -1 + for i, char in enumerate(imagesContent): + if char == '{': + if braceCount == 0: + startPos = i + braceCount += 1 + elif char == '}': + braceCount -= 1 + if braceCount == 0 and startPos >= 0: + imageStr = imagesContent[startPos:i+1] + try: + image = json.loads(imageStr) + if isinstance(image, dict): + images.append(image) + except Exception: + # Incomplete image - try to extract what we can + idMatch = re.search(r'"id"\s*:\s*"([^"]*)"', imageStr) + urlMatch = re.search(r'"url"\s*:\s*"([^"]*)"', imageStr) + if idMatch or urlMatch: + image = {} + if idMatch: + image["id"] = idMatch.group(1) + if urlMatch: + image["url"] = urlMatch.group(1) + if image: + images.append(image) + startPos = -1 + + return images + + +class JsonStructureDetector: + """Detects JSON structure type from extracted data.""" + + @staticmethod + def detect(data: Dict[str, Any], mergeId: Optional[str] = None) -> str: + """ + Detect structure type from data - GENERIC approach. + + Only checks for top-level keys, no content analysis. + + Returns: + Structure type: "elements", "documents", "files", "images", or "unknown" + """ + if "elements" in data: + structureType = "elements" + elif "documents" in data: + structureType = "documents" + elif "files" in data: + structureType = "files" + elif "images" in data: + structureType = "images" + else: + # Unknown structure - will be handled generically + structureType = "unknown" + + if mergeId: + JsonMergeLogger.logStep("DETECTION", f"Detected structure type: {structureType}", structureType) + + return structureType + + +class JsonDataMerger: + """Merges JSON data intelligently with overlap detection.""" + + @staticmethod + def merge( + accumulated: Dict[str, Any], + newFragment: Dict[str, Any], + structureType: str, + mergeId: Optional[str] = None + ) -> Dict[str, Any]: + """ + Merge two JSON data structures. + + Args: + accumulated: Previously accumulated data + newFragment: New fragment data + structureType: Detected structure type + mergeId: Optional merge ID for logging + + Returns: + Merged data structure + """ + if mergeId: + JsonMergeLogger.logStep("MERGING", f"Merging {structureType} structures", { + "acc_keys": list(accumulated.keys()) if accumulated else [], + "frag_keys": list(newFragment.keys()) if newFragment else [] + }) + + if not accumulated: + if mergeId: + JsonMergeLogger.logStep("MERGING", "No accumulated data, returning fragment", newFragment) + return newFragment if newFragment else {} + if not newFragment: + if mergeId: + JsonMergeLogger.logStep("MERGING", "No fragment data, returning accumulated", accumulated) + return accumulated + + # Merge based on structure type + if structureType == "elements": + result = JsonDataMerger._mergeElements(accumulated, newFragment) + elif structureType == "documents": + result = JsonDataMerger._mergeDocuments(accumulated, newFragment) + elif structureType == "files": + result = JsonDataMerger._mergeFiles(accumulated, newFragment) + elif structureType == "images": + result = JsonDataMerger._mergeImages(accumulated, newFragment) + else: + # Unknown structure - try to merge generically + result = JsonDataMerger._mergeGeneric(accumulated, newFragment) + + if mergeId: + JsonMergeLogger.logStep("MERGING", f"Merged {structureType} structures", result) + + return result + + @staticmethod + def _mergeElements(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: + """Merge elements structures.""" + accElements = accumulated.get("elements", []) + fragElements = newFragment.get("elements", []) + + if not accElements: + return {"elements": fragElements} if fragElements else accumulated + if not fragElements: + return {"elements": accElements} + + # Merge elements with overlap detection + mergedElements = JsonDataMerger._mergeElementList(accElements, fragElements) + + return {"elements": mergedElements} + + @staticmethod + def _mergeElementList(accElements: List[Dict[str, Any]], fragElements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Merge two element lists with overlap detection.""" + if not accElements: + return fragElements + if not fragElements: + return accElements + + # Special handling: if both have table elements, merge them intelligently + accTables = [e for e in accElements if isinstance(e, dict) and e.get("type") == "table"] + fragTables = [e for e in fragElements if isinstance(e, dict) and e.get("type") == "table"] + + if accTables and fragTables: + # Merge table elements + mergedTable = JsonDataMerger._mergeTableElements(accTables[0], fragTables[0]) + if mergedTable: + # Replace tables with merged table + otherAccElements = [e for e in accElements if not (isinstance(e, dict) and e.get("type") == "table")] + otherFragElements = [e for e in fragElements if not (isinstance(e, dict) and e.get("type") == "table")] + return otherAccElements + [mergedTable] + otherFragElements + + # Find overlap by comparing elements + overlapStart = JsonDataMerger._findOverlap(accElements, fragElements, None, "elements") + + if overlapStart > 0: + # Found overlap - remove overlapping elements from fragment + merged = accElements + fragElements[overlapStart:] + return merged + else: + # No overlap - append all + return accElements + fragElements + + @staticmethod + def _mergeTableElements(accTable: Dict[str, Any], fragTable: Dict[str, Any]) -> Dict[str, Any]: + """Merge two table elements by merging their rows.""" + accRows = JsonDataMerger._getTableRows(accTable) + fragRows = JsonDataMerger._getTableRows(fragTable) + + if not accRows: + return fragTable + if not fragRows: + return accTable + + # Find overlap in rows + overlapStart = JsonDataMerger._findOverlap(accRows, fragRows, None, "table_rows") + + # Merge rows + mergedRows = accRows + fragRows[overlapStart:] if overlapStart > 0 else accRows + fragRows + + # Build merged table + mergedTable = accTable.copy() + content = mergedTable.get("content", {}) + if not isinstance(content, dict): + content = {} + content["rows"] = mergedRows + + # Preserve headers + if "headers" not in content: + fragContent = fragTable.get("content", {}) + if isinstance(fragContent, dict) and "headers" in fragContent: + content["headers"] = fragContent["headers"] + + mergedTable["content"] = content + return mergedTable + + @staticmethod + def _findOverlap(accList: List[Any], fragList: List[Any], mergeId: Optional[str] = None, overlapType: str = "generic") -> int: + """Find overlap between two lists. Returns index where overlap starts in fragList.""" + if not accList or not fragList: + if mergeId: + JsonMergeLogger.logOverlap(overlapType, 0) + return 0 + + # Try to find longest common suffix/prefix + maxOverlap = min(len(accList), len(fragList)) + + for overlapLen in range(maxOverlap, 0, -1): + accSuffix = accList[-overlapLen:] + fragPrefix = fragList[:overlapLen] + + # Compare elements + if JsonDataMerger._listsEqual(accSuffix, fragPrefix): + if mergeId: + JsonMergeLogger.logOverlap(overlapType, overlapLen, accSuffix, fragPrefix) + return overlapLen + + if mergeId: + JsonMergeLogger.logOverlap(overlapType, 0) + return 0 + + @staticmethod + def _listsEqual(list1: List[Any], list2: List[Any]) -> bool: + """Check if two lists are equal (deep comparison for dicts).""" + if len(list1) != len(list2): + return False + + for i in range(len(list1)): + if isinstance(list1[i], dict) and isinstance(list2[i], dict): + # Compare dicts by comparing their content + if not JsonDataMerger._dictsEqual(list1[i], list2[i]): + return False + elif list1[i] != list2[i]: + return False + + return True + + @staticmethod + def _dictsEqual(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> bool: + """Check if two dicts are equal (comparing key content).""" + # For table elements, compare rows + if dict1.get("type") == "table" and dict2.get("type") == "table": + rows1 = JsonDataMerger._getTableRows(dict1) + rows2 = JsonDataMerger._getTableRows(dict2) + return rows1 == rows2 + + # For other elements, compare type and key content + if dict1.get("type") != dict2.get("type"): + return False + + # Compare content + content1 = dict1.get("content", {}) + content2 = dict2.get("content", {}) + + if isinstance(content1, dict) and isinstance(content2, dict): + # Compare rows for tables + if "rows" in content1 and "rows" in content2: + return content1["rows"] == content2["rows"] + # Compare items for lists + if "items" in content1 and "items" in content2: + return content1["items"] == content2["items"] + + return dict1 == dict2 + + @staticmethod + def _getTableRows(element: Dict[str, Any]) -> List[List[str]]: + """Extract table rows from element.""" + content = element.get("content", {}) + if isinstance(content, dict): + return content.get("rows", []) + return element.get("rows", []) + + @staticmethod + def _mergeDocuments(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: + """Merge documents structures.""" + accDocs = accumulated.get("documents", []) + fragDocs = newFragment.get("documents", []) + + if not accDocs: + return {"documents": fragDocs} if fragDocs else accumulated + if not fragDocs: + return {"documents": accDocs} + + # Merge documents (simplified - would need proper merging logic) + mergedDocs = accDocs + fragDocs + return {"documents": mergedDocs} + + @staticmethod + def _mergeFiles(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: + """Merge files structures.""" + accFiles = accumulated.get("files", []) + fragFiles = newFragment.get("files", []) + + if not accFiles: + return {"files": fragFiles} if fragFiles else accumulated + if not fragFiles: + return {"files": accFiles} + + mergedFiles = accFiles + fragFiles + return {"files": mergedFiles} + + @staticmethod + def _mergeImages(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: + """Merge images structures.""" + accImages = accumulated.get("images", []) + fragImages = newFragment.get("images", []) + + if not accImages: + return {"images": fragImages} if fragImages else accumulated + if not fragImages: + return {"images": accImages} + + mergedImages = accImages + fragImages + return {"images": mergedImages} + + @staticmethod + def _mergeGeneric(accumulated: Dict[str, Any], newFragment: Dict[str, Any]) -> Dict[str, Any]: + """Generic merge for unknown structures.""" + # Try to merge by combining keys + merged = accumulated.copy() + for key, value in newFragment.items(): + if key in merged: + # Key exists - try to merge values + if isinstance(merged[key], list) and isinstance(value, list): + merged[key] = merged[key] + value + elif isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = JsonDataMerger._mergeGeneric(merged[key], value) + else: + merged[key] = value + else: + merged[key] = value + + return merged + + +class JsonResultBuilder: + """Builds final JSON result, ensuring it's always valid.""" + + @staticmethod + def build(mergedData: Dict[str, Any], structureType: str, mergeId: Optional[str] = None) -> str: + """ + Build final JSON string from merged data. + + Args: + mergedData: Merged data structure + structureType: Detected structure type + + Returns: + Valid JSON string (never empty) + """ + if not mergedData: + # Return empty structure based on type + if structureType == "elements": + return json.dumps({"elements": []}, indent=2, ensure_ascii=False) + elif structureType == "documents": + return json.dumps({"documents": [{}]}, indent=2, ensure_ascii=False) + elif structureType == "files": + return json.dumps({"files": []}, indent=2, ensure_ascii=False) + elif structureType == "images": + return json.dumps({"images": []}, indent=2, ensure_ascii=False) + else: + return json.dumps({}, indent=2, ensure_ascii=False) + + # Ensure structure is correct - GENERIC approach + if structureType == "elements" and "elements" not in mergedData: + # Try to wrap data in elements structure + if isinstance(mergedData, dict): + # Generic: If it has any data, wrap it as an element + if mergedData: + mergedData = {"elements": [mergedData]} + if mergeId: + JsonMergeLogger.logStep("BUILDING", "Wrapping single object as element (generic)", mergedData) + else: + # Empty dict - return empty elements + mergedData = {"elements": []} + + elif structureType == "documents" and "documents" not in mergedData: + # Try to wrap data in documents structure + if isinstance(mergedData, dict): + if mergedData: + # Generic: Wrap single object in documents structure + # Try to detect if it should be chapters or sections by checking accumulated data + # But for now, use generic approach: wrap in documents with a generic key + mergedData = {"documents": [mergedData]} + if mergeId: + JsonMergeLogger.logStep("BUILDING", "Wrapping single object in documents structure (generic)", mergedData) + else: + mergedData = {"documents": [{}]} + + elif structureType == "files" and "files" not in mergedData: + # Try to wrap data in files structure + if isinstance(mergedData, dict): + if mergedData: + mergedData = {"files": [mergedData]} + if mergeId: + JsonMergeLogger.logStep("BUILDING", "Wrapping single object in files structure (generic)", mergedData) + else: + mergedData = {"files": []} + + elif structureType == "images" and "images" not in mergedData: + # Try to wrap data in images structure + if isinstance(mergedData, dict): + if mergedData: + mergedData = {"images": [mergedData]} + if mergeId: + JsonMergeLogger.logStep("BUILDING", "Wrapping single object in images structure (generic)", mergedData) + else: + mergedData = {"images": []} + + elif structureType == "unknown" and isinstance(mergedData, dict) and mergedData: + # Unknown structure but has data - wrap generically as elements + mergedData = {"elements": [mergedData]} + if mergeId: + JsonMergeLogger.logStep("BUILDING", "Unknown structure, wrapping as elements (generic)", mergedData) + + # Clean data structure before serialization + cleanedData = JsonResultBuilder._cleanDataStructure(mergedData) + + # Try to serialize + try: + jsonString = json.dumps(cleanedData, indent=2, ensure_ascii=False) + + # Validate the JSON string by trying to parse it + try: + parsed, parseErr, _ = tryParseJson(jsonString) + if parseErr is None: + # Valid JSON - return it + return jsonString + else: + # Invalid JSON - try to repair + logger.warning(f"Generated JSON is invalid: {parseErr}, attempting repair") + repaired = closeJsonStructures(jsonString) + parsed2, parseErr2, _ = tryParseJson(repaired) + if parseErr2 is None: + return repaired + else: + # Repair failed - return minimal valid structure + logger.error(f"Repair failed: {parseErr2}, returning minimal structure") + return json.dumps({"elements": []}, indent=2, ensure_ascii=False) + except Exception as parseEx: + # Parse validation failed - try repair + logger.warning(f"Parse validation failed: {parseEx}, attempting repair") + try: + repaired = closeJsonStructures(jsonString) + parsed2, parseErr2, _ = tryParseJson(repaired) + if parseErr2 is None: + return repaired + except Exception: + pass + # Return minimal valid structure + return json.dumps({"elements": []}, indent=2, ensure_ascii=False) + + except (TypeError, ValueError) as e: + logger.error(f"Error serializing JSON: {e}") + # Try to clean more aggressively and retry + try: + cleanedData2 = JsonResultBuilder._cleanDataStructure(cleanedData, aggressive=True) + jsonString = json.dumps(cleanedData2, indent=2, ensure_ascii=False) + # Validate + parsed, parseErr, _ = tryParseJson(jsonString) + if parseErr is None: + return jsonString + except Exception: + pass + # Fallback to empty structure + return json.dumps({"elements": []}, indent=2, ensure_ascii=False) + except Exception as e: + logger.error(f"Unexpected error building JSON: {e}") + # Fallback to empty structure + return json.dumps({"elements": []}, indent=2, ensure_ascii=False) + + @staticmethod + def _cleanDataStructure(data: Any, aggressive: bool = False) -> Any: + """ + Clean data structure to ensure it's JSON-serializable. + + Removes None values, ensures lists contain only valid items, + and repairs incomplete structures. + """ + if data is None: + return {} if aggressive else None + + if isinstance(data, dict): + cleaned = {} + for key, value in data.items(): + if value is None and aggressive: + continue # Skip None values in aggressive mode + cleaned[key] = JsonResultBuilder._cleanDataStructure(value, aggressive) + return cleaned + + elif isinstance(data, list): + cleaned = [] + for item in data: + cleanedItem = JsonResultBuilder._cleanDataStructure(item, aggressive) + if cleanedItem is not None or not aggressive: + cleaned.append(cleanedItem) + return cleaned + + elif isinstance(data, (str, int, float, bool)): + return data + + else: + # Unknown type - try to convert to string or skip + if aggressive: + return str(data) + return data + + +class ModularJsonMerger: + """ + Modular JSON Merger - Main entry point. + + Simple pipeline: + 1. Find overlap between JSON strings + 2. Merge strings together + 3. Parse and clean the merged JSON + """ + + @staticmethod + def _findStringOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: + """ + Find overlap between two JSON strings - GENERIC solution. + + Works for any JSON structure (arrays, objects, nested, minified, formatted). + Uses multiple strategies to find overlap regardless of JSON format. + + Strategy: + 1. Exact suffix/prefix match (fastest, works for any format) + 2. Structure-aware: Find last complete JSON elements in accumulated that match start of fragment + 3. Line-based: If JSON is formatted, use line matching (for better performance) + 4. Partial match: Handle incomplete elements at cut point + + Returns the length of the overlap (number of characters). + """ + if not accStr or not fragStr: + if mergeId: + JsonMergeLogger.logOverlap("string", 0) + return 0 + + # Strategy 1: Try exact suffix/prefix match (fastest, works for any format) + maxOverlap = min(len(accStr), len(fragStr)) + + # Start from maximum possible overlap and work backwards + for overlapLen in range(maxOverlap, 0, -1): + accSuffix = accStr[-overlapLen:] + fragPrefix = fragStr[:overlapLen] + + if accSuffix == fragPrefix: + if mergeId: + JsonMergeLogger.logOverlap("string (exact)", overlapLen, accSuffix[:200], fragPrefix[:200]) + return overlapLen + + # Strategy 2: Structure-aware overlap detection (GENERIC - works for any JSON structure) + # Find last complete JSON elements in accumulated and check if they appear at start of fragment + overlapLen = ModularJsonMerger._findStructureBasedOverlap(accStr, fragStr, mergeId) + if overlapLen > 0: + return overlapLen + + # Strategy 3: Line-based overlap (works well for formatted JSON) + # Only use if JSON appears to be formatted (has newlines) + if '\n' in accStr and '\n' in fragStr: + overlapLen = ModularJsonMerger._findLineBasedOverlap(accStr, fragStr, mergeId) + if overlapLen > 0: + return overlapLen + + # Strategy 4: Partial overlap (incomplete element at cut point) + overlapLen = ModularJsonMerger._findPartialOverlap(accStr, fragStr, mergeId) + if overlapLen > 0: + return overlapLen + + if mergeId: + JsonMergeLogger.logOverlap("string", 0) + return 0 + + @staticmethod + def _findStructureBasedOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: + """ + Find overlap by detecting complete JSON elements (structure-aware, GENERIC). + + Works for ANY JSON structure: + - Arrays: Finds last complete array elements + - Objects: Finds last complete object properties + - Nested structures: Recursively finds complete elements + - Minified or formatted JSON: Structure-aware, not format-dependent + - Any use case: section_content, chapter_structure, code_structure, etc. + + Strategy: Find last complete JSON elements in accumulated that match start of fragment. + Uses balanced bracket/brace matching to identify complete elements regardless of format. + """ + accTrimmed = accStr.rstrip() + fragTrimmed = fragStr.lstrip() + + if not accTrimmed or not fragTrimmed: + return 0 + + # Find last complete elements in accumulated by parsing backwards + # Look for complete array elements or object properties + + # Strategy: Find where accumulated has complete elements at the end + # and check if fragment starts with the same elements + + # Use a sliding window approach: check different suffix lengths from accumulated + maxCheckLength = min(2000, len(accTrimmed), len(fragTrimmed)) + + # Check in reverse order (largest to smallest) to find longest overlap first + for checkLen in range(maxCheckLength, 50, -5): # Step by 5 for performance + if checkLen > len(accTrimmed) or checkLen > len(fragTrimmed): + continue + + accSuffix = accTrimmed[-checkLen:] + fragPrefix = fragTrimmed[:checkLen] + + # Check if accSuffix ends with complete JSON element(s) and fragPrefix starts with same + # A complete element ends with proper closing brackets/braces + + # Verify that accSuffix ends with complete structure + # and fragPrefix starts with the same structure + if ModularJsonMerger._isCompleteJsonElement(accSuffix) and \ + ModularJsonMerger._startsWithSameElement(accSuffix, fragPrefix): + # Found overlap! Verify it's meaningful (not just whitespace) + if len(accSuffix.strip()) > 20: + if mergeId: + JsonMergeLogger.logOverlap("string (structure-based)", checkLen, accSuffix[:200], fragPrefix[:200]) + return checkLen + + # Alternative: Try to find common substring that represents complete elements + # Look for patterns like complete array rows or object properties + # Check last 500 chars of accumulated against first 500 chars of fragment + checkWindow = min(500, len(accTrimmed), len(fragTrimmed)) + if checkWindow > 100: + accWindow = accTrimmed[-checkWindow:] + fragWindow = fragTrimmed[:checkWindow] + + # Find longest common substring that represents complete elements + # Look for boundaries like ], [ or }, { or ", " + for i in range(checkWindow - 50, 50, -5): + accSub = accWindow[-i:] + fragSub = fragWindow[:i] + + if accSub == fragSub: + # Check if it's a complete element boundary + if ModularJsonMerger._isCompleteElementBoundary(accSub): + if mergeId: + JsonMergeLogger.logOverlap("string (structure-boundary)", i, accSub[:200], fragSub[:200]) + return i + + return 0 + + @staticmethod + def _isCompleteJsonElement(jsonStr: str) -> bool: + """Check if string ends with a complete JSON element (balanced brackets/braces).""" + jsonStr = jsonStr.strip() + if not jsonStr: + return False + + # Check if it ends with complete structure markers + # Complete array element: ends with ] or ], or ], + # Complete object element: ends with } or }, or }, + if jsonStr[-1] in ']}': + # Check if brackets/braces are balanced + braceCount = jsonStr.count('{') - jsonStr.count('}') + bracketCount = jsonStr.count('[') - jsonStr.count(']') + return braceCount == 0 and bracketCount == 0 + + return False + + @staticmethod + def _startsWithSameElement(accSuffix: str, fragPrefix: str) -> bool: + """Check if fragment prefix starts with the same element as accumulated suffix.""" + # Normalize whitespace for comparison + accNorm = accSuffix.strip() + fragNorm = fragPrefix.strip() + + # Check if fragPrefix starts with accSuffix (or vice versa for partial matches) + if fragNorm.startswith(accNorm): + return True + + # Check if they have common prefix (for partial element completion) + minLen = min(len(accNorm), len(fragNorm)) + if minLen > 20: + # Check if first 80% of accSuffix matches start of fragPrefix + checkLen = int(minLen * 0.8) + return accNorm[:checkLen] == fragNorm[:checkLen] + + return False + + @staticmethod + def _isCompleteElementBoundary(jsonStr: str) -> bool: + """Check if string represents a complete element boundary (e.g., ], [ or }, {).""" + jsonStr = jsonStr.strip() + if not jsonStr: + return False + + # Check if it contains complete element boundaries + # Pattern: ends with ], or }, or ],\n or },\n + if jsonStr.rstrip().endswith(('],', '},', ']', '}')): + return True + + # Check if it's a complete array element or object property + if '],' in jsonStr or '},' in jsonStr: + return True + + return False + + @staticmethod + def _findLineBasedOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: + """ + Find overlap using line-based matching (for formatted JSON). + """ + accLines = accStr.rstrip().split('\n') + fragLines = fragStr.lstrip().split('\n') + + # Try to find matching lines from the end of accumulated at the start of fragment + maxLinesToCheck = min(10, len(accLines), len(fragLines)) + + for numLines in range(maxLinesToCheck, 0, -1): + # Get last N lines from accumulated (excluding empty lines) + accLastLines = [line.strip() for line in accLines[-numLines:] if line.strip()] + # Get first N lines from fragment (excluding empty lines) + fragFirstLines = [line.strip() for line in fragLines[:numLines] if line.strip()] + + # Check if they match + if len(accLastLines) > 0 and len(fragFirstLines) > 0: + # Try to find where accLastLines match fragFirstLines + for i in range(len(accLastLines)): + # Check if accLastLines[i:] matches fragFirstLines[:len(accLastLines)-i] + accSuffixLines = accLastLines[i:] + fragPrefixLines = fragFirstLines[:len(accSuffixLines)] + + if accSuffixLines == fragPrefixLines and len(accSuffixLines) > 0: + # Found overlap! Calculate character length + accSuffixText = '\n'.join(accLastLines[i:]) + fragPrefixText = '\n'.join(fragPrefixLines) + + # Find where this text appears in the original strings + accPos = accStr.rfind(accSuffixText) + fragPos = fragStr.find(fragPrefixText) + + if accPos >= 0 and fragPos == 0: + # Found valid overlap + overlapLen = len(accSuffixText) + if mergeId: + JsonMergeLogger.logOverlap("string (line-based)", overlapLen, accSuffixText[:200], fragPrefixText[:200]) + return overlapLen + + return 0 + + @staticmethod + def _findPartialOverlap(accStr: str, fragStr: str, mergeId: Optional[str] = None) -> int: + """ + Find partial overlap (incomplete element at cut point). + """ + accLines = accStr.rstrip().split('\n') + fragLines = fragStr.lstrip().split('\n') + + if accLines and fragLines: + lastAccLine = accLines[-1].strip() + firstFragLine = fragLines[0].strip() + + # Check if lastAccLine is a prefix of firstFragLine (incomplete line completed) + if lastAccLine and firstFragLine.startswith(lastAccLine): + # Also check if there are more matching lines after + overlapLen = len(lastAccLine) + # Try to extend overlap with more lines + for i in range(1, min(len(accLines), len(fragLines))): + if accLines[-1-i].strip() == fragLines[i].strip(): + overlapLen += len('\n' + fragLines[i]) + else: + break + + if overlapLen > 20: # Only if meaningful overlap + if mergeId: + JsonMergeLogger.logOverlap("string (partial line)", overlapLen, lastAccLine[:200], firstFragLine[:200]) + return overlapLen + + return 0 + + @staticmethod + def _mergeStrings(accStr: str, fragStr: str, overlapLength: int) -> str: + """ + Merge two JSON strings together, removing the overlap. + Handles whitespace at cut points properly for seamless merging. + """ + if overlapLength > 0: + # Remove overlap from fragment and append + # CRITICAL: Handle whitespace properly - if accumulated ends with whitespace + # and fragment starts with the same content, we need to preserve whitespace structure + merged = accStr + fragStr[overlapLength:] + else: + # No overlap - just concatenate (might need comma or other separator) + # CRITICAL: Preserve whitespace structure when merging + + # Get trailing whitespace from accumulated (spaces, tabs, but not newlines) + accTrailingWs = "" + i = len(accStr) - 1 + while i >= 0 and accStr[i] in [' ', '\t']: + accTrailingWs = accStr[i] + accTrailingWs + i -= 1 + + # Get leading whitespace from fragment (spaces, tabs, but not newlines) + fragLeadingWs = "" + i = 0 + while i < len(fragStr) and fragStr[i] in [' ', '\t']: + fragLeadingWs += fragStr[i] + i += 1 + + # Trim for content detection but preserve whitespace structure + accTrimmed = accStr.rstrip().rstrip(',') + fragTrimmed = fragStr.lstrip().lstrip(',') + + # Check if we need a separator + if accTrimmed and fragTrimmed: + # If accumulated ends with } or ] and fragment starts with { or [, we might need comma + if (accTrimmed[-1] in '}]' and fragTrimmed[0] in '{['): + # Add comma with appropriate whitespace + merged = accTrimmed + ',' + fragLeadingWs + fragTrimmed + else: + # Merge with preserved whitespace structure + # Use the whitespace from fragment (it knows the proper spacing) + merged = accTrimmed + accTrailingWs + fragLeadingWs + fragTrimmed + else: + # One is empty - just concatenate with preserved whitespace + merged = accStr + fragStr + + return merged + + @staticmethod + def merge(accumulated: str, newFragment: str) -> Tuple[str, bool]: + """ + Merge two JSON fragments intelligently. + + Args: + accumulated: Previously accumulated JSON string + newFragment: New fragment JSON string + + Returns: + Tuple of (merged_json_string, has_overlap): + - merged_json_string: Merged JSON string (closed if no overlap, unclosed if overlap found) + - has_overlap: True if overlap was found (iterations should continue), False if no overlap (iterations should stop) + """ + # Start logging + mergeId = JsonMergeLogger.startMerge(accumulated, newFragment) + + if not accumulated: + result = newFragment if newFragment else "{}" + JsonMergeLogger.finishMerge(mergeId, result, True) + return (result, False) # No overlap if no accumulated data + if not newFragment: + JsonMergeLogger.finishMerge(mergeId, accumulated, True) + return (accumulated, False) # No overlap if no new fragment + + try: + # Normalize both strings + accNormalized = stripCodeFences(normalizeJsonText(accumulated)).strip() + fragNormalized = stripCodeFences(normalizeJsonText(newFragment)).strip() + + JsonMergeLogger._log(f"\n Normalized Accumulated ({len(accNormalized)} chars)") + accNormLines = accNormalized.split('\n') + if len(accNormLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(accNormLines)} lines)") + for line in accNormLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(accNormLines) - 10} lines omitted) ...") + for line in accNormLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in accNormLines: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f"\n Normalized New Fragment ({len(fragNormalized)} chars)") + fragNormLines = fragNormalized.split('\n') + if len(fragNormLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(fragNormLines)} lines)") + for line in fragNormLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(fragNormLines) - 10} lines omitted) ...") + for line in fragNormLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in fragNormLines: + JsonMergeLogger._log(f" {line}") + + # Step 1: Find overlap between JSON strings + JsonMergeLogger.logStep("PHASE 1", "Finding overlap between JSON strings", None) + overlapLength = ModularJsonMerger._findStringOverlap(accNormalized, fragNormalized, mergeId) + + if overlapLength > 0: + accSuffix = accNormalized[-overlapLength:] + fragPrefix = fragNormalized[:overlapLength] + JsonMergeLogger._log(f"\n Overlap found ({overlapLength} chars):") + JsonMergeLogger._log(f" Accumulated suffix: {accSuffix}") + JsonMergeLogger._log(f" Fragment prefix: {fragPrefix}") + else: + # CRITICAL: No overlap found - this means iterations should stop + JsonMergeLogger._log(f"\n ⚠️ NO OVERLAP FOUND - This indicates iterations should stop") + JsonMergeLogger._log(f" Closing JSON and returning final result") + + # Close the accumulated JSON (it's complete as far as we can tell) + closedJson = closeJsonStructures(accNormalized) + JsonMergeLogger._log(f"\n Closed JSON ({len(closedJson)} chars):") + JsonMergeLogger._log(" " + "="*78) + for line in closedJson.split('\n'): + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(" " + "="*78) + + JsonMergeLogger.finishMerge(mergeId, closedJson, True) + # Return closed JSON with has_overlap=False to indicate iterations should stop + return (closedJson, False) + + # Step 2: Merge strings together (only if overlap was found) + JsonMergeLogger.logStep("PHASE 2", f"Merging strings (overlap: {overlapLength} chars)", None) + mergedString = ModularJsonMerger._mergeStrings(accNormalized, fragNormalized, overlapLength) + + JsonMergeLogger._log(f"\n Merged String ({len(mergedString)} chars)") + mergedLines = mergedString.split('\n') + if len(mergedLines) > 10: + JsonMergeLogger._log(f" (showing first 5 and last 5 of {len(mergedLines)} lines)") + for line in mergedLines[:5]: + JsonMergeLogger._log(f" {line}") + JsonMergeLogger._log(f" ... ({len(mergedLines) - 10} lines omitted) ...") + for line in mergedLines[-5:]: + JsonMergeLogger._log(f" {line}") + else: + for line in mergedLines: + JsonMergeLogger._log(f" {line}") + + # Step 3: Return merged string (with incomplete element at end for next iteration) + JsonMergeLogger.logStep("PHASE 3", "Returning merged string (may be unclosed)", None) + JsonMergeLogger._log(f"\n Returning merged string (preserving incomplete element at end for next iteration)") + + JsonMergeLogger.finishMerge(mergeId, mergedString, True) + # Return merged string with has_overlap=True to indicate iterations should continue + return (mergedString, True) + + except Exception as e: + logger.error(f"Error in modular merger: {e}") + JsonMergeLogger.logStep("ERROR", f"Exception occurred: {str(e)}", None, error=str(e)) + # Fallback: try to return accumulated if valid + try: + accParsed, accErr, _ = tryParseJson(accumulated) + if accErr is None: + JsonMergeLogger.finishMerge(mergeId, accumulated, False) + return (accumulated, False) # No overlap on error + except Exception: + pass + # Last resort: return empty valid JSON + fallback = json.dumps({"elements": []}, indent=2, ensure_ascii=False) + JsonMergeLogger.finishMerge(mergeId, fallback, False) + return (fallback, False) # No overlap on error diff --git a/modules/serviceCenter/services/serviceAi/subJsonResponseHandling.py b/modules/serviceCenter/services/serviceAi/subJsonResponseHandling.py new file mode 100644 index 00000000..3adb613c --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subJsonResponseHandling.py @@ -0,0 +1,3121 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +JSON Response Handling Module + +Handles merging of JSON responses from multiple AI iterations, including: +- Section merging with intelligent overlap detection +- JSON fragment detection and merging +- Deep recursive structure merging +- Overlap detection for complex nested structures +- String accumulation for iterative JSON generation +""" +import json +import logging +import re +from typing import Dict, Any, List, Optional, Tuple + +from modules.shared.jsonUtils import extractJsonString, repairBrokenJson, extractSectionsFromDocument +from modules.datamodels.datamodelAi import JsonAccumulationState + +logger = logging.getLogger(__name__) + + +class JsonResponseHandler: + """Handles JSON response merging and fragment detection for iterative AI generation.""" + + @staticmethod + def mergeSectionsIntelligently( + existingSections: List[Dict[str, Any]], + newSections: List[Dict[str, Any]], + iteration: int + ) -> List[Dict[str, Any]]: + """ + Intelligently merge sections from multiple iterations. + + This is a GENERIC merging strategy that handles broken JSON iterations. + The break can occur anywhere - in any section, at any depth. + + Merging strategies (in order of priority): + 1. Same Section ID: Merge sections with identical IDs + 2. Same Content-Type + Position: If last section is incomplete and new section continues it + 3. Same Order: Merge sections with same order value + 4. Structural Analysis: Detect continuation based on content structure + + Args: + existingSections: Sections accumulated from previous iterations + newSections: Sections extracted from current iteration + iteration: Current iteration number + + Returns: + Merged list of sections + """ + if not newSections: + return existingSections + + if not existingSections: + return newSections + + mergedSections = existingSections.copy() + + for newSection in newSections: + merged = False + + # Strategy 1: Same Section ID - merge directly + newSectionId = newSection.get("id") + if newSectionId: + for i, existingSection in enumerate(mergedSections): + if existingSection.get("id") == newSectionId: + # Merge sections with same ID + mergedSections[i] = JsonResponseHandler.mergeSectionContent( + existingSection, newSection, iteration + ) + merged = True + logger.debug(f"Iteration {iteration}: Merged section by ID '{newSectionId}'") + break + + if merged: + continue + + # Strategy 2: Same Content-Type + Position (continuation detection) + # Check if last section is incomplete and new section continues it + if mergedSections: + lastSection = mergedSections[-1] + lastContentType = lastSection.get("content_type") + newContentType = newSection.get("content_type") + + if lastContentType == newContentType: + # Same content type - check if last section is incomplete + if JsonResponseHandler.isSectionIncomplete(lastSection): + # Last section is incomplete, merge with new section + mergedSections[-1] = JsonResponseHandler.mergeSectionContent( + lastSection, newSection, iteration + ) + merged = True + logger.debug(f"Iteration {iteration}: Merged section by content-type continuation ({lastContentType})") + continue + + # Strategy 3: Same Order value + newOrder = newSection.get("order") + if newOrder is not None: + for i, existingSection in enumerate(mergedSections): + existingOrder = existingSection.get("order") + if existingOrder is not None and existingOrder == newOrder: + # Merge sections with same order + mergedSections[i] = JsonResponseHandler.mergeSectionContent( + existingSection, newSection, iteration + ) + merged = True + logger.debug(f"Iteration {iteration}: Merged section by order {newOrder}") + break + + if merged: + continue + + # Strategy 4: Structural Analysis - detect continuation + # For code_block and table: if last section matches new section type, merge them + if mergedSections: + lastSection = mergedSections[-1] + lastContentType = lastSection.get("content_type") + newContentType = newSection.get("content_type") + + # Both are code blocks - merge them + if lastContentType == "code_block" and newContentType == "code_block": + mergedSections[-1] = JsonResponseHandler.mergeSectionContent( + lastSection, newSection, iteration + ) + merged = True + logger.debug(f"Iteration {iteration}: Merged code_block sections by structural analysis") + continue + + # Both are tables - merge them (common case for broken JSON iterations) + if lastContentType == "table" and newContentType == "table": + mergedSections[-1] = JsonResponseHandler.mergeSectionContent( + lastSection, newSection, iteration + ) + merged = True + logger.debug(f"Iteration {iteration}: Merged table sections by structural analysis") + continue + + # No merge strategy matched - add as new section + if not merged: + mergedSections.append(newSection) + logger.debug(f"Iteration {iteration}: Added new section '{newSection.get('id', 'no-id')}' ({newSection.get('content_type', 'unknown')})") + + return mergedSections + + @staticmethod + def isSectionIncomplete(section: Dict[str, Any]) -> bool: + """ + Check if a section is incomplete (broken at the end). + + This detects incomplete sections based on content analysis: + - Code blocks: ends mid-line, ends with comma, ends with incomplete structure + - Text sections: ends mid-sentence, ends with incomplete structure + - Other types: check for incomplete elements + """ + contentType = section.get("content_type", "") + elements = section.get("elements", []) + + if not elements: + return False + + # Handle list of elements + if isinstance(elements, list) and len(elements) > 0: + lastElement = elements[-1] + else: + lastElement = elements + + if not isinstance(lastElement, dict): + return False + + # Check code_block for incomplete code + if contentType == "code_block": + code = lastElement.get("code", "") + if code: + # Check if code ends incompletely: + # - Ends with comma (incomplete CSV line) + # - Ends with number but no newline (incomplete line) + # - Ends mid-token (e.g., "23431,23" - incomplete number) + codeStripped = code.rstrip() + if codeStripped: + # Check for incomplete patterns + if codeStripped.endswith(',') or (',' in codeStripped and not codeStripped.endswith('\n')): + # Ends with comma or has comma but no final newline - likely incomplete + return True + # Check if last line is incomplete (doesn't end with newline and has partial content) + if not code.endswith('\n') and codeStripped: + # No final newline - might be incomplete + # More sophisticated: check if last number is complete + lastLine = codeStripped.split('\n')[-1] + if lastLine and ',' in lastLine: + # Has commas but might be incomplete + parts = lastLine.split(',') + if parts and len(parts[-1]) < 5: # Last part is very short - might be incomplete + return True + + # Check table for incomplete rows + if contentType == "table": + rows = lastElement.get("rows", []) + if rows: + # Check if last row is incomplete (ends with incomplete data) + lastRow = rows[-1] if isinstance(rows, list) else [] + if isinstance(lastRow, list) and lastRow: + # CRITICAL: Check if last row doesn't have expected number of columns (if headers exist) + # This is the PRIMARY indicator of incomplete table rows + headers = lastElement.get("headers", []) + if headers and isinstance(headers, list): + expectedCols = len(headers) + if len(lastRow) < expectedCols: + logger.debug(f"Table section incomplete: last row has {len(lastRow)} columns, expected {expectedCols}") + return True + # Also check if last row ends with incomplete data (e.g., incomplete string) + lastCell = lastRow[-1] if lastRow else "" + if isinstance(lastCell, str): + # If last cell is incomplete (ends with quote or is very short), section might be incomplete + if lastCell.endswith('"') or (len(lastCell) < 3 and lastCell): + logger.debug(f"Table section incomplete: last cell appears incomplete: '{lastCell}'") + return True + # Additional check: if last row has fewer cells than previous rows, it's likely incomplete + if len(rows) > 1: + prevRow = rows[-2] if isinstance(rows, list) and len(rows) > 1 else [] + if isinstance(prevRow, list) and len(prevRow) > len(lastRow): + logger.debug(f"Table section incomplete: last row has {len(lastRow)} cells, previous row has {len(prevRow)}") + return True + + # Check paragraph/text for incomplete sentences + if contentType in ["paragraph", "heading"]: + text = lastElement.get("text", "") + if text: + # Simple heuristic: if doesn't end with sentence-ending punctuation + textStripped = text.rstrip() + if textStripped and not textStripped[-1] in '.!?': + # Might be incomplete, but this is less reliable + # Only mark as incomplete if very short (likely cut off) + if len(textStripped) < 20: + return True + + # Check lists for incomplete items + if contentType in ["bullet_list", "numbered_list"]: + items = lastElement.get("items", []) + if items and isinstance(items, list): + # Check if last item is incomplete (very short or ends with incomplete string) + lastItem = items[-1] if items else None + if isinstance(lastItem, str) and len(lastItem) < 3: + return True + + # Check image for incomplete base64 data + if contentType == "image": + imageData = lastElement.get("base64Data", "") + if imageData: + # Base64 strings should end with padding ('=' or '==') + # If it doesn't, it might be incomplete + stripped = imageData.rstrip() + if stripped and not stripped.endswith(('=', '==')): + # Check if it's a valid base64 character sequence that was cut off + if len(stripped) > 0 and stripped[-1] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=': + return True + # If length is not a multiple of 4 (base64 requirement), it might be incomplete + if len(stripped) % 4 != 0: + return True + + # GENERIC CHECK: Recursively analyze structure for incompleteness + # This works for ANY structure: arrays, objects, nested, primitives + return JsonResponseHandler._isStructureIncomplete(lastElement) + + @staticmethod + def _isStructureIncomplete(structure: Any, max_depth: int = 10) -> bool: + """ + GENERIC recursive check for incomplete structures. + + Detects incompleteness by analyzing patterns: + - Arrays: Last item shorter than previous items, incomplete patterns + - Objects: Last object has fewer keys than pattern, incomplete values + - Strings: Very short, ends abruptly, incomplete patterns + - Nested: Recursively checks nested structures + + Works for ANY JSON structure of any depth/complexity. + """ + if max_depth <= 0: + return False + + # Arrays/Lists - check for incomplete patterns + if isinstance(structure, list): + if len(structure) == 0: + return False + + # Check if last item is incomplete compared to previous items + last_item = structure[-1] + + # If we have previous items, compare structure + if len(structure) > 1: + prev_item = structure[-2] + + # If last item is a list and previous is a list, check length + if isinstance(last_item, list) and isinstance(prev_item, list): + if len(last_item) < len(prev_item): + return True # Last row/item has fewer elements - likely incomplete + + # If last item is a dict and previous is a dict, check keys + if isinstance(last_item, dict) and isinstance(prev_item, dict): + if len(last_item) < len(prev_item): + return True # Last object has fewer keys - likely incomplete + + # Recursively check last item for incompleteness + if JsonResponseHandler._isStructureIncomplete(last_item, max_depth - 1): + return True + + # Objects/Dicts - check for incomplete values + elif isinstance(structure, dict): + for key, value in structure.items(): + # Recursively check each value + if JsonResponseHandler._isStructureIncomplete(value, max_depth - 1): + return True + + # Check for incomplete strings + if isinstance(value, str): + # Very short strings might be incomplete + if len(value) > 0 and len(value) < 3: + return True + # Strings ending with incomplete patterns (comma, quote, etc.) + stripped = value.rstrip() + if stripped and stripped.endswith((',', '"', '\\')): + return True + + # Strings - check for incomplete patterns + elif isinstance(structure, str): + # Very short strings might be incomplete + if len(structure) > 0 and len(structure) < 3: + return True + # Strings ending with incomplete patterns + stripped = structure.rstrip() + if stripped and stripped.endswith((',', '"', '\\')): + return True + + return False + + @staticmethod + def mergeSectionContent( + existingSection: Dict[str, Any], + newSection: Dict[str, Any], + iteration: int + ) -> Dict[str, Any]: + """ + Merge content from two sections. + + Handles different content types: + - code_block: Append code, handle overlaps, merge incomplete lines + - paragraph/heading: Append text + - table: Merge rows + - list: Merge items + - Other: Merge elements + """ + contentType = existingSection.get("content_type", "") + existingElements = existingSection.get("elements", []) + newElements = newSection.get("elements", []) + + if not newElements: + return existingSection + + # Handle list of elements + if isinstance(existingElements, list): + existingElem = existingElements[-1] if existingElements else {} + else: + existingElem = existingElements + + if isinstance(newElements, list): + newElem = newElements[0] if newElements else {} + else: + newElem = newElements + + if not isinstance(existingElem, dict) or not isinstance(newElem, dict): + return existingSection + + # Merge based on content type + if contentType == "code_block": + existingCode = existingElem.get("code", "") + newCode = newElem.get("code", "") + + if existingCode and newCode: + mergedCode = JsonResponseHandler.mergeCodeBlocks(existingCode, newCode, iteration) + existingElem["code"] = mergedCode + # Preserve language from existing or new + if "language" not in existingElem and "language" in newElem: + existingElem["language"] = newElem["language"] + + elif contentType in ["paragraph", "heading"]: + existingText = existingElem.get("text", "") + newText = newElem.get("text", "") + + if existingText and newText: + # Append text with space if needed + if existingText.rstrip() and not existingText.rstrip()[-1] in '.!?\n': + mergedText = existingText.rstrip() + " " + newText.lstrip() + else: + mergedText = existingText.rstrip() + "\n" + newText.lstrip() + existingElem["text"] = mergedText + + elif contentType == "table": + # Merge table rows with sophisticated overlap detection + # CRITICAL: Tables can have rows in two places: + # 1. Direct: existingElem["rows"] (legacy format) + # 2. Nested: existingElem["content"]["rows"] (current format) + existingRows = None + newRows = None + + # Check nested structure first (current format) + if "content" in existingElem and isinstance(existingElem["content"], dict): + existingRows = existingElem["content"].get("rows", []) + # Fallback to direct structure (legacy format) + if not existingRows: + existingRows = existingElem.get("rows", []) + + # Check nested structure first (current format) + if "content" in newElem and isinstance(newElem["content"], dict): + newRows = newElem["content"].get("rows", []) + # Fallback to direct structure (legacy format) + if not newRows: + newRows = newElem.get("rows", []) + + if existingRows and newRows: + # Use sophisticated overlap detection that handles multiple overlapping rows + mergedRows = JsonResponseHandler.mergeRowsWithOverlap(existingRows, newRows, iteration) + # Store in nested structure (current format) + if "content" not in existingElem: + existingElem["content"] = {} + existingElem["content"]["rows"] = mergedRows + # Also set type if missing + if "type" not in existingElem: + existingElem["type"] = "table" + logger.debug(f"Iteration {iteration}: Merged table rows - existing: {len(existingRows)}, new: {len(newRows)}, total: {len(mergedRows)}") + elif newRows: + # If existing has no rows but new does, use new rows + if "content" not in existingElem: + existingElem["content"] = {} + existingElem["content"]["rows"] = newRows + if "type" not in existingElem: + existingElem["type"] = "table" + # Preserve headers from existing (or use new if existing has none) + # Headers can be in content.headers or directly in element + existingHeaders = existingElem.get("content", {}).get("headers", []) if "content" in existingElem else existingElem.get("headers", []) + newHeaders = newElem.get("content", {}).get("headers", []) if "content" in newElem else newElem.get("headers", []) + if not existingHeaders and newHeaders: + if "content" not in existingElem: + existingElem["content"] = {} + existingElem["content"]["headers"] = newHeaders + # Preserve caption from existing (or use new if existing has none) + existingCaption = existingElem.get("content", {}).get("caption") if "content" in existingElem else existingElem.get("caption") + newCaption = newElem.get("content", {}).get("caption") if "content" in newElem else newElem.get("caption") + if not existingCaption and newCaption: + if "content" not in existingElem: + existingElem["content"] = {} + existingElem["content"]["caption"] = newCaption + + elif contentType in ["bullet_list", "numbered_list"]: + # Merge list items with sophisticated overlap detection + existingItems = existingElem.get("items", []) + newItems = newElem.get("items", []) + if existingItems and newItems: + mergedItems = JsonResponseHandler.mergeItemsWithOverlap(existingItems, newItems, iteration) + existingElem["items"] = mergedItems + elif newItems: + existingElem["items"] = newItems + + elif contentType == "image": + # Images are typically complete - if new image is provided, replace existing + # But check if existing image data is incomplete (e.g., base64 string cut off) + existingImageData = existingElem.get("base64Data", "") + newImageData = newElem.get("base64Data", "") + if existingImageData and newImageData: + # If existing image data doesn't end with valid base64 padding, it might be incomplete + # Base64 padding is '=' or '==' at the end + if not existingImageData.rstrip().endswith(('=', '==')): + # Existing image might be incomplete - merge by appending new data + # This handles cases where base64 string was cut off + existingElem["base64Data"] = existingImageData + newImageData + logger.debug(f"Iteration {iteration}: Merged incomplete image base64 data") + else: + # Existing image is complete - replace with new (or keep existing if new is empty) + if newImageData: + existingElem["base64Data"] = newImageData + elif newImageData: + existingElem["base64Data"] = newImageData + # Preserve other image metadata + if not existingElem.get("altText") and newElem.get("altText"): + existingElem["altText"] = newElem["altText"] + if not existingElem.get("caption") and newElem.get("caption"): + existingElem["caption"] = newElem["caption"] + + else: + # GENERIC FALLBACK: Use deep recursive merging for complex nested structures + # This handles any content type with arbitrary depth and complexity + merged_element = JsonResponseHandler.mergeDeepStructures( + existingElem, + newElem, + iteration, + f"section.{contentType}" + ) + existingElem = merged_element + + # Update section with merged content + mergedSection = existingSection.copy() + if isinstance(existingElements, list): + # Update the last element in the list with merged content + if existingElements: + existingElements[-1] = existingElem + mergedSection["elements"] = existingElements + else: + mergedSection["elements"] = existingElem + + # Preserve metadata from new section if missing in existing + if "order" not in mergedSection and "order" in newSection: + mergedSection["order"] = newSection["order"] + + return mergedSection + + @staticmethod + def mergeCodeBlocks(existingCode: str, newCode: str, iteration: int) -> str: + """ + Merge two code blocks intelligently, handling overlaps and incomplete lines. + """ + if not existingCode: + return newCode + if not newCode: + return existingCode + + existingLines = existingCode.rstrip().split('\n') + newLines = newCode.strip().split('\n') + + if not existingLines or not newLines: + return existingCode + "\n" + newCode + + lastExistingLine = existingLines[-1].strip() + firstNewLine = newLines[0].strip() + + # Strategy 1: Exact overlap - remove duplicate line + if lastExistingLine == firstNewLine: + newLines = newLines[1:] + logger.debug(f"Iteration {iteration}: Removed exact duplicate line in code merge") + + # Strategy 2: Incomplete line merge + # If last existing line ends with comma or is incomplete, merge with first new line + elif lastExistingLine.endswith(',') or (',' in lastExistingLine and len(lastExistingLine.split(',')[-1]) < 5): + # Last line is incomplete - merge with first new line + # Remove trailing comma from existing line + mergedLine = lastExistingLine.rstrip(',') + ',' + firstNewLine.lstrip() + existingLines[-1] = mergedLine + newLines = newLines[1:] + logger.debug(f"Iteration {iteration}: Merged incomplete line with continuation") + + # Strategy 3: Partial overlap detection + # Check if first new line starts with the end of last existing line + elif ',' in lastExistingLine and ',' in firstNewLine: + lastExistingParts = lastExistingLine.split(',') + firstNewParts = firstNewLine.split(',') + + # Check for overlap: if last part of existing matches first part of new + if lastExistingParts and firstNewParts: + lastExistingPart = lastExistingParts[-1].strip() + firstNewPart = firstNewParts[0].strip() + + # If they match, there's overlap + if lastExistingPart == firstNewPart and len(lastExistingParts) > 1: + # Remove overlapping part from new line + newLines[0] = ','.join(firstNewParts[1:]) + logger.debug(f"Iteration {iteration}: Removed partial overlap in code merge") + + # Reconstruct merged code + mergedCode = '\n'.join(existingLines) + if newLines: + if mergedCode and not mergedCode.endswith('\n'): + mergedCode += '\n' + mergedCode += '\n'.join(newLines) + + return mergedCode + + @staticmethod + def detectAndParseJsonFragment( + result: str, + allSections: List[Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + """ + GENERIC fragment detection for ANY JSON structure. + + Detects if response is a JSON fragment (continuation content) rather than full document structure. + Works for ANY JSON type: arrays, objects, primitives, nested structures of any depth/complexity. + + Fragment = Any JSON that: + 1. Does NOT have "documents" or "sections" keys (not full document structure) + 2. Can be ANY structure: array, object, nested, primitive, etc. + 3. Is continuation content that needs to be merged into existing sections + + Examples (all handled generically): + - Array: [["37643", ...], ...] (table rows, list items, any array) + - Object: {"rows": [...], "headers": [...]} (partial element) + - Primitive: "continuation text" (rare but possible) + - Nested: {"data": {"items": [...]}} (any nested structure) + + Returns fragment info dict with: + - fragment_data: The parsed fragment content (ANY type) + - target_section_id: ID of last incomplete section (generic, not type-specific) + + CRITICAL: Fully generic - no specific logic for tables, paragraphs, etc. + """ + try: + extracted = extractJsonString(result) + parsed = json.loads(extracted) + + # GENERIC fragment detection: Check if it's NOT a full document structure + is_full_document = False + if isinstance(parsed, dict): + # Full document structure has "documents" or "sections" keys + if "documents" in parsed or "sections" in parsed: + is_full_document = True + + # If it's a full document structure, it's not a fragment + if is_full_document: + return None + + # Otherwise, it's a fragment (can be ANY structure: array, object, primitive, nested) + # Find target: last incomplete section (generic, regardless of content type) + target_section_id = JsonResponseHandler.findLastIncompleteSectionId(allSections) + + logger.info(f"Detected GENERIC JSON fragment (type: {type(parsed).__name__}), target: {target_section_id}") + + return { + "fragment_data": parsed, # Can be ANY JSON structure + "target_section_id": target_section_id + } + + except Exception as e: + logger.error(f"Error detecting JSON fragment: {e}") + logger.debug(f"Fragment detection failed for result: {result[:500]}...") + + return None + + @staticmethod + def findLastIncompleteSectionId( + allSections: List[Dict[str, Any]] + ) -> Optional[str]: + """ + GENERIC: Find the last incomplete section (regardless of content type). + + This is fully generic - works for ANY content type, ANY structure. + Returns the ID of the last section that is incomplete, or None if all are complete. + """ + # Find the last incomplete section (generic, not type-specific) + for section in reversed(allSections): + if JsonResponseHandler.isSectionIncomplete(section): + return section.get("id") + # If no incomplete section found, return last section as fallback + if allSections: + return allSections[-1].get("id") + return None + + @staticmethod + def mergeFragmentIntoSection( + fragment: Dict[str, Any], + allSections: List[Dict[str, Any]], + iteration: int + ) -> Optional[List[Dict[str, Any]]]: + """ + GENERIC fragment merging for ANY JSON structure. + + Merges a JSON fragment (ANY structure: array, object, nested, primitive) into the last incomplete section. + Uses ONLY deep recursive merging - no specific logic for content types. + + Handles ALL cases: + 1. Fragments with overlap (detected and merged intelligently) + 2. Fragments without overlap (continuation after cut-off, appended) + 3. Any JSON structure (arrays, objects, nested, primitives) + 4. Accumulative merging (uses merged data from past iterations) + + CRITICAL: Fully generic - works for ANY JSON structure, ANY content type. + NO FALLBACKS: Returns None if merge fails (no target section found). + """ + fragment_data = fragment.get("fragment_data") + target_section_id = fragment.get("target_section_id") + + if fragment_data is None: + logger.error(f"Iteration {iteration}: ❌ Fragment has no fragment_data - merge FAILED") + return None + + # Find the target section (last incomplete section, generic) + target_section = None + target_index = -1 + + if target_section_id: + for i, section in enumerate(allSections): + if section.get("id") == target_section_id: + target_section = section + target_index = i + break + + # NO FALLBACKS: If target not found by ID, try to find incomplete section + if not target_section: + for i, section in enumerate(reversed(allSections)): + if JsonResponseHandler.isSectionIncomplete(section): + target_section = section + target_index = len(allSections) - 1 - i + break + + # NO FALLBACKS: If no target found, merge FAILS + if not target_section: + logger.error(f"Iteration {iteration}: ❌ MERGE FAILED - No target section found for fragment!") + logger.error(f"Iteration {iteration}: Available sections: {[s.get('id') + ' (' + s.get('content_type', 'unknown') + ')' for s in allSections]}") + return None + + # Get the last element from target section (where fragment will be merged) + merged_section = target_section.copy() + elements = merged_section.get("elements", []) + + if not isinstance(elements, list): + elements = [elements] if elements else [] + + if not elements: + elements = [{}] + + last_element = elements[-1] if elements else {} + if not isinstance(last_element, dict): + last_element = {} + elements.append(last_element) + + # CRITICAL: GENERIC fragment merging for ALL structure types + # Automatically detects the structure type and merges accordingly + # Works for: tables, lists, code blocks, paragraphs, images, and any nested structures + merged_element = JsonResponseHandler._mergeFragmentIntoElement( + last_element, + fragment_data, + target_section, + iteration, + f"section.{target_section_id}.fragment" + ) + + # Update elements with merged content + elements[-1] = merged_element + merged_section["elements"] = elements + + # Update allSections (this ensures accumulative merging - merged data is used for next iteration) + merged_sections = allSections.copy() + merged_sections[target_index] = merged_section + + logger.info(f"Iteration {iteration}: ✅ Merged GENERIC fragment (type: {type(fragment_data).__name__}) into section '{target_section_id}'") + + # Log merged JSON for debugging + try: + from modules.shared.debugLogger import writeDebugFile + merged_json_str = json.dumps(merged_sections, indent=2, ensure_ascii=False) + writeDebugFile(merged_json_str, f"merged_json_iteration_{iteration}.json") + except Exception as e: + logger.debug(f"Iteration {iteration}: Failed to write merged JSON debug file: {e}") + + return merged_sections + + @staticmethod + def completeIncompleteStructures(allSections: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Complete any incomplete structures in sections by ensuring proper JSON structure. + + This ensures JSON is properly closed even if merge failed or iterations stopped early. + Works generically for ANY structure type - recursively processes all nested structures. + + Returns sections with completed structures. + """ + completed_sections = [] + for section in allSections: + completed_section = JsonResponseHandler._completeStructure(section) + completed_sections.append(completed_section) + return completed_sections + + @staticmethod + def _completeStructure(structure: Any) -> Any: + """ + Recursively complete incomplete structures by ensuring arrays/objects are properly structured. + Works generically for ANY JSON structure - no specific logic for content types. + """ + if isinstance(structure, dict): + completed = {} + for key, value in structure.items(): + completed[key] = JsonResponseHandler._completeStructure(value) + return completed + elif isinstance(structure, list): + completed = [] + for item in structure: + completed.append(JsonResponseHandler._completeStructure(item)) + return completed + else: + # Primitive value - return as is + return structure + + @staticmethod + def getContentTypeForFragment(fragment_type: str) -> str: + """Map fragment type to content type.""" + mapping = { + "table_rows": "table", + "table_element": "table", + "code_lines": "code_block", + "code_element": "code_block", + "list_items": "bullet_list" + } + return mapping.get(fragment_type, "paragraph") + + @staticmethod + def deepCompare(obj1: Any, obj2: Any, max_depth: int = 10) -> bool: + """ + Deep recursive comparison of two JSON-serializable objects. + Handles nested structures of any depth and complexity. + + Args: + obj1: First object to compare + obj2: Second object to compare + max_depth: Maximum recursion depth to prevent infinite loops + + Returns: + True if objects are deeply equal, False otherwise + """ + if max_depth <= 0: + return False + + # Type check + if type(obj1) != type(obj2): + return False + + # Primitive types + if isinstance(obj1, (str, int, float, bool, type(None))): + return obj1 == obj2 + + # Lists/arrays - compare element by element + if isinstance(obj1, list): + if len(obj1) != len(obj2): + return False + return all(JsonResponseHandler.deepCompare(item1, item2, max_depth - 1) + for item1, item2 in zip(obj1, obj2)) + + # Dicts/objects - compare key by key + if isinstance(obj1, dict): + if set(obj1.keys()) != set(obj2.keys()): + return False + return all(JsonResponseHandler.deepCompare(obj1[key], obj2[key], max_depth - 1) + for key in obj1.keys()) + + # Fallback for other types + return obj1 == obj2 + + @staticmethod + def findLongestCommonSuffix( + existing_list: List[Any], + new_list: List[Any], + min_overlap: int = 1 + ) -> int: + """ + Find the longest common suffix of existing_list that matches a prefix of new_list. + + This handles cases where multiple elements overlap: + - existing: [A, B, C, D] + - new: [C, D, E, F] + - overlap: [C, D] (length 2) + + Returns the length of the overlap (0 if no overlap found). + """ + if not existing_list or not new_list: + return 0 + + max_overlap = min(len(existing_list), len(new_list)) + + # Try all possible overlap lengths (from longest to shortest) + for overlap_len in range(max_overlap, min_overlap - 1, -1): + existing_suffix = existing_list[-overlap_len:] + new_prefix = new_list[:overlap_len] + + # Deep compare suffix and prefix + if all(JsonResponseHandler.deepCompare(existing_suffix[i], new_prefix[i]) + for i in range(overlap_len)): + return overlap_len + + return 0 + + @staticmethod + def findPartialOverlap( + existing_item: Any, + new_item: Any + ) -> Tuple[bool, Optional[Any]]: + """ + Detect if new_item completes an incomplete existing_item. + + Handles cases like: + - existing: ["37643", "37649", "37657", "37663", "37691", "37693", "37699", "37717", "37747", "376"] + - new: ["37643", "37649", ...] + + Returns (is_partial_overlap, merged_item) if partial overlap detected, else (False, None). + """ + # Check if both are lists + if isinstance(existing_item, list) and isinstance(new_item, list): + if not existing_item or not new_item: + return False, None + + # Check if last element of existing is incomplete and matches first of new + last_existing = existing_item[-1] + first_new = new_item[0] + + # If last existing is a string and first new is a string + if isinstance(last_existing, str) and isinstance(first_new, str): + # Check if last existing is incomplete (very short, ends with number, etc.) + if len(last_existing) < 10 and first_new.startswith(last_existing): + # Partial overlap - merge them + merged_last = last_existing + first_new[len(last_existing):] + merged_item = existing_item[:-1] + [merged_last] + new_item[1:] + return True, merged_item + + # Check if last existing is incomplete list and first new completes it + if isinstance(last_existing, list) and isinstance(first_new, list): + if len(last_existing) < len(first_new): + # Check if last existing is prefix of first new + if first_new[:len(last_existing)] == last_existing: + # Merge: replace incomplete last with complete first + merged_item = existing_item[:-1] + [first_new] + new_item[1:] + return True, merged_item + + # Check if existing is incomplete string and new completes it + if isinstance(existing_item, str) and isinstance(new_item, str): + if len(existing_item) < 50 and new_item.startswith(existing_item): + # Partial overlap + merged = existing_item + new_item[len(existing_item):] + return True, merged + + return False, None + + @staticmethod + def mergeRowsWithOverlap( + existing_rows: List[List[str]], + new_rows: List[List[str]], + iteration: int + ) -> List[List[str]]: + """ + Merge table rows with sophisticated overlap detection. + Handles multiple overlapping rows and partial overlaps. + """ + if not new_rows: + return existing_rows + if not existing_rows: + return new_rows + + # Strategy 1: Find longest common suffix/prefix overlap + overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing_rows, new_rows, min_overlap=1) + if overlap_len > 0: + logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping table rows, removing duplicates") + return existing_rows + new_rows[overlap_len:] + + # Strategy 2: Check for partial overlap in last row + if len(existing_rows) > 0 and len(new_rows) > 0: + last_existing = existing_rows[-1] + first_new = new_rows[0] + + is_partial, merged_row = JsonResponseHandler.findPartialOverlap(last_existing, first_new) + if is_partial: + logger.debug(f"Iteration {iteration}: Found partial overlap in table rows, merging") + return existing_rows[:-1] + [merged_row] + new_rows[1:] + + # Strategy 3: Simple first/last comparison (fallback) + if isinstance(existing_rows[-1], list) and isinstance(new_rows[0], list): + if list(existing_rows[-1]) == list(new_rows[0]): + logger.debug(f"Iteration {iteration}: Removed duplicate table row (exact match)") + return existing_rows + new_rows[1:] + + # No overlap detected - append all new rows + return existing_rows + new_rows + + @staticmethod + def mergeItemsWithOverlap( + existing_items: List[str], + new_items: List[str], + iteration: int + ) -> List[str]: + """ + Merge list items with sophisticated overlap detection. + Handles multiple overlapping items and partial overlaps. + """ + if not new_items: + return existing_items + if not existing_items: + return new_items + + # Strategy 1: Find longest common suffix/prefix overlap + overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing_items, new_items, min_overlap=1) + if overlap_len > 0: + logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping list items, removing duplicates") + return existing_items + new_items[overlap_len:] + + # Strategy 2: Check for partial overlap in last item + if len(existing_items) > 0 and len(new_items) > 0: + is_partial, merged_item = JsonResponseHandler.findPartialOverlap(existing_items[-1], new_items[0]) + if is_partial: + logger.debug(f"Iteration {iteration}: Found partial overlap in list items, merging") + return existing_items[:-1] + [merged_item] + new_items[1:] + + # Strategy 3: Simple first/last comparison (fallback) + if existing_items[-1] == new_items[0]: + logger.debug(f"Iteration {iteration}: Removed duplicate list item (exact match)") + return existing_items + new_items[1:] + + # No overlap detected - append all new items + return existing_items + new_items + + @staticmethod + def mergeDeepStructures( + existing: Any, + new: Any, + iteration: int, + path: str = "root" + ) -> Any: + """ + FULLY GENERIC recursive merge for ANY JSON structure of arbitrary depth/complexity. + + Handles ALL cases generically: + 1. Arrays/Lists: Overlap detection (suffix/prefix), partial overlap, no overlap (continuation) + 2. Objects/Dicts: Key-by-key merge with overlap detection for nested structures + 3. Primitives: Equality check, replacement if different + 4. Nested structures: Recursively handles any depth/complexity + + Overlap detection strategies (all generic): + - Array overlap: Finds longest common suffix/prefix, handles partial overlaps + - Object overlap: Detected recursively through key matching and deep comparison + - No overlap: Appends/merges continuation content after cut-off point + + CRITICAL: Fully generic - no specific logic for content types. + Works for ANY JSON structure: arrays, objects, nested, primitives, any combination. + """ + # Type check + if type(existing) != type(new): + # Types don't match - return new (replacement) + logger.debug(f"Iteration {iteration}: Types don't match at {path} ({type(existing).__name__} vs {type(new).__name__}), replacing") + return new + + # Lists/arrays - GENERIC merge with overlap detection + if isinstance(existing, list) and isinstance(new, list): + if not new: + return existing + if not existing: + return new + + # Strategy 1: Find longest common suffix/prefix overlap (handles multiple overlapping elements) + overlap_len = JsonResponseHandler.findLongestCommonSuffix(existing, new, min_overlap=1) + if overlap_len > 0: + logger.debug(f"Iteration {iteration}: Found {overlap_len} overlapping elements at {path}, removing duplicates") + return existing + new[overlap_len:] + + # Strategy 2: Check for partial overlap in last element (incomplete element completion) + if len(existing) > 0 and len(new) > 0: + is_partial, merged_item = JsonResponseHandler.findPartialOverlap(existing[-1], new[0]) + if is_partial: + logger.debug(f"Iteration {iteration}: Found partial overlap at {path}, merging incomplete element") + return existing[:-1] + [merged_item] + new[1:] + + # Strategy 3: No overlap detected - continuation after cut-off point + # This handles the case where new data starts exactly after the cut-off + logger.debug(f"Iteration {iteration}: No overlap at {path}, appending continuation content ({len(new)} items)") + return existing + new + + # Dicts/objects - GENERIC merge with recursive overlap detection + if isinstance(existing, dict) and isinstance(new, dict): + merged = existing.copy() + + # Check for object-level overlap: if new object is subset/superset of existing + # This handles cases where same object structure appears in both + existing_keys = set(existing.keys()) + new_keys = set(new.keys()) + + # If new is subset of existing and values match, it's overlap (skip) + if new_keys.issubset(existing_keys): + all_match = True + for key in new_keys: + if not JsonResponseHandler.deepCompare(existing[key], new[key]): + all_match = False + break + if all_match: + logger.debug(f"Iteration {iteration}: Object at {path} is subset overlap, skipping") + return existing + + # Merge key-by-key with recursive overlap detection + for key, new_value in new.items(): + if key in merged: + # Key exists - merge recursively (handles nested overlap detection) + merged[key] = JsonResponseHandler.mergeDeepStructures( + merged[key], + new_value, + iteration, + f"{path}.{key}" + ) + else: + # New key - add it (continuation content) + merged[key] = new_value + logger.debug(f"Iteration {iteration}: Added new key '{key}' at {path} (continuation)") + + return merged + + # Primitives - equality check + if existing == new: + return existing + # Different primitive values - return new (continuation/replacement) + logger.debug(f"Iteration {iteration}: Primitive at {path} differs, using new value") + return new + + @staticmethod + def _mergeFragmentIntoElement( + last_element: Dict[str, Any], + fragment_data: Any, + target_section: Dict[str, Any], + iteration: int, + path: str + ) -> Dict[str, Any]: + """ + GENERIC fragment merging for ALL structure types. + + Automatically detects the structure type and merges fragments accordingly. + Works for: tables, lists, code blocks, paragraphs, images, and any nested structures. + + Strategy: + 1. Analyze last_element structure to determine content location (content.rows, content.items, etc.) + 2. Detect fragment type (array, object, primitive) + 3. Merge fragment into appropriate location using mergeDeepStructures + + Args: + last_element: The existing element to merge into + fragment_data: The fragment data to merge (can be any JSON structure) + target_section: The target section (for content_type detection) + iteration: Current iteration number + path: Path for logging + + Returns: + Merged element + """ + contentType = target_section.get("content_type", "") + elementType = last_element.get("type", "") + + # Determine the content structure path based on element type and content type + # This handles both nested (content.rows) and flat (rows) structures + contentPath = None + fragmentIsArray = isinstance(fragment_data, list) and len(fragment_data) > 0 + + # Detect structure type and determine merge path + if contentType == "table" or elementType == "table": + # Tables: merge into content.rows or rows + if "content" in last_element and isinstance(last_element["content"], dict): + contentPath = "content.rows" + else: + contentPath = "rows" + elif contentType in ["bullet_list", "numbered_list", "list"] or elementType in ["bullet_list", "numbered_list", "list"]: + # Lists: merge into content.items or items + if "content" in last_element and isinstance(last_element["content"], dict): + contentPath = "content.items" + else: + contentPath = "items" + elif contentType == "code_block" or elementType == "code_block": + # Code blocks: merge into content.code or code + if "content" in last_element and isinstance(last_element["content"], dict): + contentPath = "content.code" + else: + contentPath = "code" + elif contentType in ["paragraph", "heading"] or elementType in ["paragraph", "heading"]: + # Text: merge into content.text or text + if "content" in last_element and isinstance(last_element["content"], dict): + contentPath = "content.text" + else: + contentPath = "text" + elif contentType == "image" or elementType == "image": + # Images: merge into base64Data + contentPath = "base64Data" + + # If we have a specific content path, merge into that location + if contentPath: + # Split path (e.g., "content.rows" -> ["content", "rows"]) + pathParts = contentPath.split(".") + + # Ensure nested structure exists + current = last_element + for i, part in enumerate(pathParts[:-1]): + if part not in current: + current[part] = {} + elif not isinstance(current[part], dict): + current[part] = {} + current = current[part] + + # Get existing content at target path + targetKey = pathParts[-1] + existingContent = current.get(targetKey, []) + + # Merge fragment into existing content + # CRITICAL: Handle both array fragments and object fragments generically + if fragmentIsArray: + # Fragment is an array - merge arrays + if isinstance(existingContent, list): + # Check if fragment is array of arrays (e.g., table rows) or array of primitives + if len(fragment_data) > 0 and isinstance(fragment_data[0], list): + # Array of arrays - use rows merge for tables, generic merge for others + if contentPath.endswith(".rows"): + mergedContent = JsonResponseHandler.mergeRowsWithOverlap(existingContent, fragment_data, iteration) + else: + # Generic array-of-arrays merge + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent, + fragment_data, + iteration, + f"{path}.{targetKey}" + ) + else: + # Array of primitives - use items merge for lists, generic merge for others + if contentPath.endswith(".items"): + mergedContent = JsonResponseHandler.mergeItemsWithOverlap(existingContent, fragment_data, iteration) + else: + # Generic array merge using mergeDeepStructures + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent, + fragment_data, + iteration, + f"{path}.{targetKey}" + ) + else: + # Existing content is not a list - replace with fragment + mergedContent = fragment_data + elif isinstance(fragment_data, dict): + # Fragment is an object - check if it contains nested content (e.g., {"content": {"rows": [...]}}) + # If fragment has same structure as target, merge nested content + if "content" in fragment_data and isinstance(fragment_data["content"], dict): + fragmentNested = fragment_data["content"] + # Check if fragment has the same key as our target (e.g., fragment.content.rows) + if targetKey in fragmentNested: + # Fragment has nested content matching our target - merge that content + fragmentNestedContent = fragmentNested[targetKey] + if isinstance(existingContent, list) and isinstance(fragmentNestedContent, list): + # Both are lists - merge them + if contentPath.endswith(".rows"): + mergedContent = JsonResponseHandler.mergeRowsWithOverlap(existingContent, fragmentNestedContent, iteration) + elif contentPath.endswith(".items"): + mergedContent = JsonResponseHandler.mergeItemsWithOverlap(existingContent, fragmentNestedContent, iteration) + else: + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent, + fragmentNestedContent, + iteration, + f"{path}.{targetKey}" + ) + else: + # Use deep merge for nested content + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent if existingContent else {}, + fragmentNestedContent, + iteration, + f"{path}.{targetKey}" + ) + else: + # Fragment has different structure - merge entire fragment object + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent if existingContent else {}, + fragment_data, + iteration, + f"{path}.{targetKey}" + ) + else: + # Fragment is a simple object - use deep merge + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent if existingContent else {}, + fragment_data, + iteration, + f"{path}.{targetKey}" + ) + else: + # Fragment is a primitive or unknown type - use deep merge + mergedContent = JsonResponseHandler.mergeDeepStructures( + existingContent if existingContent else {}, + fragment_data, + iteration, + f"{path}.{targetKey}" + ) + + # Update the merged content + current[targetKey] = mergedContent + + # Ensure type is set + if elementType and "type" not in last_element: + last_element["type"] = elementType + elif contentType and "type" not in last_element: + last_element["type"] = contentType + + logger.info(f"Iteration {iteration}: ✅ Merged fragment into {contentPath} for section '{target_section.get('id')}'") + return last_element + + # No specific content path - use generic deep merge + # This handles any structure type generically + merged_element = JsonResponseHandler.mergeDeepStructures( + last_element, + fragment_data, + iteration, + path + ) + + logger.info(f"Iteration {iteration}: ✅ Merged GENERIC fragment (type: {type(fragment_data).__name__}) into section '{target_section.get('id')}'") + return merged_element + + @staticmethod + def cleanEncodingIssues(jsonString: str) -> str: + """ + GENERIC function to remove problematic encoding parts from JSON string. + + Works for ANY JSON structure - removes problematic characters/bytes. + + Args: + jsonString: JSON string that may have encoding issues + + Returns: + Cleaned JSON string + """ + try: + # Try to decode/encode to detect issues + jsonString.encode('utf-8').decode('utf-8') + return jsonString + except UnicodeError: + # Remove problematic parts + cleaned = jsonString.encode('utf-8', errors='ignore').decode('utf-8', errors='ignore') + logger.warning("Removed encoding issues from JSON string") + return cleaned + + @staticmethod + def mergeJsonStringsWithOverlap( + accumulated: str, + newFragment: str + ) -> Tuple[str, bool]: + """ + Merge JSON fragments intelligently using modular parser. + + Uses the new ModularJsonMerger for clean, robust merging. + Falls back to legacy code only if new merger fails completely. + + Args: + accumulated: Previously accumulated JSON string (may be incomplete/fragmented) + newFragment: New fragment string to append (may be incomplete/fragmented) + + Returns: + Tuple of (merged_json_string, has_overlap): + - merged_json_string: Combined JSON string with fragments properly merged + - has_overlap: True if overlap was found (iterations should continue), False if no overlap (iterations should stop) + """ + if not accumulated: + result = newFragment if newFragment else "{}" + return (result, False) # No overlap if no accumulated data + if not newFragment: + return (accumulated, False) # No overlap if no new fragment + + # Use new modular merger + try: + from .subJsonMerger import ModularJsonMerger + result, hasOverlap = ModularJsonMerger.merge(accumulated, newFragment) + # IMPORTANT: ModularJsonMerger returns unclosed JSON if overlap found (with incomplete element at end) + # If no overlap, returns closed JSON (iterations should stop) + if result and result.strip() and result.strip() != "{}": + # Return result with overlap flag + return (result, hasOverlap) + except Exception as e: + logger.debug(f"Modular merger failed, using fallback: {e}") + + # Fallback to legacy merger (simplified) + + accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() + newFragmentExtracted = stripCodeFences(normalizeJsonText(newFragment)).strip() + + # Try simple string merge with repair + try: + # Close structures + accClosed = closeJsonStructures(accumulatedExtracted) if accumulatedExtracted else "{}" + fragClosed = closeJsonStructures(newFragmentExtracted) if newFragmentExtracted else "{}" + + # Try to parse both + accParsed, accErr, _ = tryParseJson(accClosed) + fragParsed, fragErr, _ = tryParseJson(fragClosed) + + # If both parse, merge structurally + if accErr is None and fragErr is None: + merged = JsonResponseHandler._mergeParsedJson(accParsed, fragParsed) + if merged: + result = json.dumps(merged, indent=2, ensure_ascii=False) + return (result, False) # No overlap in fallback - close and stop + + # If only accumulated parses, return it + if accErr is None and accParsed: + result = json.dumps(accParsed, indent=2, ensure_ascii=False) + return (result, False) # No overlap - close and stop + except Exception: + pass + + # Last resort: return accumulated (at least we have that) - close it + if accumulatedExtracted: + try: + closed = closeJsonStructures(accumulatedExtracted) + return (closed, False) # No overlap - close and stop + except Exception: + return (accumulatedExtracted, False) # No overlap - return as-is + + result = accumulated if accumulated else "{}" + return (result, False) # No overlap - return as-is + + @staticmethod + def _mergeParsedJson(accParsed: Any, fragParsed: Any) -> Optional[Dict[str, Any]]: + """Simple merge of two parsed JSON objects.""" + if isinstance(accParsed, dict) and isinstance(fragParsed, dict): + # Merge dicts + merged = accParsed.copy() + + # Merge elements if both have them + if "elements" in accParsed and "elements" in fragParsed: + accElements = accParsed.get("elements", []) + fragElements = fragParsed.get("elements", []) + # Simple merge - append new elements + merged["elements"] = accElements + fragElements + elif "elements" in fragParsed: + merged["elements"] = fragParsed["elements"] + + # Merge other keys + for key, value in fragParsed.items(): + if key != "elements": + if key in merged and isinstance(merged[key], list) and isinstance(value, list): + merged[key] = merged[key] + value + else: + merged[key] = value + + return merged + + return None + + @staticmethod + def _normalizeToElementsStructure( + jsonString: str, + originalString: str + ) -> Optional[Dict[str, Any]]: + """ + Normalize any JSON structure (Dict, List, None, or parse error) to {"elements": [...]} format. + + Handles: + - Dict with "elements" → return as-is + - Dict without "elements" but with "type" → wrap in elements array + - List → wrap in elements structure + - Parse error → try repairBrokenJson + - None → return None + + Args: + jsonString: Extracted JSON string + originalString: Original string (for context) + + Returns: + Normalized Dict with "elements" array, or None if normalization fails + """ + if not jsonString: + return None + + + # Try to parse directly first + try: + parsed = json.loads(jsonString) + parseErr = None + except Exception as e: + parseErr = e + parsed = None + + # If parsing failed, try closing structures first (for incomplete fragments) + if parseErr is not None: + try: + closed = closeJsonStructures(jsonString) + parsed = json.loads(closed) + parseErr = None + except Exception: + pass + + # If still failed, try repairBrokenJson ONLY if it looks like document structure + # For other structures (like section_content), use fragment detection instead + if parseErr is not None: + # Check if this looks like a document structure (has "documents" or "sections") + isDocumentStructure = '"documents"' in jsonString or '"sections"' in jsonString + + if isDocumentStructure: + # Use repairBrokenJson for document structures + repaired = repairBrokenJson(jsonString) + if repaired: + parsed = repaired + parseErr = None + else: + # Still can't parse - try to detect fragment structure + return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) + else: + # For non-document structures, skip repairBrokenJson and go straight to fragment detection + # repairBrokenJson tries to extract "sections" which doesn't work for other structures + return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) + + # Normalize based on type + if parsed is None: + return None + elif isinstance(parsed, dict): + # Already a dict + if "elements" in parsed: + return parsed + elif "type" in parsed: + # Single element - wrap in elements array + return {"elements": [parsed]} + else: + # Unknown dict structure - try to extract elements + return JsonResponseHandler._extractElementsFromDict(parsed) + elif isinstance(parsed, list): + # List - check if it's a list of elements or a fragment + if parsed and isinstance(parsed[0], dict) and "type" in parsed[0]: + # List of elements + return {"elements": parsed} + else: + # Fragment list (e.g., array of rows) - detect structure + return JsonResponseHandler._detectAndNormalizeFragment(jsonString, originalString) + else: + # Primitive type - can't normalize + return None + + @staticmethod + def _detectAndNormalizeFragment( + jsonString: str, + originalString: str + ) -> Optional[Dict[str, Any]]: + """ + Detect fragment structure and normalize it. + + Fragments can be: + - Array of arrays (table rows): `[["row1"], ["row2"]]` or `["1947", "16883"], ["1948", "16889"]` + - Array of strings (list items): `["item1", "item2"]` + - Incomplete structure: `["item1", "item2", ` (ends with comma) + - Partial object: `{"type": "table", "content": {"rows": [["1947"...` (cut mid-string) + + Returns normalized structure or None if detection fails. + """ + jsonStripped = jsonString.strip() + + # Strategy 1: Check if it's an array fragment + if jsonStripped.startswith('['): + # Try to parse as array + + # Close incomplete structures + closed = closeJsonStructures(jsonStripped) + parsed, parseErr, _ = tryParseJson(closed) + + if parseErr is None and isinstance(parsed, list): + # Check structure: array of arrays (table rows) or array of strings (list items) + if parsed and isinstance(parsed[0], list): + # Array of arrays - likely table rows fragment + return { + "elements": [{ + "type": "table", + "content": { + "rows": parsed + } + }] + } + elif parsed and isinstance(parsed[0], str): + # Array of strings - likely list items fragment + return { + "elements": [{ + "type": "bullet_list", + "content": { + "items": parsed + } + }] + } + elif parseErr is not None: + # Can't parse - try regex extraction for table rows + rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) + if rows: + return { + "elements": [{ + "type": "table", + "content": { + "rows": rows + } + }] + } + + # Strategy 2: Check if it's a partial object (cut mid-structure) + # Look for patterns like: {"elements": [...] or {"type": "table"... + if jsonStripped.startswith('{'): + + # Try to close and parse + closed = closeJsonStructures(jsonStripped) + parsed, parseErr, _ = tryParseJson(closed) + + if parseErr is None and isinstance(parsed, dict): + # Successfully parsed - normalize it + return JsonResponseHandler._normalizeToElementsStructure(closed, originalString) + elif parseErr is not None: + # Can't parse - try to extract table rows from the raw string + # This handles cases like: {"elements": [{"type": "table", "content": {"rows": [["1947"... + rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) + if rows: + return { + "elements": [{ + "type": "table", + "content": { + "rows": rows + } + }] + } + + # Try to extract any array patterns that might be table rows + # Look for patterns like: ["1947", "10000"], ["1948", "10100"] + import re + # Pattern: ["value1", "value2"], ["value3", "value4"] + rowPattern = r'\["([^"]*)",\s*"([^"]*)"\]' + matches = re.findall(rowPattern, jsonStripped) + if matches and len(matches) >= 2: + # Found multiple row patterns - likely table rows + rows = [[match[0], match[1]] for match in matches] + return { + "elements": [{ + "type": "table", + "content": { + "rows": rows + } + }] + } + + # Strategy 3: Try to extract rows from any text (even if not starting with [ or {) + rows = JsonResponseHandler._extractRowsFromFragment(jsonStripped) + if rows: + return { + "elements": [{ + "type": "table", + "content": { + "rows": rows + } + }] + } + + return None + + @staticmethod + def _extractElementsFromDict(d: Dict[str, Any]) -> Dict[str, Any]: + """ + Try to extract elements from unknown dict structure. + Returns normalized structure or empty elements array. + """ + # Check common patterns + if "sections" in d: + # Document structure with sections + sections = d.get("sections", []) + elements = [] + for section in sections: + if isinstance(section, dict) and "elements" in section: + elements.extend(section.get("elements", [])) + return {"elements": elements} + + # Unknown structure - return empty + return {"elements": []} + + @staticmethod + def _mergeJsonStructuresGeneric( + accumulatedObj: Dict[str, Any], + newFragmentObj: Dict[str, Any], + accumulatedRaw: str, + newFragmentRaw: str, + overlapElements: Optional[List[Dict[str, Any]]] = None + ) -> Optional[Dict[str, Any]]: + """ + GENERIC merge of two JSON structures, handling overlaps and missing parts. + + Strategy: + 1. Extract elements from both structures (both are normalized to {"elements": [...]}) + 2. Use overlap elements if provided to identify merge point + 3. Detect if both have same structure (same content type) + 4. Group elements by type + 5. Merge elements of same type using content-type-specific logic with overlap detection + 6. Handle overlaps and missing parts intelligently + + Args: + accumulatedObj: Normalized accumulated JSON object (guaranteed to have "elements") + newFragmentObj: Normalized new fragment JSON object (guaranteed to have "elements") + accumulatedRaw: Raw accumulated string (for fragment detection) + newFragmentRaw: Raw new fragment string (for fragment detection) + overlapElements: Optional list of overlap elements from continuation response + + Returns: + Merged JSON object or None if merging fails + """ + try: + # Step 1: Extract elements (both are normalized, so this should always work) + accumulatedElements = accumulatedObj.get("elements", []) if isinstance(accumulatedObj, dict) else [] + newFragmentElements = newFragmentObj.get("elements", []) if isinstance(newFragmentObj, dict) else [] + + if not accumulatedElements and not newFragmentElements: + # No elements found - try to extract from raw strings + # Try to extract any valid JSON structure from raw strings + + # Try accumulated first + if accumulatedRaw: + try: + closedAccumulated = closeJsonStructures(accumulatedRaw) + parsed, parseErr, _ = tryParseJson(closedAccumulated) + if parseErr is None and parsed: + normalized = JsonResponseHandler._normalizeToElementsStructure(closedAccumulated, accumulatedRaw) + if normalized: + return normalized + except Exception: + pass + + # Try new fragment + if newFragmentRaw: + try: + closedFragment = closeJsonStructures(newFragmentRaw) + parsed, parseErr, _ = tryParseJson(closedFragment) + if parseErr is None and parsed: + normalized = JsonResponseHandler._normalizeToElementsStructure(closedFragment, newFragmentRaw) + if normalized: + return normalized + except Exception: + pass + + # If still nothing, return empty structure (never None) + return {"elements": []} + + # Step 2: Use overlap elements to identify merge point + # If overlap elements are provided, use them to find where to merge + if overlapElements and isinstance(overlapElements, list) and len(overlapElements) > 0: + # Find overlap in accumulated elements + overlapStartIndex = JsonResponseHandler._findOverlapStartIndex(accumulatedElements, overlapElements) + if overlapStartIndex >= 0: + # Remove overlapping elements from accumulated (they'll be replaced by continuation) + accumulatedElements = accumulatedElements[:overlapStartIndex] + logger.debug(f"Found overlap at index {overlapStartIndex}, removed {len(accumulatedElements) - overlapStartIndex} overlapping elements") + + # Step 3: Detect if newFragment is a continuation fragment + # Check if newFragment starts with array elements (fragment, not full JSON) + isFragment = JsonResponseHandler._isFragment(newFragmentRaw, newFragmentElements) + + # Step 4: Group elements by type for intelligent merging + accumulatedByType = {} + for elem in accumulatedElements: + if isinstance(elem, dict): + elemType = elem.get("type", "unknown") + if elemType not in accumulatedByType: + accumulatedByType[elemType] = [] + accumulatedByType[elemType].append(elem) + + newFragmentByType = {} + for elem in newFragmentElements: + if isinstance(elem, dict): + elemType = elem.get("type", "unknown") + if elemType not in newFragmentByType: + newFragmentByType[elemType] = [] + newFragmentByType[elemType].append(elem) + + # Step 5: Merge elements intelligently + mergedElements = [] + allTypes = set(accumulatedByType.keys()) | set(newFragmentByType.keys()) + + for elemType in allTypes: + accElems = accumulatedByType.get(elemType, []) + fragElems = newFragmentByType.get(elemType, []) + + if not accElems: + # Only in fragment - add all + mergedElements.extend(fragElems) + elif not fragElems: + # Only in accumulated - add all + mergedElements.extend(accElems) + else: + # Both have elements of this type - merge them using content-type-specific logic + mergedElem = JsonResponseHandler._mergeElementsOfSameTypeGeneric( + accElems[0], fragElems[0], elemType, accumulatedRaw, newFragmentRaw, isFragment + ) + if mergedElem: + mergedElements.append(mergedElem) + + # Step 6: Reconstruct base structure + if mergedElements: + return {"elements": mergedElements} + else: + # No merged elements - return accumulated if available (NEVER return None) + if accumulatedElements: + return {"elements": accumulatedElements} + # If no accumulated, return new fragment if available + if newFragmentElements: + return {"elements": newFragmentElements} + # Last resort: return empty structure (never None) + return {"elements": []} + + except Exception as e: + logger.debug(f"Structure-based merge failed: {e}") + import traceback + logger.debug(traceback.format_exc()) + return None + + @staticmethod + def _isFragment(jsonString: str, elements: List[Dict[str, Any]]) -> bool: + """ + Detect if JSON string is a fragment (not a complete JSON object). + + Fragments: + - Start with `[` but not `[{"` (array fragment, not full elements array) + - Start with array elements like `["cell1", "cell2"],` (table rows fragment) + - Don't have full structure (missing outer object with "elements") + - Are continuations of previous structure + """ + jsonStripped = jsonString.strip() + + # Check if it starts with array (fragment) + if jsonStripped.startswith('['): + # Check if it's a full elements array `[{"type": ...}]` or a fragment `["cell1", "cell2"]` + if jsonStripped.startswith('[{"') or jsonStripped.startswith('[{'): + # Could be full structure - check if it has "type" field + if elements and isinstance(elements[0], dict) and "type" in elements[0]: + return False # Full structure + # Otherwise it's a fragment (array of primitives or incomplete) + return True + + # Check if it starts with object but missing "elements" wrapper + if jsonStripped.startswith('{'): + # Check if it has "elements" field + if '"elements"' not in jsonStripped[:200]: # Check first 200 chars + # Might be a single element fragment + return True + + # Check if elements are incomplete (no full structure) + if elements and isinstance(elements[0], dict): + # Check if first element is missing required fields + firstElem = elements[0] + if "type" not in firstElem and "content" not in firstElem: + return True + + return False + + @staticmethod + def _mergeElementsOfSameTypeGeneric( + accumulatedElem: Dict[str, Any], + newFragmentElem: Dict[str, Any], + elemType: str, + accumulatedRaw: str, + newFragmentRaw: str, + isFragment: bool + ) -> Optional[Dict[str, Any]]: + """ + GENERIC merge of two elements of the same type, with content-type-specific optimizations. + + Content-type-specific merging: + - table: Merge rows arrays with overlap detection + - paragraph: Merge text content + - code_block: Merge code strings + - bullet_list/numbered_list: Merge items arrays + - heading: Use new fragment (usually complete) + - image: Use new fragment (usually complete) + - Other: Generic deep merge + + Args: + accumulatedElem: Accumulated element + newFragmentElem: New fragment element + elemType: Content type (table, paragraph, etc.) + accumulatedRaw: Raw accumulated string + newFragmentRaw: Raw new fragment string + isFragment: Whether newFragment is a fragment (continuation) + + Returns: + Merged element or None if merging fails + """ + if elemType == "table": + return JsonResponseHandler._mergeTableElementsGeneric( + accumulatedElem, newFragmentElem, accumulatedRaw, newFragmentRaw, isFragment + ) + elif elemType == "paragraph": + return JsonResponseHandler._mergeParagraphElements( + accumulatedElem, newFragmentElem, isFragment + ) + elif elemType == "code_block": + return JsonResponseHandler._mergeCodeBlockElements( + accumulatedElem, newFragmentElem, isFragment + ) + elif elemType in ["bullet_list", "numbered_list"]: + return JsonResponseHandler._mergeListElements( + accumulatedElem, newFragmentElem, isFragment + ) + elif elemType in ["heading", "image"]: + # Usually complete - use new fragment if it exists, otherwise accumulated + return newFragmentElem if newFragmentElem else accumulatedElem + else: + # Generic merge: use mergeDeepStructures + return JsonResponseHandler.mergeDeepStructures( + accumulatedElem, newFragmentElem, 0, f"element_merge.{elemType}" + ) + + @staticmethod + def _mergeTableElementsGeneric( + accumulatedElem: Dict[str, Any], + newFragmentElem: Dict[str, Any], + accumulatedRaw: str, + newFragmentRaw: str, + isFragment: bool + ) -> Dict[str, Any]: + """ + GENERIC merge of two table elements with content-type-specific optimizations. + + Handles: + - Overlapping rows (detect duplicates by comparing row content) + - Missing headers (complete with existing headers) + - Incomplete rows (complete with null values if needed) + - Fragment rows (if newFragment is a fragment, extract rows from raw string) + + Args: + accumulatedElem: Accumulated table element + newFragmentElem: New fragment table element + accumulatedRaw: Raw accumulated string (for fragment detection) + newFragmentRaw: Raw new fragment string (for fragment extraction) + isFragment: Whether newFragment is a fragment + + Returns: + Merged table element + """ + # Extract content (handle both nested and flat structures) + accContent = accumulatedElem.get("content", {}) + if not accContent and "rows" in accumulatedElem: + accContent = accumulatedElem + + fragContent = newFragmentElem.get("content", {}) + if not fragContent and "rows" in newFragmentElem: + fragContent = newFragmentElem + + # Extract rows + accRows = accContent.get("rows", []) if isinstance(accContent, dict) else [] + + # If fragment, try to extract rows from raw string + fragRows = fragContent.get("rows", []) if isinstance(fragContent, dict) else [] + if isFragment and not fragRows: + fragRows = JsonResponseHandler._extractRowsFromFragment(newFragmentRaw) + + # Extract headers (complete missing with existing) + accHeaders = accContent.get("headers", []) if isinstance(accContent, dict) else [] + fragHeaders = fragContent.get("headers", []) if isinstance(fragContent, dict) else [] + mergedHeaders = accHeaders if accHeaders else fragHeaders + + # Merge rows with overlap detection + mergedRows = JsonResponseHandler._mergeRowsWithOverlapDetection(accRows, fragRows) + + # Reconstruct table element + mergedContent = { + "headers": mergedHeaders, + "rows": mergedRows + } + + # Preserve other fields (caption, etc.) + if isinstance(accContent, dict) and "caption" in accContent: + mergedContent["caption"] = accContent["caption"] + elif isinstance(fragContent, dict) and "caption" in fragContent: + mergedContent["caption"] = fragContent["caption"] + + return { + "type": "table", + "content": mergedContent + } + + @staticmethod + def _extractRowsFromFragment(fragmentRaw: str) -> List[List[str]]: + """ + Extract table rows from fragment string. + + Handles fragments like: + - `["1947", "16883"], ["1948", "16889"], ...` + - `"rows": [["1947", "10000"], ["1948", "10100"]...` + - Incomplete fragments cut mid-string + Also handles fragments with more than 2 columns. + """ + import re + rows = [] + + # Pattern 1: Array of arrays with 2 columns `["cell1", "cell2"], ["cell3", "cell4"]` + # This pattern matches complete arrays: ["value1", "value2"] + pattern2Col = r'\["([^"]*)",\s*"([^"]*)"\]' + matches2Col = re.findall(pattern2Col, fragmentRaw) + + if matches2Col and len(matches2Col) >= 2: # Need at least 2 rows to be confident + for match in matches2Col: + if len(match) == 2: + rows.append([match[0], match[1]]) + if rows: + return rows + + # Pattern 2: Array of arrays with variable columns (more robust) + # Find all array patterns: ["...", "...", ...] + # Use non-greedy matching but ensure we get complete arrays + arrayPattern = r'\[(.*?)\]' + arrayMatches = re.findall(arrayPattern, fragmentRaw) + + # Filter to only arrays that look like table rows (have multiple quoted values) + validArrays = [] + for arrayContent in arrayMatches: + # Extract quoted strings from array content + cellPattern = r'"([^"]*)"' + cells = re.findall(cellPattern, arrayContent) + # Only consider arrays with 2+ cells (likely table rows) + if len(cells) >= 2: + validArrays.append(cells) + + if validArrays and len(validArrays) >= 2: # Need at least 2 rows + return validArrays + + # Pattern 3: Look for "rows": [...] pattern in incomplete JSON + # This handles cases like: "rows": [["1947", "10000"], ["1948", "10100"]... + rowsPattern = r'"rows"\s*:\s*\[(.*?)(?:\]|$)' + rowsMatch = re.search(rowsPattern, fragmentRaw, re.DOTALL) + if rowsMatch: + rowsContent = rowsMatch.group(1) + # Extract all array patterns from rows content + arrayPattern = r'\[(.*?)\]' + arrayMatches = re.findall(arrayPattern, rowsContent) + for arrayContent in arrayMatches: + cellPattern = r'"([^"]*)"' + cells = re.findall(cellPattern, arrayContent) + if len(cells) >= 2: # At least 2 columns + rows.append(cells) + if rows: + return rows + + # Pattern 4: Try to parse as JSON array (handles complete arrays) + + # Try to close incomplete structures + closed = closeJsonStructures(fragmentRaw.strip()) + parsed, parseErr, _ = tryParseJson(closed) + + if parseErr is None and isinstance(parsed, list): + if parsed and isinstance(parsed[0], list): + # Array of arrays - table rows + return parsed + elif parsed and isinstance(parsed[0], str): + # Array of strings - might be single column table + return [[item] for item in parsed] + + # Pattern 5: Last resort - extract any array patterns we can find + # Even if incomplete, try to extract what we can + if not rows: + # Find all patterns like ["value1", "value2"] even if incomplete + # Use a more lenient pattern that handles incomplete strings + incompletePattern = r'\["([^"]*)"(?:,\s*"([^"]*)")?' + incompleteMatches = re.findall(incompletePattern, fragmentRaw) + for match in incompleteMatches: + if match[0]: # First value exists + if match[1]: # Second value exists + rows.append([match[0], match[1]]) + else: + # Only one value - might be incomplete, skip for now + pass + + return rows + + @staticmethod + def _mergeParagraphElements( + accumulatedElem: Dict[str, Any], + newFragmentElem: Dict[str, Any], + isFragment: bool + ) -> Dict[str, Any]: + """Merge two paragraph elements.""" + accContent = accumulatedElem.get("content", {}) + fragContent = newFragmentElem.get("content", {}) + + accText = accContent.get("text", "") if isinstance(accContent, dict) else "" + fragText = fragContent.get("text", "") if isinstance(fragContent, dict) else "" + + # Merge text (remove overlap if fragment) + mergedText = accText + fragText if not isFragment else (accText.rstrip() + " " + fragText.lstrip()) + + return { + "type": "paragraph", + "content": {"text": mergedText} + } + + @staticmethod + def _mergeCodeBlockElements( + accumulatedElem: Dict[str, Any], + newFragmentElem: Dict[str, Any], + isFragment: bool + ) -> Dict[str, Any]: + """Merge two code block elements.""" + accContent = accumulatedElem.get("content", {}) + fragContent = newFragmentElem.get("content", {}) + + accCode = accContent.get("code", "") if isinstance(accContent, dict) else "" + fragCode = fragContent.get("code", "") if isinstance(fragContent, dict) else "" + + accLanguage = accContent.get("language") if isinstance(accContent, dict) else None + fragLanguage = fragContent.get("language") if isinstance(fragContent, dict) else None + + mergedCode = accCode + "\n" + fragCode if fragCode else accCode + mergedLanguage = accLanguage or fragLanguage + + result = { + "type": "code_block", + "content": {"code": mergedCode} + } + if mergedLanguage: + result["content"]["language"] = mergedLanguage + + return result + + @staticmethod + def _mergeListElements( + accumulatedElem: Dict[str, Any], + newFragmentElem: Dict[str, Any], + isFragment: bool + ) -> Dict[str, Any]: + """Merge two list elements (bullet_list or numbered_list).""" + accContent = accumulatedElem.get("content", {}) + fragContent = newFragmentElem.get("content", {}) + + accItems = accContent.get("items", []) if isinstance(accContent, dict) else [] + fragItems = fragContent.get("items", []) if isinstance(fragContent, dict) else [] + + # Merge items with overlap detection + mergedItems = JsonResponseHandler._mergeItemsWithOverlapDetection(accItems, fragItems) + + elemType = accumulatedElem.get("type") or newFragmentElem.get("type") + + return { + "type": elemType, + "content": {"items": mergedItems} + } + + @staticmethod + def _findOverlapStartIndex( + accumulatedElements: List[Dict[str, Any]], + overlapElements: List[Dict[str, Any]] + ) -> int: + """ + Find the start index in accumulatedElements where overlapElements begin. + + This helps identify where to merge continuation elements by matching + the overlap elements with the end of accumulated elements. + + Args: + accumulatedElements: List of accumulated elements + overlapElements: List of overlap elements from continuation response + + Returns: + Index where overlap starts, or -1 if not found + """ + if not overlapElements or not accumulatedElements: + return -1 + + # Try to find overlap by matching element structures + # Start from the end of accumulatedElements and work backwards + overlapLen = len(overlapElements) + accLen = len(accumulatedElements) + + if overlapLen > accLen: + return -1 + + # Try matching from different start positions + for startIdx in range(max(0, accLen - overlapLen), accLen): + # Check if elements from startIdx match overlapElements + matches = True + for i in range(min(overlapLen, accLen - startIdx)): + accElem = accumulatedElements[startIdx + i] + overlapElem = overlapElements[i] + + # Compare element types + if isinstance(accElem, dict) and isinstance(overlapElem, dict): + accType = accElem.get("type") + overlapType = overlapElem.get("type") + if accType != overlapType: + matches = False + break + + # For tables, compare row counts or last rows + if accType == "table": + accRows = accElem.get("rows", []) or (accElem.get("content", {}).get("rows", []) if isinstance(accElem.get("content"), dict) else []) + overlapRows = overlapElem.get("rows", []) or (overlapElem.get("content", {}).get("rows", []) if isinstance(overlapElem.get("content"), dict) else []) + if accRows and overlapRows: + # Check if last rows match + if len(accRows) >= len(overlapRows): + lastAccRows = accRows[-len(overlapRows):] + if lastAccRows != overlapRows: + matches = False + break + # For lists, compare items + elif accType in ["bullet_list", "numbered_list"]: + accItems = accElem.get("items", []) or (accElem.get("content", {}).get("items", []) if isinstance(accElem.get("content"), dict) else []) + overlapItems = overlapElem.get("items", []) or (overlapElem.get("content", {}).get("items", []) if isinstance(overlapElem.get("content"), dict) else []) + if accItems and overlapItems: + if len(accItems) >= len(overlapItems): + lastAccItems = accItems[-len(overlapItems):] + if lastAccItems != overlapItems: + matches = False + break + else: + matches = False + break + + if matches: + return startIdx + + return -1 + + @staticmethod + def _mergeRowsWithOverlapDetection( + accRows: List[List[str]], + fragRows: List[List[str]] + ) -> List[List[str]]: + """ + Merge two row arrays, detecting and removing overlaps. + + Overlap detection: Compare rows to find duplicates. + Missing parts: Complete with null values if needed. + """ + if not accRows: + return fragRows + if not fragRows: + return accRows + + # Find overlap by comparing last rows of accRows with first rows of fragRows + overlapStart = 0 + maxOverlap = min(len(accRows), len(fragRows)) + + # Find the longest overlap + for overlapLen in range(maxOverlap, 0, -1): + accSuffix = accRows[-overlapLen:] + fragPrefix = fragRows[:overlapLen] + + # Compare rows (exact match) + if accSuffix == fragPrefix: + overlapStart = overlapLen + break + + # Merge: accumulated rows + non-overlapping fragment rows + merged = accRows + fragRows[overlapStart:] + + return merged + + @staticmethod + def _mergeItemsWithOverlapDetection( + accItems: List[str], + fragItems: List[str] + ) -> List[str]: + """ + Merge two item arrays (for lists), detecting and removing overlaps. + + Overlap detection: Compare items to find duplicates. + """ + if not accItems: + return fragItems + if not fragItems: + return accItems + + # Find overlap by comparing last items of accItems with first items of fragItems + overlapStart = 0 + maxOverlap = min(len(accItems), len(fragItems)) + + # Find the longest overlap + for overlapLen in range(maxOverlap, 0, -1): + accSuffix = accItems[-overlapLen:] + fragPrefix = fragItems[:overlapLen] + + # Compare items (exact match) + if accSuffix == fragPrefix: + overlapStart = overlapLen + break + + # Merge: accumulated items + non-overlapping fragment items + merged = accItems + fragItems[overlapStart:] + + return merged + + @staticmethod + def _extractOverlapAndContinuation(jsonString: str) -> Tuple[Optional[List[Dict[str, Any]]], Optional[str]]: + """ + Extract overlap and continuation sections from AI response with explicit overlap structure. + + Expected format: + { + "overlap": [...], // Elements to repeat for merging + "continuation": [...] // New elements to add + } + + Or alternative format: + { + "overlap": "...", // Overlap as string + "continuation": "..." // Continuation as string + } + + Args: + jsonString: JSON string that may contain overlap/continuation structure + + Returns: + Tuple of (overlap_elements, continuation_json_string) or (None, None) if not found + """ + if not jsonString: + return None, None + + + # Extract and normalize JSON + extracted = stripCodeFences(normalizeJsonText(jsonString)).strip() + if not extracted: + return None, None + + # Try to parse + try: + closed = closeJsonStructures(extracted) + parsed, parseErr, _ = tryParseJson(closed) + + if parseErr is None and isinstance(parsed, dict): + # Check for overlap/continuation structure + overlap = parsed.get("overlap") + continuation = parsed.get("continuation") + + if overlap is not None and continuation is not None: + # Found explicit overlap structure + overlapElements = None + continuationJson = None + + # Extract overlap elements + if isinstance(overlap, list): + overlapElements = overlap + elif isinstance(overlap, str): + # Overlap is a string - try to parse it + try: + overlapParsed, _, _ = tryParseJson(closeJsonStructures(overlap)) + if isinstance(overlapParsed, list): + overlapElements = overlapParsed + except Exception: + pass + + # Extract continuation JSON + if isinstance(continuation, (dict, list)): + continuationJson = json.dumps(continuation, indent=2, ensure_ascii=False) + elif isinstance(continuation, str): + continuationJson = continuation + + if overlapElements is not None and continuationJson: + return overlapElements, continuationJson + except Exception: + pass + + return None, None + + @staticmethod + def _mergeWithExplicitOverlap( + accumulated: str, + continuationJson: str, + overlapElements: List[Dict[str, Any]] + ) -> str: + """ + Merge accumulated JSON with continuation JSON using explicit overlap information. + + Strategy: + 1. Find overlap in accumulated using overlapElements + 2. Remove overlapping elements from accumulated + 3. Append continuation JSON + + Args: + accumulated: Previously accumulated JSON string + continuationJson: Continuation JSON string (new content) + overlapElements: List of overlap elements from AI response + + Returns: + Merged JSON string + """ + if not accumulated: + return continuationJson + if not continuationJson: + return accumulated + + + # Normalize accumulated + accumulatedExtracted = stripCodeFences(normalizeJsonText(accumulated)).strip() + accumulatedNormalized = JsonResponseHandler._normalizeToElementsStructure( + accumulatedExtracted, accumulated + ) + + # Normalize continuation + continuationExtracted = stripCodeFences(normalizeJsonText(continuationJson)).strip() + continuationNormalized = JsonResponseHandler._normalizeToElementsStructure( + continuationExtracted, continuationJson + ) + + # If both normalized successfully, use structure-based merge with overlap + if accumulatedNormalized and continuationNormalized: + merged = JsonResponseHandler._mergeJsonStructuresGeneric( + accumulatedNormalized, continuationNormalized, accumulatedExtracted, continuationExtracted, + overlapElements=overlapElements + ) + if merged: + return json.dumps(merged, indent=2, ensure_ascii=False) + + # Fallback: use overlap elements to find merge point in accumulated + # Find where overlap elements match in accumulated + if accumulatedNormalized and overlapElements: + accumulatedElements = accumulatedNormalized.get("elements", []) + overlapStartIndex = JsonResponseHandler._findOverlapStartIndex(accumulatedElements, overlapElements) + + if overlapStartIndex >= 0: + # Remove overlapping elements + accumulatedElements = accumulatedElements[:overlapStartIndex] + accumulatedNormalized["elements"] = accumulatedElements + + # Merge continuation + if continuationNormalized: + continuationElements = continuationNormalized.get("elements", []) + accumulatedElements.extend(continuationElements) + accumulatedNormalized["elements"] = accumulatedElements + return json.dumps(accumulatedNormalized, indent=2, ensure_ascii=False) + + # Last resort: simple concatenation + return JsonResponseHandler._mergeJsonStringsWithOverlapFallback(accumulated, continuationJson) + + @staticmethod + def _extractValidJsonPrefix(jsonString: str) -> str: + """ + Extract the longest valid JSON prefix from a string that may be cut randomly. + + Strategy: + 1. Try to find the longest prefix that can be closed and parsed + 2. Handle random cuts (mid-string, mid-number, etc.) + 3. Return the longest valid prefix found + + Args: + jsonString: JSON string that may be cut randomly + + Returns: + Longest valid JSON prefix, or empty string if none found + """ + if not jsonString or not jsonString.strip(): + return "" + + + # Strategy 1: Try progressive truncation to find longest valid JSON + # Use binary search-like approach for efficiency + bestValid = "" + bestLength = 0 + maxLen = len(jsonString) + + # Generate test lengths: full, 95%, 90%, ..., 10% + testLengths = [] + for percent in range(100, 9, -5): + testLen = int(maxLen * percent / 100) + if testLen > bestLength: + testLengths.append(testLen) + + # Also test specific points near the end (common cut points) + for offset in [200, 100, 50, 20, 10, 5, 2, 1]: + if maxLen > offset: + testLen = maxLen - offset + if testLen > bestLength: + testLengths.append(testLen) + + # Sort and deduplicate + testLengths = sorted(set(testLengths), reverse=True) + + for testLen in testLengths: + if testLen <= bestLength: + continue # Already found better + + testStr = jsonString[:testLen] + if not testStr.strip(): + continue + + # Try to close and parse + try: + closed = closeJsonStructures(testStr) + parsed, parseErr, _ = tryParseJson(closed) + + if parseErr is None and parsed is not None: + # Valid JSON found + if testLen > bestLength: + bestValid = closed + bestLength = testLen + except Exception: + continue + + # Strategy 2: If we found valid JSON, return it + if bestValid: + return bestValid + + # Strategy 3: Try to extract balanced JSON (find first complete structure) + jsonStripped = jsonString.strip() + + if jsonStripped.startswith('{') or jsonStripped.startswith('['): + # Try to extract balanced JSON + balanced = extractFirstBalancedJson(jsonStripped) + if balanced and balanced != jsonStripped: + try: + closed = closeJsonStructures(balanced) + parsed, parseErr, _ = tryParseJson(closed) + if parseErr is None: + return closed + except Exception: + pass + + # Strategy 4: Try to repair by removing incomplete trailing structures + # Find the last complete element/item before the cut + try: + # For arrays: find last complete element + if jsonStripped.startswith('['): + # Find last complete array element + lastComma = jsonStripped.rfind(',') + if lastComma > 0: + # Try prefix up to last comma + prefix = jsonStripped[:lastComma].strip() + if prefix.endswith(','): + prefix = prefix[:-1].strip() + if prefix: + closed = closeJsonStructures(prefix + ']') + parsed, parseErr, _ = tryParseJson(closed) + if parseErr is None: + return closed + + # For objects: find last complete key-value pair + elif jsonStripped.startswith('{'): + # Find last complete key-value pair + lastComma = jsonStripped.rfind(',') + if lastComma > 0: + # Try prefix up to last comma + prefix = jsonStripped[:lastComma].strip() + if prefix.endswith(','): + prefix = prefix[:-1].strip() + if prefix: + closed = closeJsonStructures(prefix + '}') + parsed, parseErr, _ = tryParseJson(closed) + if parseErr is None: + return closed + except Exception: + pass + + # Last resort: return empty (caller will handle) + return "" + + @staticmethod + def _smartConcatenate(accumulated: str, newFragment: str) -> str: + """ + Smart concatenation that tries to merge JSON fragments intelligently. + + Strategy: + 1. Extract valid JSON from both fragments + 2. Parse both as JSON objects/arrays + 3. Merge them structurally + 4. Return valid JSON + + Args: + accumulated: Accumulated JSON string + newFragment: New fragment to append + + Returns: + Merged string with valid JSON, or empty if merging not possible + """ + if not accumulated or not newFragment: + return "" + + + # Extract valid JSON prefixes from both + accumulatedValid = JsonResponseHandler._extractValidJsonPrefix(accumulated) + newFragmentValid = JsonResponseHandler._extractValidJsonPrefix(newFragment) + + if not accumulatedValid: + accumulatedValid = accumulated + if not newFragmentValid: + newFragmentValid = newFragment + + # Try to parse both + try: + closedAccumulated = closeJsonStructures(accumulatedValid) + parsedAccumulated, parseErr1, _ = tryParseJson(closedAccumulated) + + closedNewFragment = closeJsonStructures(newFragmentValid) + parsedNewFragment, parseErr2, _ = tryParseJson(closedNewFragment) + + # If both parse successfully, merge structurally + if parseErr1 is None and parseErr2 is None: + # Normalize both to elements structure + accNormalized = JsonResponseHandler._normalizeToElementsStructure(closedAccumulated, accumulated) + newNormalized = JsonResponseHandler._normalizeToElementsStructure(closedNewFragment, newFragment) + + if accNormalized and newNormalized: + merged = JsonResponseHandler._mergeJsonStructuresGeneric( + accNormalized, newNormalized, closedAccumulated, closedNewFragment + ) + if merged: + return json.dumps(merged, indent=2, ensure_ascii=False) + + # If only accumulated parses, return it + if parseErr1 is None and parsedAccumulated: + return json.dumps(parsedAccumulated, indent=2, ensure_ascii=False) + + # If only new fragment parses, return it + if parseErr2 is None and parsedNewFragment: + return json.dumps(parsedNewFragment, indent=2, ensure_ascii=False) + except Exception: + pass + + # Fallback: Try simple string concatenation with repair + accumulatedStripped = accumulated.strip() + newFragmentStripped = newFragment.strip() + + # If accumulated doesn't end with } or ], it might be incomplete + if accumulatedStripped and not accumulatedStripped.endswith(('}', ']')): + try: + closedAccumulated = closeJsonStructures(accumulatedStripped) + + # Check if newFragment starts with continuation + if newFragmentStripped.startswith(','): + # Remove leading comma and append + merged = closedAccumulated.rstrip() + newFragmentStripped.lstrip(',').strip() + elif newFragmentStripped.startswith(('}', ']')): + # Fragment starts with closing - might be completing accumulated + merged = closedAccumulated.rstrip() + newFragmentStripped + else: + # Try to append as continuation + # Check if we need a comma separator + if not closedAccumulated.rstrip().endswith((',', '[', '{')): + merged = closedAccumulated.rstrip() + ',' + newFragmentStripped + else: + merged = closedAccumulated.rstrip() + newFragmentStripped + + # Try to repair and parse the merged result + repaired = closeJsonStructures(merged) + parsed, parseErr, _ = tryParseJson(repaired) + if parseErr is None: + return json.dumps(parsed, indent=2, ensure_ascii=False) + except Exception: + pass + + # If smart concatenation failed, return empty (caller will handle) + return "" + + @staticmethod + def _mergeJsonStringsWithOverlapFallback( + accumulated: str, + newFragment: str + ) -> str: + """ + Fallback overlap detection using string comparison. + Used when both strings are complete JSON structures or fragments. + + CRITICAL: Never returns empty JSON - always returns at least accumulated. + """ + if not accumulated: + return newFragment if newFragment else "{}" + if not newFragment: + return accumulated + + + # Strategy 1: Try to extract valid JSON parts from both fragments + # This handles random cuts better by finding the longest valid prefix/suffix + + # Extract valid JSON from accumulated (find longest valid prefix) + accumulatedValid = JsonResponseHandler._extractValidJsonPrefix(accumulated) + + # Extract valid JSON from newFragment (find longest valid prefix) + newFragmentValid = JsonResponseHandler._extractValidJsonPrefix(newFragment) + + # If we have valid JSON from both, try structure-based merge + if accumulatedValid and newFragmentValid: + try: + parsedAccumulated, parseErr1, _ = tryParseJson(closeJsonStructures(accumulatedValid)) + parsedNewFragment, parseErr2, _ = tryParseJson(closeJsonStructures(newFragmentValid)) + + if parseErr1 is None and parseErr2 is None: + # Both are valid JSON - try structure merge + accNormalized = JsonResponseHandler._normalizeToElementsStructure(accumulatedValid, accumulated) + newNormalized = JsonResponseHandler._normalizeToElementsStructure(newFragmentValid, newFragment) + + if accNormalized and newNormalized: + merged = JsonResponseHandler._mergeJsonStructuresGeneric( + accNormalized, newNormalized, accumulatedValid, newFragmentValid + ) + if merged: + return json.dumps(merged, indent=2, ensure_ascii=False) + except Exception: + pass + + # Strategy 2: Find longest common suffix/prefix match (character-level overlap) + maxOverlapLen = min(len(accumulated), len(newFragment)) + + # Start from maximum possible overlap down to 1 character + # But limit to reasonable overlap (max 50% of shorter string) + maxReasonableOverlap = min(maxOverlapLen, min(len(accumulated), len(newFragment)) // 2) + + for overlapLen in range(maxReasonableOverlap, 0, -1): + accumulatedSuffix = accumulated[-overlapLen:] + newFragmentPrefix = newFragment[:overlapLen] + + if accumulatedSuffix == newFragmentPrefix: + # Found overlap - remove duplicate part + logger.debug(f"Found overlap of {overlapLen} characters, removing duplicate") + merged = accumulated + newFragment[overlapLen:] + # Ensure result is not empty + if merged and merged.strip(): + return merged + + # Strategy 3: No overlap found - try smart concatenation + # Check if we can append newFragment to accumulated without breaking JSON structure + merged = JsonResponseHandler._smartConcatenate(accumulated, newFragment) + if merged and merged.strip(): + return merged + + # Strategy 4: Last resort - simple concatenation (but ensure non-empty and valid JSON) + result = accumulated + newFragment + if not result or result.strip() in ['{}', '[]', '']: + # Return accumulated as fallback (at least we have that) + return accumulated if accumulated else "{}" + + # CRITICAL: Try to repair and validate the merged result + try: + repaired = closeJsonStructures(result) + parsed, parseErr, _ = tryParseJson(repaired) + if parseErr is None: + # Valid JSON - return it + return json.dumps(parsed, indent=2, ensure_ascii=False) + else: + # Still invalid - try to extract valid parts + validPrefix = JsonResponseHandler._extractValidJsonPrefix(result) + if validPrefix: + parsedPrefix, parseErr2, _ = tryParseJson(validPrefix) + if parseErr2 is None: + return json.dumps(parsedPrefix, indent=2, ensure_ascii=False) + except Exception: + pass + + # If repair failed, return accumulated (at least we have that) + if accumulated: + try: + repairedAccumulated = closeJsonStructures(accumulated) + parsedAcc, parseErrAcc, _ = tryParseJson(repairedAccumulated) + if parseErrAcc is None: + return json.dumps(parsedAcc, indent=2, ensure_ascii=False) + except Exception: + pass + return accumulated + + # Last resort: return empty structure + return "{}" + + @staticmethod + def isJsonComplete(parsedJson: Dict[str, Any]) -> bool: + """ + GENERIC function to check if parsed JSON structure is complete. + + Works for ANY JSON structure - no specific logic for content types. + + Completeness checks (all generic): + - All arrays are properly closed + - All objects are properly closed + - No incomplete structures + - Recursive validation of nested structures + + Args: + parsedJson: Parsed JSON object + + Returns: + True if JSON is complete, False otherwise + """ + def _checkStructureComplete(obj: Any, depth: int = 0) -> bool: + """Recursively check if structure is complete.""" + if depth > 50: # Prevent infinite recursion + return True + + if isinstance(obj, dict): + # Check all values recursively + for value in obj.values(): + if not _checkStructureComplete(value, depth + 1): + return False + return True + elif isinstance(obj, list): + # Check all items recursively + for item in obj: + if not _checkStructureComplete(item, depth + 1): + return False + return True + else: + # Primitive value - always complete + return True + + try: + return _checkStructureComplete(parsedJson) + except Exception as e: + logger.debug(f"Error checking JSON completeness: {e}") + return False + + @staticmethod + def finalizeJson(parsedJson: Dict[str, Any]) -> Dict[str, Any]: + """ + GENERIC function to finalize complete JSON by adding missing closing elements and repairing corruption. + + Works for ANY JSON structure - no specific logic for content types. + + Steps (all generic): + 1. Analyze structure for missing closing elements (recursively) + 2. Add closing brackets/braces where needed + 3. Repair any remaining corruption + 4. Validate final structure + + Args: + parsedJson: Parsed JSON object that needs finalization + + Returns: + Finalized JSON object + """ + # For now, just return as-is since parsing succeeded + # If needed, can add logic to check for incomplete structures + # and add closing elements + return parsedJson + + @staticmethod + def extractKpiValuesFromJson( + parsedJson: Dict[str, Any], + kpis: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """ + Extract current KPI values from parsed JSON and update KPI objects. + + Args: + parsedJson: Parsed JSON object + kpis: List of KPI objects (will be updated with currentValue) + + Returns: + Updated list of KPI objects with currentValue set + """ + updatedKpis = [] + + for kpi in kpis: + kpiId = kpi.get("id") + jsonPath = kpi.get("jsonPath") + + if not kpiId or not jsonPath: + continue + + # Create copy of KPI object + updatedKpi = kpi.copy() + + try: + # Extract value using JSON path + # Simple path format: "sections[0].elements[0].items" or "sections[0].elements[0].rows" + value = JsonResponseHandler._extractValueByPath(parsedJson, jsonPath) + + # Handle None (path doesn't exist - incomplete JSON) + if value is None: + updatedKpi["currentValue"] = kpi.get("currentValue", 0) + logger.debug(f"KPI {kpiId} path {jsonPath} not found in JSON (incomplete), keeping current value {updatedKpi['currentValue']}") + # Count items/rows/elements based on type + elif isinstance(value, list): + updatedKpi["currentValue"] = len(value) + logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: list with {len(value)} items") + elif isinstance(value, (int, float)): + updatedKpi["currentValue"] = int(value) + logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: numeric value {int(value)}") + else: + updatedKpi["currentValue"] = 0 + logger.debug(f"Extracted KPI {kpiId} from path {jsonPath}: non-list/non-numeric value, set to 0") + + except Exception as e: + logger.warning(f"Error extracting KPI {kpiId} from path {jsonPath}: {e}") + updatedKpi["currentValue"] = kpi.get("currentValue", 0) + + updatedKpis.append(updatedKpi) + + return updatedKpis + + @staticmethod + def extractKpiValuesFromIncompleteJson( + jsonString: str, + kpis: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """ + Extract KPI values from incomplete JSON string. + Uses existing JSON completion function to close incomplete structures, then extracts KPIs. + + Args: + jsonString: Incomplete JSON string + kpis: List of KPI objects + + Returns: + Updated list of KPI objects with currentValue set + """ + updatedKpis = [] + + for kpi in kpis: + kpiId = kpi.get("id") + jsonPath = kpi.get("jsonPath") + + if not kpiId or not jsonPath: + continue + + updatedKpi = kpi.copy() + + try: + # Use existing JSON completion function to close incomplete structures + + # Extract JSON string and complete it with missing closing elements + extracted = extractJsonString(jsonString) + completed = closeJsonStructures(extracted) + + # Parse completed JSON + parsed = json.loads(completed) + + # Extract value using path + value = JsonResponseHandler._extractValueByPath(parsed, jsonPath) + + # Handle None (path doesn't exist - incomplete JSON) + if value is None: + updatedKpi["currentValue"] = kpi.get("currentValue", 0) + logger.debug(f"KPI {kpiId} path {jsonPath} not found in completed JSON (still incomplete), keeping current value {updatedKpi['currentValue']}") + # Count items/rows/elements based on type + elif isinstance(value, list): + updatedKpi["currentValue"] = len(value) + logger.debug(f"Extracted KPI {kpiId} from completed JSON: list with {len(value)} items") + elif isinstance(value, (int, float)): + updatedKpi["currentValue"] = int(value) + logger.debug(f"Extracted KPI {kpiId} from completed JSON: numeric value {int(value)}") + else: + updatedKpi["currentValue"] = 0 + logger.debug(f"Extracted KPI {kpiId} from completed JSON: non-list/non-numeric value, set to 0") + + except Exception as e: + logger.warning(f"Error extracting KPI {kpiId} from incomplete JSON: {e}") + updatedKpi["currentValue"] = kpi.get("currentValue", 0) + + updatedKpis.append(updatedKpi) + + return updatedKpis + + @staticmethod + def _extractValueByPath(obj: Any, path: str) -> Any: + """ + Extract value from object using dot-notation path with array indices. + + Example: "sections[0].elements[0].items" + Returns None if path doesn't exist (for incomplete JSON handling). + """ + parts = path.split('.') + current = obj + + for part in parts: + if '[' in part and ']' in part: + # Handle array access: "sections[0]" + key = part[:part.index('[')] + index = int(part[part.index('[') + 1:part.index(']')]) + + if key: + if isinstance(current, dict): + current = current.get(key) + if current is None: + return None # Key doesn't exist + else: + return None # Can't access key on non-dict + + if isinstance(current, list): + if 0 <= index < len(current): + current = current[index] + else: + # Index out of range - return None for incomplete JSON + return None + else: + # Not a list, can't index + return None + else: + # Handle dict access + if isinstance(current, dict): + current = current.get(part) + if current is None: + return None # Key doesn't exist + else: + return None # Can't access key on non-dict + + return current + + @staticmethod + def validateKpiProgression( + accumulationState: JsonAccumulationState, + updatedKpis: List[Dict[str, Any]] + ) -> Tuple[bool, str]: + """ + Validate KPI progression from parsed JSON. + + Validation rules: + - Proceed if: At least ONE KPI increased + - Stop if: Any KPI went backwards → return (False, "KPI went backwards") + - Stop if: No KPIs progressed → return (False, "No progress") + - Finish if: All KPIs completed OR JSON is complete → return (True, "Complete") + + Args: + accumulationState: Current accumulation state (contains kpis) + updatedKpis: Updated KPI objects with currentValue set + + Returns: + Tuple of (shouldProceed, reason) + """ + if not accumulationState.kpis: + # No KPIs defined - always proceed + return True, "No KPIs defined" + + # Build dict of last values for comparison + lastValues = {kpi.get("id"): kpi.get("currentValue", 0) for kpi in accumulationState.kpis} + logger.debug(f"KPI validation: lastValues = {lastValues}") + logger.debug(f"KPI validation: updatedKpis = {[(kpi.get('id'), kpi.get('currentValue')) for kpi in updatedKpis]}") + + # Check if any KPI went backwards + for updatedKpi in updatedKpis: + kpiId = updatedKpi.get("id") + currentValue = updatedKpi.get("currentValue", 0) + + if kpiId in lastValues: + lastValue = lastValues[kpiId] + if currentValue < lastValue: + logger.warning(f"KPI {kpiId} went BACKWARDS: {lastValue} → {currentValue}") + return False, f"KPI {kpiId} went backwards" + + # Check if all KPIs are completed + allCompleted = True + for updatedKpi in updatedKpis: + targetValue = updatedKpi.get("targetValue", 0) + currentValue = updatedKpi.get("currentValue", 0) + + if currentValue < targetValue: + allCompleted = False + break + + if allCompleted: + logger.info("All KPIs completed") + return True, "All KPIs completed" + + # Check if at least one KPI progressed + atLeastOneProgressed = False + for updatedKpi in updatedKpis: + kpiId = updatedKpi.get("id") + currentValue = updatedKpi.get("currentValue", 0) + + if kpiId in lastValues: + lastValue = lastValues[kpiId] + if currentValue > lastValue: + atLeastOneProgressed = True + logger.info(f"KPI {kpiId} progressed: {lastValue} → {currentValue}") + break + else: + # First time seeing this KPI - if it has a value, it's progress + if currentValue > 0: + atLeastOneProgressed = True + logger.info(f"KPI {kpiId} initialized: {currentValue}") + break + + if not atLeastOneProgressed: + logger.warning(f"No KPIs progressed. Last values: {lastValues}, Current values: {[(kpi.get('id'), kpi.get('currentValue')) for kpi in updatedKpis]}") + return False, "No progress" + + return True, "Progress detected" + + @staticmethod + def accumulateAndParseJsonFragments( + accumulatedJsonString: str, + newFragmentString: str, + allSections: List[Dict[str, Any]], + iteration: int + ) -> Tuple[str, List[Dict[str, Any]], bool, Optional[Dict[str, Any]]]: + """ + Accumulate JSON fragments and parse when complete. + + GENERIC function that handles: + 1. Concatenating JSON strings with overlap detection + 2. Parsing the accumulated string + 3. Extracting sections (partial if incomplete, final if complete) + 4. Determining completion status + + Args: + accumulatedJsonString: Previously accumulated JSON string + newFragmentString: New fragment string from current iteration + allSections: Sections extracted so far (for prompt context) + iteration: Current iteration number + + Returns: + Tuple of: + - accumulatedJsonString: Updated accumulated string + - sections: Extracted sections (partial if incomplete, final if complete) + - isComplete: True if JSON is complete and valid + - parsedResult: Parsed JSON object (if parsing succeeded) + """ + + # Step 1: Clean encoding issues from accumulated string (check end of first delivered part) + cleanedAccumulated = JsonResponseHandler.cleanEncodingIssues(accumulatedJsonString) + + # Step 2: Clean encoding issues from new fragment + cleanedFragment = JsonResponseHandler.cleanEncodingIssues(newFragmentString) + + # Step 3: Concatenate with overlap handling + combinedString, hasOverlap = JsonResponseHandler.mergeJsonStringsWithOverlap( + cleanedAccumulated, + cleanedFragment + ) + # Note: hasOverlap indicates if iterations should continue, but this function + # doesn't control iterations, so we just use the merged string + + # Step 4: Try to parse + try: + extracted = extractJsonString(combinedString) + parsedResult = json.loads(extracted) + + # Step 5: Parsing succeeded - check completeness + isComplete = JsonResponseHandler.isJsonComplete(parsedResult) + + if isComplete: + # Step 6: Complete JSON - finalize + finalizedJson = JsonResponseHandler.finalizeJson(parsedResult) + sections = extractSectionsFromDocument(finalizedJson) + logger.info(f"Iteration {iteration}: JSON accumulation complete, extracted {len(sections)} sections") + return combinedString, sections, True, finalizedJson + else: + # Step 7: Incomplete but parseable - extract partial sections + sections = extractSectionsFromDocument(parsedResult) + logger.info(f"Iteration {iteration}: JSON accumulation incomplete but parseable, extracted {len(sections)} partial sections") + return combinedString, sections, False, parsedResult + + except json.JSONDecodeError: + # Step 8: Still broken - repair and extract partial sections + repaired = repairBrokenJson(combinedString) + if repaired: + sections = extractSectionsFromDocument(repaired) + logger.info(f"Iteration {iteration}: JSON accumulation repaired, extracted {len(sections)} sections") + return combinedString, sections, False, repaired + else: + # Repair failed - continue with data BEFORE merging the problematic piece + # Return previous accumulated string (before adding new fragment) + # This ensures we don't lose previously accumulated data + logger.warning(f"Iteration {iteration}: Repair failed, continuing with previous accumulated data") + return accumulatedJsonString, [], False, None + diff --git a/modules/serviceCenter/services/serviceAi/subLoopingUseCases.py b/modules/serviceCenter/services/serviceAi/subLoopingUseCases.py new file mode 100644 index 00000000..a2828108 --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subLoopingUseCases.py @@ -0,0 +1,293 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Generic Looping Use Case System + +Provides parametrized looping infrastructure supporting different JSON formats and use cases. +""" + +import logging +from dataclasses import dataclass, field +from typing import Dict, Any, List, Optional, Callable + +logger = logging.getLogger(__name__) + +# Callback functions for use-case-specific logic + +def _handleSectionContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for section_content: return raw result to preserve all JSON blocks.""" + final_json = result # Return raw response to preserve all JSON blocks + # Write final merged result for section_content (overwrites iteration 1 response with complete merged result) + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_response") + return final_json + + +def _handleChapterStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for chapter_structure: format JSON and write debug file.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + # Write final result for chapter structure + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") + return final_json + + +def _handleCodeStructureFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for code_structure: format JSON and write debug file.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + # Write final result for code structure + if services and hasattr(services, 'utils') and hasattr(services.utils, 'writeDebugFile'): + services.utils.writeDebugFile(final_json, f"{debugPrefix}_final_result") + return final_json + + +def _handleCodeContentFinalResult(result: str, parsedJsonForUseCase: Any, extractedJsonForUseCase: str, + debugPrefix: str, services: Any) -> str: + """Handle final result for code_content: format JSON.""" + import json + final_json = json.dumps(parsedJsonForUseCase, indent=2, ensure_ascii=False) if parsedJsonForUseCase else (extractedJsonForUseCase or result) + return final_json + + +def _normalizeSectionContentJson(parsed: Any, useCaseId: str) -> Any: + """Normalize JSON structure for section_content use case.""" + # For section_content, expect {"elements": [...]} structure + if isinstance(parsed, list): + # Check if list contains strings (invalid format) or element objects + if parsed and isinstance(parsed[0], str): + # Invalid format - list of strings instead of elements + # Try to convert strings to paragraph elements as fallback + logger.debug(f"Received list of strings instead of elements array, converting to paragraph elements") + elements = [] + for text in parsed: + if isinstance(text, str) and text.strip(): + elements.append({ + "type": "paragraph", + "content": { + "text": text.strip() + } + }) + return {"elements": elements} if elements else {"elements": []} + else: + # Convert plain list of elements to elements structure + return {"elements": parsed} + elif isinstance(parsed, dict): + # If it already has "elements", return as-is + if "elements" in parsed: + return parsed + # If it has "type" and looks like an element, wrap in elements array + elif parsed.get("type"): + return {"elements": [parsed]} + # Otherwise, assume it's already in correct format + else: + return parsed + + # For other use cases, return as-is (they have their own structures) + return parsed + + +def _normalizeDefaultJson(parsed: Any, useCaseId: str) -> Any: + """Default normalizer: return as-is.""" + return parsed + + +@dataclass +class LoopingUseCase: + """Configuration for a specific looping use case.""" + + # Identification + useCaseId: str # "section_content", "chapter_structure", "code_structure", "code_content" + + # JSON Format Detection + jsonTemplate: Dict[str, Any] # Expected JSON structure template + detectionKeys: List[str] # Keys to check for format detection (e.g., ["elements"], ["chapters"], ["files"]) + detectionPath: str # JSONPath to check (e.g., "documents[0].chapters", "files[0].content") + + # Prompt Building + initialPromptBuilder: Optional[Callable] = None # Function to build initial prompt + continuationPromptBuilder: Optional[Callable] = None # Function to build continuation prompt + + # Accumulation & Merging + accumulator: Optional[Callable] = None # Function to accumulate fragments + merger: Optional[Callable] = None # Function to merge accumulated data + + # Continuation Context + continuationContextBuilder: Optional[Callable] = None # Build continuation context for this format + + # Result Building + resultBuilder: Optional[Callable] = None # Build final result from accumulated data + + # Use-case-specific handlers (callbacks to avoid if/elif chains in generic code) + finalResultHandler: Optional[Callable] = None # Handle final result formatting and debug file writing + jsonNormalizer: Optional[Callable] = None # Normalize JSON structure for this use case + + # Metadata + supportsAccumulation: bool = True # Whether this use case supports accumulation + requiresExtraction: bool = False # Whether this requires extraction (like sections) + + +class LoopingUseCaseRegistry: + """Registry of all looping use cases.""" + + def __init__(self): + self.useCases: Dict[str, LoopingUseCase] = {} + self._registerDefaultUseCases() + + def register(self, useCase: LoopingUseCase): + """Register a new use case.""" + self.useCases[useCase.useCaseId] = useCase + logger.debug(f"Registered looping use case: {useCase.useCaseId}") + + def get(self, useCaseId: str) -> Optional[LoopingUseCase]: + """Get use case by ID.""" + return self.useCases.get(useCaseId) + + def detectUseCase(self, parsedJson: Dict[str, Any]) -> Optional[str]: + """Detect which use case matches the JSON structure.""" + for useCaseId, useCase in self.useCases.items(): + if self._matchesFormat(parsedJson, useCase): + return useCaseId + return None + + def _matchesFormat(self, json: Dict[str, Any], useCase: LoopingUseCase) -> bool: + """Check if JSON matches use case format.""" + # Check top-level keys + for key in useCase.detectionKeys: + if key in json: + return True + + # Check nested path using simple dictionary traversal (no jsonpath_ng needed) + if useCase.detectionPath: + try: + # Simple path matching without jsonpath_ng + # Format: "documents[0].chapters" or "files[0].content" + pathParts = useCase.detectionPath.split(".") + current = json + + for part in pathParts: + # Handle array indices like "documents[0]" + if "[" in part and "]" in part: + key = part.split("[")[0] + index = int(part.split("[")[1].split("]")[0]) + if isinstance(current, dict) and key in current: + if isinstance(current[key], list) and 0 <= index < len(current[key]): + current = current[key][index] + else: + return False + else: + return False + else: + # Regular key access + if isinstance(current, dict) and part in current: + current = current[part] + else: + return False + + # If we successfully traversed the path, it matches + return True + except Exception as e: + logger.debug(f"Path matching failed for {useCase.useCaseId}: {e}") + + return False + + def _registerDefaultUseCases(self): + """Register default use cases.""" + + # Use Case 1: Section Content Generation + # Returns JSON with "elements" array directly + self.register(LoopingUseCase( + useCaseId="section_content", + jsonTemplate={"elements": []}, + detectionKeys=["elements"], + detectionPath="", + initialPromptBuilder=None, # Will use default prompt builder + continuationPromptBuilder=None, # Will use default continuation builder + accumulator=None, # Direct return, no accumulation + merger=None, + continuationContextBuilder=None, # Will use default continuation context + resultBuilder=None, # Return JSON directly + finalResultHandler=_handleSectionContentFinalResult, + jsonNormalizer=_normalizeSectionContentJson, + supportsAccumulation=False, + requiresExtraction=False + )) + + # Use Case 2: Chapter Structure Generation + # Returns JSON with "documents[0].chapters" structure + self.register(LoopingUseCase( + useCaseId="chapter_structure", + jsonTemplate={"documents": [{"chapters": []}]}, + detectionKeys=["chapters"], + detectionPath="documents[0].chapters", + initialPromptBuilder=None, + continuationPromptBuilder=None, + accumulator=None, # Direct return, no accumulation + merger=None, + continuationContextBuilder=None, + resultBuilder=None, # Return JSON directly + finalResultHandler=_handleChapterStructureFinalResult, + jsonNormalizer=_normalizeDefaultJson, + supportsAccumulation=False, + requiresExtraction=False + )) + + # Use Case 3: Code Structure Generation + self.register(LoopingUseCase( + useCaseId="code_structure", + jsonTemplate={ + "metadata": { + "language": "", + "projectType": "single_file|multi_file", + "projectName": "" + }, + "files": [ + { + "id": "", + "filename": "", + "fileType": "", + "dependencies": [], + "imports": [], + "functions": [], + "classes": [] + } + ] + }, + detectionKeys=["files"], + detectionPath="files", + initialPromptBuilder=None, + continuationPromptBuilder=None, + accumulator=None, # Direct return + merger=None, + continuationContextBuilder=None, + resultBuilder=None, + finalResultHandler=_handleCodeStructureFinalResult, + jsonNormalizer=_normalizeDefaultJson, + supportsAccumulation=False, + requiresExtraction=False + )) + + # Use Case 5: Code Content Generation (NEW) + self.register(LoopingUseCase( + useCaseId="code_content", + jsonTemplate={"files": [{"content": "", "functions": []}]}, + detectionKeys=["content", "functions"], + detectionPath="files[0].content", + initialPromptBuilder=None, + continuationPromptBuilder=None, + accumulator=None, # Will use default accumulator + merger=None, # Will use default merger + continuationContextBuilder=None, + resultBuilder=None, # Will use default result builder + finalResultHandler=_handleCodeContentFinalResult, + jsonNormalizer=_normalizeDefaultJson, + supportsAccumulation=True, + requiresExtraction=False + )) + + logger.info(f"Registered {len(self.useCases)} default looping use cases") + diff --git a/modules/serviceCenter/services/serviceAi/subResponseParsing.py b/modules/serviceCenter/services/serviceAi/subResponseParsing.py new file mode 100644 index 00000000..68c123ac --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subResponseParsing.py @@ -0,0 +1,275 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Response Parsing Module + +Handles parsing of AI responses, including: +- Section extraction from responses +- JSON completeness detection +- Loop detection +- Document metadata extraction +- Final result building +""" +import json +import logging +from typing import Dict, Any, List, Optional, Tuple + +from modules.shared.jsonUtils import extractJsonString, repairBrokenJson, extractSectionsFromDocument +from .subJsonResponseHandling import JsonResponseHandler +from modules.datamodels.datamodelAi import JsonAccumulationState + +logger = logging.getLogger(__name__) + + +class ResponseParser: + """Handles parsing of AI responses and completion detection.""" + + def __init__(self, services): + """Initialize ResponseParser with service center access.""" + self.services = services + + def extractSectionsFromResponse( + self, + result: str, + iteration: int, + debugPrefix: str, + allSections: List[Dict[str, Any]] = None, + accumulationState: Optional[JsonAccumulationState] = None + ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]: + """ + Extract sections from AI response, handling both valid and broken JSON. + + NEW BEHAVIOR: + - First iteration: Check if complete, if not start accumulation + - Subsequent iterations: Accumulate strings, parse when complete + + Returns: + Tuple of: + - sections: Extracted sections + - wasJsonComplete: True if JSON is complete + - parsedResult: Parsed JSON object + - updatedAccumulationState: Updated accumulation state (None if not in accumulation mode) + """ + if allSections is None: + allSections = [] + + if iteration == 1: + # First iteration - check if complete + parsed = None + try: + extracted = extractJsonString(result) + parsed = json.loads(extracted) + + # Check completeness + if JsonResponseHandler.isJsonComplete(parsed): + # Complete JSON - no accumulation needed + sections = extractSectionsFromDocument(parsed) + logger.info(f"Iteration 1: Complete JSON detected, no accumulation needed") + return sections, True, parsed, None # No accumulation + except Exception: + pass + + # Incomplete - try to extract partial sections from broken JSON + logger.info(f"Iteration 1: Incomplete JSON detected, attempting to extract partial sections") + + partialSections = [] + if parsed: + # Try to extract sections from parsed (even if incomplete) + partialSections = extractSectionsFromDocument(parsed) + else: + # Try to repair broken JSON and extract sections + try: + repaired = repairBrokenJson(result) + if repaired: + partialSections = extractSectionsFromDocument(repaired) + parsed = repaired # Use repaired version for accumulation state + except Exception: + pass # If repair fails, continue with empty sections + + + # Define KPIs (async call - need to handle this) + # For now, create accumulation state without KPIs, will be updated after async call + accumulationState = JsonAccumulationState( + accumulatedJsonString=result, + isAccumulationMode=True, + lastParsedResult=parsed, + allSections=partialSections, + kpis=[] + ) + + # Note: KPI definition will be done in the caller (async context) + return partialSections, False, parsed, accumulationState + + else: + # Subsequent iterations - accumulate + if accumulationState and accumulationState.isAccumulationMode: + accumulated, sections, isComplete, parsedResult = \ + JsonResponseHandler.accumulateAndParseJsonFragments( + accumulationState.accumulatedJsonString, + result, + allSections, + iteration + ) + + # Update accumulation state + accumulationState.accumulatedJsonString = accumulated + accumulationState.lastParsedResult = parsedResult + accumulationState.allSections = allSections + sections if sections else allSections + accumulationState.isAccumulationMode = not isComplete + + # Log accumulated JSON for debugging + if parsedResult: + accumulated_json_str = json.dumps(parsedResult, indent=2, ensure_ascii=False) + self.services.utils.writeDebugFile(accumulated_json_str, f"{debugPrefix}_accumulated_json_iteration_{iteration}.json") + + return sections, isComplete, parsedResult, accumulationState + else: + # No accumulation mode - process normally (shouldn't happen) + logger.warning(f"Iteration {iteration}: No accumulation state but iteration > 1") + return [], False, None, None + + def shouldContinueGeneration( + self, + allSections: List[Dict[str, Any]], + iteration: int, + wasJsonComplete: bool, + rawResponse: str = None + ) -> bool: + """ + Determine if AI generation loop should continue. + + CRITICAL: This is ONLY about AI Loop Completion, NOT Action DoD! + Action DoD is checked AFTER the AI Loop completes in _refineDecide. + + Simple logic: + - If JSON parsing failed or incomplete → continue (needs more content) + - If JSON parses successfully and is complete → stop (all content delivered) + - Loop detection prevents infinite loops + + CRITICAL: JSON completeness is determined by parsing, NOT by last character check! + Returns True if we should continue, False if AI Loop is done. + """ + if len(allSections) == 0: + return True # No sections yet, continue + + # CRITERION 1: If JSON was incomplete/broken (parsing failed or incomplete) - continue to repair/complete + if not wasJsonComplete: + logger.info(f"Iteration {iteration}: JSON incomplete/broken - continuing to complete") + return True + + # CRITERION 2: JSON is complete (parsed successfully) - check for loop detection + if self._isStuckInLoop(allSections, iteration): + logger.warning(f"Iteration {iteration}: Detected potential infinite loop - stopping AI loop") + return False + + # JSON is complete and not stuck in loop - done + logger.info(f"Iteration {iteration}: JSON complete - AI loop done") + return False + + def _isStuckInLoop( + self, + allSections: List[Dict[str, Any]], + iteration: int + ) -> bool: + """ + Detect if we're stuck in a loop (same content being repeated). + + Generic approach: Check if recent iterations are adding minimal or duplicate content. + """ + if iteration < 3: + return False # Need at least 3 iterations to detect a loop + + if len(allSections) == 0: + return False + + # Check if last section is very small (might be stuck) + lastSection = allSections[-1] + elements = lastSection.get("elements", []) + + if isinstance(elements, list) and elements: + lastElem = elements[-1] if elements else {} + else: + lastElem = elements if isinstance(elements, dict) else {} + + # Check content size of last section + lastSectionSize = 0 + if isinstance(lastElem, dict): + for key, value in lastElem.items(): + if isinstance(value, str): + lastSectionSize += len(value) + elif isinstance(value, list): + lastSectionSize += len(str(value)) + + # If last section is very small and we've done many iterations, might be stuck + if lastSectionSize < 100 and iteration > 10: + logger.warning(f"Potential loop detected: iteration {iteration}, last section size {lastSectionSize}") + return True + + return False + + def extractDocumentMetadata( + self, + parsedResult: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + """ + Extract document metadata (title, filename) from parsed AI response. + Returns dict with 'title' and 'filename' keys if found, None otherwise. + """ + if not isinstance(parsedResult, dict): + return None + + # Try to get from documents array (preferred structure) + if "documents" in parsedResult and isinstance(parsedResult["documents"], list) and len(parsedResult["documents"]) > 0: + firstDoc = parsedResult["documents"][0] + if isinstance(firstDoc, dict): + title = firstDoc.get("title") + filename = firstDoc.get("filename") + if title or filename: + return { + "title": title, + "filename": filename + } + + return None + + def buildFinalResultFromSections( + self, + allSections: List[Dict[str, Any]], + documentMetadata: Optional[Dict[str, Any]] = None + ) -> str: + """ + Build final JSON result from accumulated sections. + Uses AI-provided metadata (title, filename) if available. + """ + if not allSections: + return "" + + # Extract metadata from AI response if available + title = "Generated Document" + filename = "document.json" + if documentMetadata: + if documentMetadata.get("title"): + title = documentMetadata["title"] + if documentMetadata.get("filename"): + filename = documentMetadata["filename"] + + # Build documents structure + # Assuming single document for now + documents = [{ + "id": "doc_1", + "title": title, + "filename": filename, + "sections": allSections + }] + + result = { + "metadata": { + "split_strategy": "single_document", + "source_documents": [], + "extraction_method": "ai_generation" + }, + "documents": documents + } + + return json.dumps(result, indent=2) + diff --git a/modules/serviceCenter/services/serviceAi/subStructureFilling.py b/modules/serviceCenter/services/serviceAi/subStructureFilling.py new file mode 100644 index 00000000..4df52b56 --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subStructureFilling.py @@ -0,0 +1,2593 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Structure Filling Module + +Handles filling document structure with content, including: +- Filling sections with content parts +- Building section generation prompts +- Aggregation logic +""" +import json +import logging +import copy +import asyncio +from typing import Dict, Any, List, Optional, Tuple + +from modules.datamodels.datamodelExtraction import ContentPart +from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.workflows.processing.shared.stateTools import checkWorkflowStopped + +logger = logging.getLogger(__name__) + + +class StructureFiller: + """Handles filling document structure with content.""" + + # Default concurrency limit for parallel generation (chapters/sections) + DEFAULT_MAX_CONCURRENT_GENERATION = 16 + + def __init__(self, services, aiService): + """Initialize StructureFiller with service center and AI service access.""" + self.services = services + self.aiService = aiService + + def _getMaxConcurrentGeneration(self, options: Optional[AiCallOptions] = None) -> int: + """Get max concurrent generation limit, configurable via options.""" + if options and hasattr(options, 'maxConcurrentGeneration'): + return options.maxConcurrentGeneration + return self.DEFAULT_MAX_CONCURRENT_GENERATION + + def _getUserLanguage(self) -> str: + """Get user language for document generation""" + try: + if self.services: + # Prefer detected language if available (from user intention analysis) + if hasattr(self.services, 'currentUserLanguage') and self.services.currentUserLanguage: + return self.services.currentUserLanguage + # Fallback to user's preferred language + elif hasattr(self.services, 'user') and self.services.user and hasattr(self.services.user, 'language'): + return self.services.user.language + except Exception: + pass + return 'en' # Default fallback + + def _getDocumentLanguage(self, structure: Dict[str, Any], documentId: str) -> str: + """ + Get language for a specific document from structure. + Falls back to user language if not specified. + + Args: + structure: The document structure with documents array + documentId: The ID of the document to get language for + + Returns: + ISO 639-1 language code (e.g., "de", "en", "fr") + """ + # Try to find document in structure + for doc in structure.get("documents", []): + if doc.get("id") == documentId: + docLanguage = doc.get("language") + if docLanguage: + return docLanguage + + # Fallback to metadata language + metadataLanguage = structure.get("metadata", {}).get("language") + if metadataLanguage: + return metadataLanguage + + # Fallback to user language + return self._getUserLanguage() + + def _extractContentPartInfo(self, chapter: Dict[str, Any]) -> Tuple[List[str], Dict[str, Any]]: + """ + Extract contentPartIds and contentPartInstructions from chapter's contentParts structure. + + Returns: + tuple: (contentPartIds list, contentPartInstructions dict) + """ + contentParts = chapter.get("contentParts", {}) + contentPartIds = list(contentParts.keys()) + # Extract instructions (entries with "instruction" field) and captions (entries with "caption" field) + contentPartInstructions = {} + for partId, partInfo in contentParts.items(): + if isinstance(partInfo, dict): + if "instruction" in partInfo: + contentPartInstructions[partId] = {"instruction": partInfo["instruction"]} + elif "caption" in partInfo: + # For entries with only caption (no instruction), still add to dict so it's available + contentPartInstructions[partId] = {"caption": partInfo["caption"]} + return contentPartIds, contentPartInstructions + + def _getContentPartCaption(self, chapter: Dict[str, Any], partId: str) -> Optional[str]: + """ + Get caption for a contentPart from chapter's contentParts structure. + Returns None if no caption is available. + + Args: + chapter: Chapter dict + partId: ContentPart ID + + Returns: + Caption string or None + """ + if "contentParts" in chapter: + contentParts = chapter.get("contentParts", {}) + partInfo = contentParts.get(partId) + if isinstance(partInfo, dict) and "caption" in partInfo: + return partInfo["caption"] + return None + + async def fillStructure( + self, + structure: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + parentOperationId: str, + language: Optional[str] = None + ) -> Dict[str, Any]: + """ + Phase 5D: Chapter-Content-Generierung (Zwei-Phasen-Ansatz). + + Phase 5D.1: Generiert Sections-Struktur für jedes Chapter + Phase 5D.2: Füllt Sections mit ContentParts + + Args: + structure: Struktur-Dict mit documents und chapters (nicht sections!) + contentParts: Alle vorbereiteten ContentParts + userPrompt: User-Anfrage + parentOperationId: Parent Operation-ID für ChatLog-Hierarchie + language: Language identified from user intention analysis (e.g., "de", "en", "fr") + + Returns: + Gefüllte Struktur mit elements in jeder Section (nach Flattening) + """ + # Erstelle Operation-ID für Struktur-Abfüllen + fillOperationId = f"{parentOperationId}_structure_filling" + + # Validate structure has chapters + hasChapters = False + for doc in structure.get("documents", []): + if "chapters" in doc: + hasChapters = True + break + + if not hasChapters: + error_msg = "Structure must have chapters. Legacy section-based structure is not supported." + logger.error(error_msg) + raise ValueError(error_msg) + + # Get language from services (user intention analysis) or parameter + if language is None: + language = self._getUserLanguage() + logger.debug(f"Using language from services (user intention analysis): {language}") + else: + logger.debug(f"Using provided language parameter: {language}") + + # Starte ChatLog mit Parent-Referenz + chapterCount = sum(len(doc.get("chapters", [])) for doc in structure.get("documents", [])) + self.services.chat.progressLogStart( + fillOperationId, + "Chapter Content Generation", + "Filling", + f"Processing {chapterCount} chapters", + parentOperationId=parentOperationId + ) + + try: + filledStructure = copy.deepcopy(structure) + + # Get options from AI service if available (for concurrency control) + # Default concurrency limit (16) will be used if options is None + options = None + # Note: Options can be passed via fillStructure if needed in the future + + # Phase 5D.1: Sections-Struktur für jedes Chapter generieren + filledStructure = await self._generateChapterSectionsStructure( + filledStructure, contentParts, userPrompt, fillOperationId, language, options + ) + + # Phase 5D.2: Sections mit ContentParts füllen + filledStructure = await self._fillChapterSections( + filledStructure, contentParts, userPrompt, fillOperationId, language, options + ) + + # Flattening: Chapters zu Sections konvertieren + flattenedStructure = self._flattenChaptersToSections(filledStructure) + + # Füge ContentParts-Metadaten zur Struktur hinzu (für Validierung) + flattenedStructure = self._addContentPartsMetadata(flattenedStructure, contentParts) + + # State 4 Validation: Validate and auto-fix filled structure + # Validation 4.1: Filled structure missing 'documents' field + if "documents" not in flattenedStructure: + raise ValueError("Filled structure missing 'documents' field - cannot auto-fix") + + for doc in flattenedStructure["documents"]: + # Validation 4.4: Verify language is preserved from input structure + # Language MUST be preserved from Phase 3 structure (validated in State 3) + if "language" not in doc: + raise ValueError(f"Document {doc.get('id')} missing language in filled structure - should have been preserved from Phase 3") + + # Validate language format + if not isinstance(doc["language"], str) or len(doc["language"]) != 2: + raise ValueError(f"Document {doc.get('id')} has invalid language format in filled structure: {doc['language']} - should be 2-character ISO 639-1 code") + + # CRITICAL: flattenedStructure has sections, not chapters! + # After flattening, chapters are converted to sections, so we need to validate sections directly + for section in doc.get("sections", []): + # Validation 4.2: Section missing 'elements' field + if "elements" not in section: + section["elements"] = [] + logger.info(f"Section {section.get('id')} missing 'elements' - created empty list") + + # Validation 4.3: Section has empty elements list - ALLOW (intentionally empty is OK) + # No action needed - empty elements are allowed + + # ChatLog abschließen + self.services.chat.progressLogFinish(fillOperationId, True) + + return flattenedStructure + + except Exception as e: + self.services.chat.progressLogFinish(fillOperationId, False) + logger.error(f"Error in fillStructure: {str(e)}") + raise + + async def _generateSingleChapterSectionsStructure( + self, + chapter: Dict[str, Any], + chapterIndex: int, + chapterId: str, + chapterLevel: int, + chapterTitle: str, + generationHint: str, + contentPartIds: List[str], + contentPartInstructions: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + language: str, + outputFormat: str, + parentOperationId: str, + totalChapters: int + ) -> None: + """ + Generate sections structure for a single chapter (used for parallel processing). + Modifies chapter dict in place. + """ + try: + # Update progress for chapter structure generation + progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 + self.services.chat.progressLogUpdate( + parentOperationId, + progress, + f"Generating sections for Chapter {chapterIndex}/{totalChapters}: {chapterTitle}" + ) + + chapterPrompt = self._buildChapterSectionsStructurePrompt( + chapterId=chapterId, + chapterLevel=chapterLevel, + chapterTitle=chapterTitle, + generationHint=generationHint, + contentPartIds=contentPartIds, + contentPartInstructions=contentPartInstructions, + contentParts=contentParts, + userPrompt=userPrompt, + language=language, + outputFormat=outputFormat + ) + + # AI-Call für Chapter-Struktur-Generierung + # Note: Debug logging is handled by callAiPlanning + checkWorkflowStopped(self.services) + aiResponse = await self.aiService.callAiPlanning( + prompt=chapterPrompt, + debugType=f"chapter_structure_{chapterId}" + ) + + sectionsStructure = json.loads( + self.services.utils.jsonExtractString(aiResponse) + ) + + chapter["sections"] = sectionsStructure.get("sections", []) + + # Setze useAiCall Flag (falls nicht von AI gesetzt) + # WICHTIG: useAiCall kann nur true sein, wenn mindestens ein ContentPart Format "extracted" hat! + # "object" und "reference" Formate werden direkt als Elemente hinzugefügt, benötigen kein AI. + for section in chapter["sections"]: + if "useAiCall" not in section: + contentType = section.get("content_type", "paragraph") + sectionContentPartIds = section.get("contentPartIds", []) + + # Prüfe ob mindestens ein ContentPart Format "extracted" hat + hasExtractedPart = False + for partId in sectionContentPartIds: + part = self._findContentPartById(partId, contentParts) + if part: + contentFormat = part.metadata.get("contentFormat", "unknown") + if contentFormat == "extracted": + hasExtractedPart = True + break + + # useAiCall kann nur true sein, wenn extracted Parts vorhanden sind + useAiCall = False + if hasExtractedPart: + # Prüfe ob Transformation nötig ist + useAiCall = contentType != "paragraph" + + # Prüfe contentPartInstructions für Transformation + if not useAiCall: + for partId in sectionContentPartIds: + instruction = contentPartInstructions.get(partId, {}).get("instruction", "") + if instruction and instruction.lower() not in ["include full text", "include all content", "use full extracted text"]: + useAiCall = True + break + + section["useAiCall"] = useAiCall + logger.debug(f"Section {section.get('id')}: useAiCall={useAiCall} (hasExtractedPart={hasExtractedPart}, contentType={contentType})") + + # Update progress after chapter completion + progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 + self.services.chat.progressLogUpdate( + parentOperationId, + progress, + f"Chapter {chapterIndex}/{totalChapters} completed: {chapterTitle}" + ) + + except Exception as e: + logger.error(f"Error generating sections structure for chapter {chapterId}: {str(e)}") + # Set empty sections on error + chapter["sections"] = [] + # Update progress even on error + progress = chapterIndex / totalChapters if totalChapters > 0 else 1.0 + self.services.chat.progressLogUpdate( + parentOperationId, + progress, + f"Chapter {chapterIndex}/{totalChapters} error: {chapterTitle}" + ) + raise + + async def _generateChapterSectionsStructure( + self, + chapterStructure: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + parentOperationId: str, + language: str, + options: Optional[AiCallOptions] = None + ) -> Dict[str, Any]: + """ + Phase 5D.1: Generiert Sections-Struktur für jedes Chapter (ohne Content) in parallel. + Sections enthalten: content_type, contentPartIds, generationHint, useAiCall + """ + # Count total chapters for progress tracking + totalChapters = sum(len(doc.get("chapters", [])) for doc in chapterStructure.get("documents", [])) + + # Get concurrency limit + maxConcurrent = self._getMaxConcurrentGeneration(options) + semaphore = asyncio.Semaphore(maxConcurrent) + + # Collect all chapters with their indices for parallel processing + chapterTasks = [] + chapterIndex = 0 + + for doc in chapterStructure.get("documents", []): + docId = doc.get("id", "unknown") + # Get language for this specific document + docLanguage = self._getDocumentLanguage(chapterStructure, docId) + # Get output format for this specific document + docFormat = doc.get("outputFormat", "txt") + + for chapter in doc.get("chapters", []): + chapterIndex += 1 + chapterId = chapter.get("id", "unknown") + chapterLevel = chapter.get("level", 1) + chapterTitle = chapter.get("title", "Untitled Chapter") + generationHint = chapter.get("generationHint", "") + contentPartIds, contentPartInstructions = self._extractContentPartInfo(chapter) + + # Create task for parallel processing with semaphore + async def processChapterWithSemaphore(chapter, chapterIndex, chapterId, chapterLevel, chapterTitle, generationHint, contentPartIds, contentPartInstructions, docLanguage, docFormat): + checkWorkflowStopped(self.services) + async with semaphore: + return await self._generateSingleChapterSectionsStructure( + chapter=chapter, + chapterIndex=chapterIndex, + chapterId=chapterId, + chapterLevel=chapterLevel, + chapterTitle=chapterTitle, + generationHint=generationHint, + contentPartIds=contentPartIds, + contentPartInstructions=contentPartInstructions, + contentParts=contentParts, + userPrompt=userPrompt, + language=docLanguage, # Use document-specific language + outputFormat=docFormat, # Use document-specific format + parentOperationId=parentOperationId, + totalChapters=totalChapters + ) + + task = processChapterWithSemaphore( + chapter, chapterIndex, chapterId, chapterLevel, chapterTitle, generationHint, contentPartIds, contentPartInstructions, docLanguage, docFormat + ) + chapterTasks.append((chapterIndex, chapter, task)) + + # Execute all chapter tasks in parallel with concurrency control + if chapterTasks: + # Create list of tasks (without indices for gather) + tasks = [task for _, _, task in chapterTasks] + + # Execute in parallel with error handling + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Process results in order and handle errors + for (originalIndex, originalChapter, _), result in zip(chapterTasks, results): + if isinstance(result, Exception): + logger.error(f"Error processing chapter {originalChapter.get('id')}: {str(result)}") + # Chapter already has empty sections set by _generateSingleChapterSectionsStructure + # Continue processing other chapters + + return chapterStructure + + async def _processAiResponseForSection( + self, + aiResponse: Any, + contentType: str, + operationType: OperationTypeEnum, + sectionId: str, + generationHint: str, + generatedElements: List[Dict[str, Any]], + section: Dict[str, Any] + ) -> List[Dict[str, Any]]: + """ + Helper method to process AI response and extract elements. + Handles both IMAGE_GENERATE and DATA_ANALYSE operation types. + """ + elements = [] + + # Handle IMAGE_GENERATE differently - returns image data directly + if contentType == "image" and operationType == OperationTypeEnum.IMAGE_GENERATE: + import base64 + base64Data = "" + + # Convert image data to base64 string if needed + if isinstance(aiResponse.content, bytes): + base64Data = base64.b64encode(aiResponse.content).decode('utf-8') + elif isinstance(aiResponse.content, str): + # Check if it's already a JSON structure + try: + jsonContent = json.loads(self.services.utils.jsonExtractString(aiResponse.content)) + if isinstance(jsonContent, dict) and jsonContent.get("type") == "image": + elements.append(jsonContent) + logger.debug("AI returned proper JSON image structure") + base64Data = None # Signal that image was already processed + elif isinstance(jsonContent, list) and len(jsonContent) > 0: + if isinstance(jsonContent[0], dict) and jsonContent[0].get("type") == "image": + elements.extend(jsonContent) + logger.debug("AI returned proper JSON image structure in list") + base64Data = None # Signal that image was already processed + else: + base64Data = "" # Continue with normal processing + else: + base64Data = "" # Continue with normal processing + except (json.JSONDecodeError, ValueError, AttributeError): + base64Data = "" # Will be processed below + + # Process base64 if not already handled above + if base64Data is None: + # Already processed as JSON, skip base64 processing + pass + elif aiResponse.content.startswith("data:image/"): + # Extract base64 from data URI + base64Data = aiResponse.content.split(",", 1)[1] + else: + content_stripped = aiResponse.content.strip() + if len(content_stripped) > 100 and all(c in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r\t " for c in content_stripped[:200]): + base64Data = content_stripped.replace("\n", "").replace("\r", "").replace("\t", "").replace(" ", "") + else: + base64Data = aiResponse.content + else: + base64Data = "" + + # Always create proper JSON structure for images (if not already processed) + if base64Data is None: + # Image already processed as JSON, skip + pass + elif base64Data: + # Get caption from section if available + caption = section.get("caption") or section.get("metadata", {}).get("caption") or "" + elements.append({ + "type": "image", + "content": { + "base64Data": base64Data, + "altText": generationHint or "Generated image", + "caption": caption # Use caption from section if available + }, + "caption": caption # Also at element level for compatibility + }) + logger.debug(f"Created proper JSON image structure with base64Data length: {len(base64Data)}") + else: + logger.warning(f"IMAGE_GENERATE returned empty or invalid content for section {sectionId}") + elements.append({ + "type": "error", + "message": f"Image generation returned empty or invalid content", + "sectionId": sectionId + }) + else: + # For non-image content: Use already parsed elements from _callAiWithLooping + if generatedElements: + elements.extend(generatedElements) + else: + # Fallback: Try to parse JSON response directly with repair logic + try: + from modules.shared.jsonUtils import tryParseJson, repairBrokenJson + + # Use tryParseJson which handles extraction and basic parsing + fallbackElements, parseError, cleanedStr = tryParseJson(aiResponse.content) + + # If parsing failed, try repair + if parseError and isinstance(aiResponse.content, str): + logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}") + repairedJson = repairBrokenJson(aiResponse.content) + if repairedJson: + fallbackElements = repairedJson + parseError = None + logger.info(f"Successfully repaired JSON for section {sectionId}") + + if parseError: + raise parseError + + if isinstance(fallbackElements, list): + elements.extend(fallbackElements) + elif isinstance(fallbackElements, dict) and "elements" in fallbackElements: + elements.extend(fallbackElements["elements"]) + elif isinstance(fallbackElements, dict) and fallbackElements.get("type"): + elements.append(fallbackElements) + except (json.JSONDecodeError, ValueError) as json_error: + logger.error(f"Error parsing JSON response for section {sectionId}: {str(json_error)}") + elements.append({ + "type": "error", + "message": f"Failed to parse JSON response: {str(json_error)}", + "sectionId": sectionId + }) + + return elements + + async def _processSingleSection( + self, + section: Dict[str, Any], + sectionIndex: int, + totalSections: int, + chapterIndex: int, + totalChapters: int, + chapterId: str, + chapterOperationId: str, + fillOperationId: str, + contentParts: List[ContentPart], + userPrompt: str, + all_sections_list: List[Dict[str, Any]], + language: str, + outputFormat: str = "txt", + calculateOverallProgress: callable = None + ) -> List[Dict[str, Any]]: + """ + Process a single section and return its elements. + Used for parallel processing of sections within a chapter. + """ + sectionId = section.get("id") + sectionTitle = section.get("title", sectionId) + contentPartIds = section.get("contentPartIds", []) + contentFormats = section.get("contentFormats", {}) + generationHint = section.get("generationHint") or section.get("generation_hint") + contentType = section.get("content_type", "paragraph") + useAiCall = section.get("useAiCall", False) + + # Update overall progress at start of section + overallProgress = calculateOverallProgress(chapterIndex - 1, totalChapters, sectionIndex, totalSections) + self.services.chat.progressLogUpdate( + fillOperationId, + overallProgress, + f"Chapter {chapterIndex}/{totalChapters}, Section {sectionIndex + 1}/{totalSections}: {sectionTitle}" + ) + + # WICHTIG: Wenn keine ContentParts vorhanden sind UND kein generationHint, kann kein AI-Call gemacht werden + if len(contentPartIds) == 0 and not generationHint: + useAiCall = False + logger.debug(f"Section {sectionId}: No content parts and no generation hint, setting useAiCall=False") + elif len(contentPartIds) == 0 and generationHint and not useAiCall: + useAiCall = True + logger.info(f"Section {sectionId}: Overriding useAiCall=True (has generationHint but no content parts)") + + elements = [] + + # Prüfe ob Aggregation nötig ist + needsAggregation = self._needsAggregation( + contentType=contentType, + contentPartCount=len(contentPartIds) + ) + + logger.info(f"Processing section {sectionId}: contentType={contentType}, contentPartCount={len(contentPartIds)}, useAiCall={useAiCall}, needsAggregation={needsAggregation}, hasGenerationHint={bool(generationHint)}") + + try: + if needsAggregation and useAiCall: + # Aggregation: Alle Parts zusammen verarbeiten + sectionParts = [ + self._findContentPartById(pid, contentParts) + for pid in contentPartIds + ] + sectionParts = [p for p in sectionParts if p is not None] + + if sectionParts: + # Filtere nur extracted Parts für Aggregation (reference/object werden separat behandelt) + extractedParts = [ + p for p in sectionParts + if contentFormats.get(p.id, p.metadata.get("contentFormat")) == "extracted" + ] + nonExtractedParts = [ + p for p in sectionParts + if contentFormats.get(p.id, p.metadata.get("contentFormat")) != "extracted" + ] + + # Verarbeite non-extracted Parts separat (reference, object) + for part in nonExtractedParts: + contentFormat = contentFormats.get(part.id, part.metadata.get("contentFormat")) + + if contentFormat == "reference": + elements.append({ + "type": "reference", + "documentReference": part.metadata.get("documentReference"), + "label": part.metadata.get("usageHint", part.label) + }) + elif contentFormat == "object": + if part.typeGroup == "image": + # Validate that image data exists + if not part.data: + logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (object format). Skipping image element.") + elements.append({ + "type": "error", + "message": f"Image ContentPart {part.id} has no data", + "sectionId": sectionId + }) + else: + # Get caption from section (priority: section.caption > part.metadata.caption) + caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") + elements.append({ + "type": "image", + "content": { + "base64Data": part.data, + "altText": part.metadata.get("usageHint", part.label), + "caption": caption # Use caption from section + }, + "caption": caption # Also at element level for compatibility + }) + else: + elements.append({ + "type": part.typeGroup, + "content": { + "data": part.data, + "mimeType": part.mimeType, + "label": part.metadata.get("usageHint", part.label) + } + }) + + # Extract images with Vision AI if needed (before aggregation) + processedExtractedParts = [] + for part in extractedParts: + # Check if this is an image that needs Vision AI extraction + if (part.typeGroup == "image" and + part.metadata.get("needsVisionExtraction") == True and + part.metadata.get("intent") == "extract"): + + logger.info(f"Section {sectionId}: Extracting text from image {part.id} using Vision AI") + try: + extractionPrompt = part.metadata.get("extractionPrompt") or "Extract all text content from this image. Return only the extracted text, no additional formatting." + + # Write debug file for image extraction prompt + if self.services and hasattr(self.services, 'utils') and hasattr(self.services.utils, 'writeDebugFile'): + try: + partId = part.id[:8] if part.id else "unknown" + partLabelSafe = (part.label or "image").replace(" ", "_").replace("/", "_").replace("\\", "_")[:30] + debugPrefix = f"extraction_image_{partId}_{partLabelSafe}" + self.services.utils.writeDebugFile(extractionPrompt, f"{debugPrefix}_prompt") + logger.debug(f"Wrote image extraction prompt debug file: {debugPrefix}_prompt") + except Exception as debugError: + logger.warning(f"Failed to write image extraction debug file: {str(debugError)}") + + # Call Vision AI to extract text from image + visionRequest = AiCallRequest( + prompt=extractionPrompt, + context="", + options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), + contentParts=[part] + ) + + checkWorkflowStopped(self.services) + visionResponse = await self.aiService.callAi(visionRequest) + + # Write debug file for image extraction response + if self.services and hasattr(self.services, 'utils') and hasattr(self.services.utils, 'writeDebugFile'): + try: + partId = part.id[:8] if part.id else "unknown" + partLabelSafe = (part.label or "image").replace(" ", "_").replace("/", "_").replace("\\", "_")[:30] + debugPrefix = f"extraction_image_{partId}_{partLabelSafe}" + responseContent = visionResponse.content if visionResponse and visionResponse.content else "" + self.services.utils.writeDebugFile(responseContent, f"{debugPrefix}_response") + logger.debug(f"Wrote image extraction response debug file: {debugPrefix}_response") + except Exception as debugError: + logger.warning(f"Failed to write image extraction response debug file: {str(debugError)}") + + if visionResponse and visionResponse.content: + # Create text part with extracted content + textPart = ContentPart( + id=f"vision_extracted_{part.id}", + label=f"Extracted text from {part.label or 'Image'}", + typeGroup="text", + mimeType="text/plain", + data=visionResponse.content.strip(), + metadata={ + **part.metadata, + "contentFormat": "extracted", + "extractionMethod": "vision", + "sourceImagePartId": part.id, + "needsVisionExtraction": False # Already extracted + } + ) + processedExtractedParts.append(textPart) + logger.info(f"✅ Extracted text from image {part.id}: {len(visionResponse.content)} chars") + else: + logger.warning(f"⚠️ Vision AI extraction returned no content for image {part.id}") + # Keep original image part, but mark extraction as attempted + part.metadata["needsVisionExtraction"] = False + part.metadata["visionExtractionFailed"] = True + processedExtractedParts.append(part) + except Exception as e: + logger.error(f"❌ Vision AI extraction failed for image {part.id}: {str(e)}") + # Keep original image part, but mark extraction as attempted + part.metadata["needsVisionExtraction"] = False + part.metadata["visionExtractionFailed"] = True + processedExtractedParts.append(part) + else: + # Not an image needing extraction, or already processed + processedExtractedParts.append(part) + + # Aggregiere extracted Parts mit AI (now with Vision-extracted text parts) + if processedExtractedParts: + logger.debug(f"Section {sectionId}: Aggregating {len(processedExtractedParts)} extracted parts with AI") + isAggregation = True + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( + section=section, + contentParts=processedExtractedParts, + userPrompt=userPrompt, + generationHint=generationHint, + allSections=all_sections_list, + sectionIndex=sectionIndex, + isAggregation=isAggregation, + language=language, + outputFormat=outputFormat + ) + + sectionOperationId = f"{fillOperationId}_section_{sectionId}" + self.services.chat.progressLogStart( + sectionOperationId, + "Section Generation (Aggregation)", + f"Section {sectionIndex + 1}/{totalSections}", + f"{sectionTitle} ({len(extractedParts)} parts)", + parentOperationId=chapterOperationId + ) + + try: + self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") + + self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") + + operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE + + if operationType == OperationTypeEnum.IMAGE_GENERATE: + maxPromptLength = 4000 + if len(generationPrompt) > maxPromptLength: + logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") + generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] + + # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + generationPrompt, + f"{chapterId}_section_{sectionId}_prompt" + ) + + request = AiCallRequest( + prompt=generationPrompt, + contentParts=[], + options=AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + ) + checkWorkflowStopped(self.services) + aiResponse = await self.aiService.callAi(request) + generatedElements = [] + + # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), + f"{chapterId}_section_{sectionId}_response" + ) + else: + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation + + options = AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + + checkWorkflowStopped(self.services) + aiResponseJson = await self.aiService.callAiWithLooping( + prompt=generationPrompt, + options=options, + debugPrefix=f"{chapterId}_section_{sectionId}", + promptBuilder=buildSectionPromptWithContinuation, + promptArgs={ + "section": section, + "contentParts": extractedParts, + "userPrompt": userPrompt, + "generationHint": generationHint, + "allSections": all_sections_list, + "sectionIndex": sectionIndex, + "isAggregation": isAggregation, + "templateStructure": templateStructure, + "basePrompt": generationPrompt + }, + operationId=sectionOperationId, + userPrompt=userPrompt, + contentParts=extractedParts, + useCaseId="section_content" # REQUIRED: Explicit use case ID + ) + + try: + # Use tryParseJson which handles extraction and basic parsing + from modules.shared.jsonUtils import tryParseJson, repairBrokenJson + + # Check if response contains multiple JSON blocks (separated by --- or multiple ```json blocks) + # This can happen when AI returns multiple complete responses + if isinstance(aiResponseJson, str) and ("---" in aiResponseJson or aiResponseJson.count("```json") > 1): + logger.info(f"Section {sectionId}: Detected multiple JSON blocks in response, attempting to merge") + generatedElements = self._extractAndMergeMultipleJsonBlocks(aiResponseJson, contentType, sectionId) + else: + parsedResponse, parseError, cleanedStr = tryParseJson(aiResponseJson) + + # If parsing failed, try repair + if parseError and isinstance(aiResponseJson, str): + logger.warning(f"Initial JSON parse failed for section {sectionId}, attempting repair: {str(parseError)}") + repairedJson = repairBrokenJson(aiResponseJson) + if repairedJson: + parsedResponse = repairedJson + parseError = None + logger.info(f"Successfully repaired JSON for section {sectionId}") + + if parseError: + raise parseError + + if isinstance(parsedResponse, list): + generatedElements = parsedResponse + elif isinstance(parsedResponse, dict): + if "elements" in parsedResponse: + generatedElements = parsedResponse["elements"] + elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: + firstSection = parsedResponse["sections"][0] + generatedElements = firstSection.get("elements", []) + elif parsedResponse.get("type"): + generatedElements = [parsedResponse] + else: + generatedElements = [] + else: + generatedElements = [] + + class AiResponse: + def __init__(self, content): + self.content = content + + aiResponse = AiResponse(aiResponseJson) + except Exception as parseError: + logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") + class AiResponse: + def __init__(self, content): + self.content = content + aiResponse = AiResponse(aiResponseJson) + generatedElements = [] + + self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") + # Note: Debug files are written by _callAiWithLooping using debugPrefix + + self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") + + # Process AI response + responseElements = await self._processAiResponseForSection( + aiResponse=aiResponse, + contentType=contentType, + operationType=operationType, + sectionId=sectionId, + generationHint=generationHint, + generatedElements=generatedElements, + section=section + ) + elements.extend(responseElements) + + self.services.chat.progressLogFinish(sectionOperationId, True) + + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed" + ) + + except Exception as e: + self.services.chat.progressLogFinish(sectionOperationId, False) + elements.append({ + "type": "error", + "message": f"Error generating section {sectionId}: {str(e)}", + "sectionId": sectionId + }) + logger.error(f"Error generating section {sectionId}: {str(e)}") + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" + ) + + else: + # Einzelverarbeitung: Jeder Part einzeln ODER Generation ohne ContentParts + if len(contentPartIds) == 0 and useAiCall and generationHint: + # Generate content from scratch using only generationHint + logger.debug(f"Processing section {sectionId}: No content parts, generating from generationHint only") + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( + section=section, + contentParts=[], + userPrompt=userPrompt, + generationHint=generationHint, + allSections=all_sections_list, + sectionIndex=sectionIndex, + isAggregation=False, + language=language, + outputFormat=outputFormat + ) + + sectionOperationId = f"{fillOperationId}_section_{sectionId}" + self.services.chat.progressLogStart( + sectionOperationId, + "Section Generation", + f"Section {sectionIndex + 1}/{totalSections}", + f"{sectionTitle} (from generationHint)", + parentOperationId=chapterOperationId + ) + + try: + self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") + + self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") + + operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE + + if operationType == OperationTypeEnum.IMAGE_GENERATE: + maxPromptLength = 4000 + if len(generationPrompt) > maxPromptLength: + logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") + generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] + + # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + generationPrompt, + f"{chapterId}_section_{sectionId}_prompt" + ) + + request = AiCallRequest( + prompt=generationPrompt, + contentParts=[], + options=AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + ) + aiResponse = await self.aiService.callAi(request) + generatedElements = [] + + # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), + f"{chapterId}_section_{sectionId}_response" + ) + else: + isAggregation = False + + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation + + options = AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + + aiResponseJson = await self.aiService.callAiWithLooping( + prompt=generationPrompt, + options=options, + debugPrefix=f"{chapterId}_section_{sectionId}", + promptBuilder=self.buildSectionPromptWithContinuation, + promptArgs={ + "section": section, + "contentParts": [], + "userPrompt": userPrompt, + "generationHint": generationHint, + "allSections": all_sections_list, + "sectionIndex": sectionIndex, + "isAggregation": isAggregation, + "templateStructure": templateStructure, + "basePrompt": generationPrompt, + "language": language + }, + operationId=sectionOperationId, + userPrompt=userPrompt, + contentParts=[], + useCaseId="section_content" # REQUIRED: Explicit use case ID + ) + + try: + parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) + if isinstance(parsedResponse, list): + generatedElements = parsedResponse + elif isinstance(parsedResponse, dict): + if "elements" in parsedResponse: + generatedElements = parsedResponse["elements"] + elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: + firstSection = parsedResponse["sections"][0] + generatedElements = firstSection.get("elements", []) + elif parsedResponse.get("type"): + generatedElements = [parsedResponse] + else: + generatedElements = [] + else: + generatedElements = [] + + class AiResponse: + def __init__(self, content): + self.content = content + + aiResponse = AiResponse(aiResponseJson) + except Exception as parseError: + logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") + class AiResponse: + def __init__(self, content): + self.content = content + aiResponse = AiResponse(aiResponseJson) + generatedElements = [] + + self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") + # Note: Debug files are written by _callAiWithLooping using debugPrefix + + self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") + + responseElements = await self._processAiResponseForSection( + aiResponse=aiResponse, + contentType=contentType, + operationType=operationType, + sectionId=sectionId, + generationHint=generationHint, + generatedElements=generatedElements, + section=section + ) + elements.extend(responseElements) + + self.services.chat.progressLogFinish(sectionOperationId, True) + + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed" + ) + + except Exception as e: + self.services.chat.progressLogFinish(sectionOperationId, False) + elements.append({ + "type": "error", + "message": f"Error generating section {sectionId}: {str(e)}", + "sectionId": sectionId + }) + logger.error(f"Error generating section {sectionId}: {str(e)}") + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" + ) + + # Einzelverarbeitung: Jeder Part einzeln + for partId in contentPartIds: + part = self._findContentPartById(partId, contentParts) + if not part: + continue + + contentFormat = contentFormats.get(partId, part.metadata.get("contentFormat")) + + if contentFormat == "reference": + elements.append({ + "type": "reference", + "documentReference": part.metadata.get("documentReference"), + "label": part.metadata.get("usageHint", part.label) + }) + + elif contentFormat == "object": + if part.typeGroup == "image": + # Validate that image data exists + if not part.data: + logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (object format). Skipping image element.") + elements.append({ + "type": "error", + "message": f"Image ContentPart {part.id} has no data", + "sectionId": sectionId + }) + else: + # Get caption from section (priority: section.caption > part.metadata.caption) + caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") + elements.append({ + "type": "image", + "content": { + "base64Data": part.data, + "altText": part.metadata.get("usageHint", part.label), + "caption": caption # Use caption from section + }, + "caption": caption # Also at element level for compatibility + }) + else: + elements.append({ + "type": part.typeGroup, + "content": { + "data": part.data, + "mimeType": part.mimeType, + "label": part.metadata.get("usageHint", part.label) + } + }) + + elif contentFormat == "extracted": + # CRITICAL: If useAiCall is true, extracted parts are used as input for AI generation + # and should NOT be added as elements. Only add extracted text as element if useAiCall is false. + if useAiCall: + # Extracted part will be used as input for AI call - skip adding as element + logger.debug(f"Section {sectionId}: Skipping extracted part {part.id} as element (useAiCall=true, will be used as AI input)") + # Continue to process this part for AI call, but don't add as element yet + # Check if this is an image that needs Vision AI extraction + originalPartId = part.id + if (part.typeGroup == "image" and + part.metadata.get("needsVisionExtraction") == True and + part.metadata.get("intent") == "extract"): + + logger.info(f"Section {sectionId}: Extracting text from single image {part.id} using Vision AI") + try: + extractionPrompt = part.metadata.get("extractionPrompt") or "Extract all text content from this image. Return only the extracted text, no additional formatting." + + # Call Vision AI to extract text from image + visionRequest = AiCallRequest( + prompt=extractionPrompt, + context="", + options=AiCallOptions(operationType=OperationTypeEnum.IMAGE_ANALYSE), + contentParts=[part] + ) + + checkWorkflowStopped(self.services) + visionResponse = await self.aiService.callAi(visionRequest) + + if visionResponse and visionResponse.content: + # Replace image part with text part for further processing + part = ContentPart( + id=f"vision_extracted_{originalPartId}", + label=f"Extracted text from {part.label or 'Image'}", + typeGroup="text", + mimeType="text/plain", + data=visionResponse.content.strip(), + metadata={ + **part.metadata, + "contentFormat": "extracted", + "extractionMethod": "vision", + "sourceImagePartId": originalPartId, + "needsVisionExtraction": False # Already extracted + } + ) + logger.info(f"✅ Extracted text from image {originalPartId}: {len(visionResponse.content)} chars") + else: + logger.warning(f"⚠️ Vision AI extraction returned no content for image {originalPartId}") + part.metadata["needsVisionExtraction"] = False + part.metadata["visionExtractionFailed"] = True + except Exception as e: + logger.error(f"❌ Vision AI extraction failed for image {originalPartId}: {str(e)}") + part.metadata["needsVisionExtraction"] = False + part.metadata["visionExtractionFailed"] = True + + if useAiCall and generationHint: + # AI-Call mit einzelnen ContentPart (now may be text part after Vision extraction) + logger.debug(f"Processing section {sectionId}: Single extracted part with AI call") + generationPrompt, templateStructure = self._buildSectionGenerationPrompt( + section=section, + contentParts=[part], + userPrompt=userPrompt, + generationHint=generationHint, + allSections=all_sections_list, + sectionIndex=sectionIndex, + isAggregation=False, + language=language, + outputFormat=outputFormat + ) + + sectionOperationId = f"{fillOperationId}_section_{sectionId}" + self.services.chat.progressLogStart( + sectionOperationId, + "Section Generation", + f"Section {sectionIndex + 1}/{totalSections}", + f"{sectionTitle} (single part)", + parentOperationId=chapterOperationId + ) + + try: + self.services.chat.progressLogUpdate(sectionOperationId, 0.2, "Building generation prompt") + + self.services.chat.progressLogUpdate(sectionOperationId, 0.4, "Calling AI for content generation") + + operationType = OperationTypeEnum.IMAGE_GENERATE if contentType == "image" else OperationTypeEnum.DATA_ANALYSE + + if operationType == OperationTypeEnum.IMAGE_GENERATE: + maxPromptLength = 4000 + if len(generationPrompt) > maxPromptLength: + logger.warning(f"Truncating DALL-E prompt from {len(generationPrompt)} to {maxPromptLength} characters") + generationPrompt = generationPrompt[:maxPromptLength].rsplit('\n', 1)[0] + + # Write debug file for IMAGE_GENERATE (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + generationPrompt, + f"{chapterId}_section_{sectionId}_prompt" + ) + + request = AiCallRequest( + prompt=generationPrompt, + contentParts=[], + options=AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + ) + aiResponse = await self.aiService.callAi(request) + generatedElements = [] + + # Write debug file for IMAGE_GENERATE response (direct callAi, no _callAiWithLooping) + self.services.utils.writeDebugFile( + aiResponse.content if hasattr(aiResponse, 'content') else str(aiResponse), + f"{chapterId}_section_{sectionId}_response" + ) + else: + isAggregation = False + + # Use consolidated class method + buildSectionPromptWithContinuation = self.buildSectionPromptWithContinuation + + options = AiCallOptions( + operationType=operationType, + priority=PriorityEnum.BALANCED, + processingMode=ProcessingModeEnum.DETAILED + ) + + aiResponseJson = await self.aiService.callAiWithLooping( + prompt=generationPrompt, + options=options, + debugPrefix=f"{chapterId}_section_{sectionId}", + promptBuilder=self.buildSectionPromptWithContinuation, + promptArgs={ + "section": section, + "contentParts": [part], + "userPrompt": userPrompt, + "generationHint": generationHint, + "allSections": all_sections_list, + "sectionIndex": sectionIndex, + "isAggregation": isAggregation, + "services": self.services, + "templateStructure": templateStructure, + "basePrompt": generationPrompt, + "language": language + }, + operationId=sectionOperationId, + userPrompt=userPrompt, + contentParts=[part], + useCaseId="section_content" # REQUIRED: Explicit use case ID + ) + + try: + parsedResponse = json.loads(self.services.utils.jsonExtractString(aiResponseJson)) + if isinstance(parsedResponse, list): + generatedElements = parsedResponse + elif isinstance(parsedResponse, dict): + if "elements" in parsedResponse: + generatedElements = parsedResponse["elements"] + elif "sections" in parsedResponse and len(parsedResponse["sections"]) > 0: + firstSection = parsedResponse["sections"][0] + generatedElements = firstSection.get("elements", []) + elif parsedResponse.get("type"): + generatedElements = [parsedResponse] + else: + generatedElements = [] + else: + generatedElements = [] + + class AiResponse: + def __init__(self, content): + self.content = content + + aiResponse = AiResponse(aiResponseJson) + except Exception as parseError: + logger.error(f"Error parsing response from _callAiWithLooping for section {sectionId}: {str(parseError)}") + class AiResponse: + def __init__(self, content): + self.content = content + aiResponse = AiResponse(aiResponseJson) + generatedElements = [] + + self.services.chat.progressLogUpdate(sectionOperationId, 0.6, "Processing AI response") + # Note: Debug files are written by _callAiWithLooping using debugPrefix + + self.services.chat.progressLogUpdate(sectionOperationId, 0.8, "Validating generated content") + + responseElements = await self._processAiResponseForSection( + aiResponse=aiResponse, + contentType=contentType, + operationType=operationType, + sectionId=sectionId, + generationHint=generationHint, + generatedElements=generatedElements, + section=section + ) + elements.extend(responseElements) + + self.services.chat.progressLogFinish(sectionOperationId, True) + + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed" + ) + + except Exception as e: + self.services.chat.progressLogFinish(sectionOperationId, False) + elements.append({ + "type": "error", + "message": f"Error generating section {sectionId}: {str(e)}", + "sectionId": sectionId + }) + logger.error(f"Error generating section {sectionId}: {str(e)}") + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed (with errors)" + ) + else: + # Füge extrahierten Content direkt hinzu (kein AI-Call) + # CRITICAL: If content_type is "image", we must render an image, not extracted text + if contentType == "image": + # Section wants to display an image - find the image part + if part.typeGroup == "image": + # Direct image part - use it + logger.debug(f"Processing section {sectionId}: Single extracted IMAGE part WITHOUT AI call") + # Validate that image data exists + if not part.data: + logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (extracted format without AI call). Skipping image element.") + elements.append({ + "type": "error", + "message": f"Image ContentPart {part.id} has no data", + "sectionId": sectionId + }) + else: + # Get caption from section (priority: section.caption > part.metadata.caption) + caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") + elements.append({ + "type": "image", + "content": { + "base64Data": part.data, + "altText": part.metadata.get("usageHint", part.label), + "caption": caption # Use caption from section + }, + "caption": caption # Also at element level for compatibility + }) + elif part.typeGroup == "text" and part.metadata.get("sourceImagePartId"): + # This is a vision-extracted text part - find the original image object part + sourceImagePartId = part.metadata.get("sourceImagePartId") + logger.debug(f"Processing section {sectionId}: Found vision-extracted text part, looking for original image object part: {sourceImagePartId}") + + # Try to find the object part (format: "obj_...") + objectPartId = part.metadata.get("relatedObjectPartId") + objectPart = None + + if objectPartId: + objectPart = self._findContentPartById(objectPartId, contentParts) + + # If not found via metadata, search through all contentParts for object part + if not objectPart: + # Search for object part that references the source image part ID + for candidatePart in contentParts: + if (candidatePart.metadata.get("contentFormat") == "object" and + candidatePart.typeGroup == "image" and + sourceImagePartId in candidatePart.id): + objectPart = candidatePart + objectPartId = candidatePart.id + logger.debug(f"Section {sectionId}: Found object part {objectPartId} by searching all contentParts") + break + + if objectPart and objectPart.typeGroup == "image" and objectPart.data: + logger.info(f"Section {sectionId}: Found object part {objectPartId} for image rendering") + caption = section.get("caption") or section.get("metadata", {}).get("caption") or objectPart.metadata.get("caption", "") + elements.append({ + "type": "image", + "content": { + "base64Data": objectPart.data, + "altText": objectPart.metadata.get("usageHint", objectPart.label), + "caption": caption + }, + "caption": caption + }) + else: + logger.warning(f"Section {sectionId}: No object part found for vision-extracted text part {part.id} (sourceImagePartId={sourceImagePartId}), cannot render image") + elements.append({ + "type": "error", + "message": f"Cannot render image: no object part found for extracted text part (sourceImagePartId={sourceImagePartId})", + "sectionId": sectionId + }) + else: + logger.warning(f"Section {sectionId}: ContentPart {part.id} is not an image (typeGroup={part.typeGroup}), but section content_type is 'image'. Cannot render image.") + elements.append({ + "type": "error", + "message": f"Cannot render image: ContentPart is not an image type", + "sectionId": sectionId + }) + else: + # content_type is not "image" - add extracted text as normal + if part.typeGroup == "image": + logger.debug(f"Processing section {sectionId}: Single extracted IMAGE part WITHOUT AI call") + # Validate that image data exists + if not part.data: + logger.warning(f"Section {sectionId}: Image ContentPart {part.id} has no data (extracted format without AI call). Skipping image element.") + elements.append({ + "type": "error", + "message": f"Image ContentPart {part.id} has no data", + "sectionId": sectionId + }) + else: + # Get caption from section (priority: section.caption > part.metadata.caption) + caption = section.get("caption") or section.get("metadata", {}).get("caption") or part.metadata.get("caption", "") + elements.append({ + "type": "image", + "content": { + "base64Data": part.data, + "altText": part.metadata.get("usageHint", part.label), + "caption": caption # Use caption from section + }, + "caption": caption # Also at element level for compatibility + }) + else: + logger.debug(f"Processing section {sectionId}: Single extracted TEXT part WITHOUT AI call") + elements.append({ + "type": "extracted_text", + "content": part.data, + "source": part.metadata.get("documentId"), + "extractionPrompt": part.metadata.get("extractionPrompt") + }) + + # Update progress after section completion + chapterProgress = (sectionIndex + 1) / totalSections if totalSections > 0 else 1.0 + self.services.chat.progressLogUpdate( + chapterOperationId, + chapterProgress, + f"Section {sectionIndex + 1}/{totalSections} completed" + ) + + overallProgress = calculateOverallProgress(chapterIndex - 1, totalChapters, sectionIndex + 1, totalSections) + self.services.chat.progressLogUpdate( + fillOperationId, + overallProgress, + f"Chapter {chapterIndex}/{totalChapters}, Section {sectionIndex + 1}/{totalSections} completed" + ) + + except Exception as e: + logger.error(f"Unexpected error processing section {sectionId}: {str(e)}") + elements.append({ + "type": "error", + "message": f"Unexpected error processing section {sectionId}: {str(e)}", + "sectionId": sectionId + }) + + return elements + + async def _fillChapterSections( + self, + chapterStructure: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + parentOperationId: str, + language: str, + options: Optional[AiCallOptions] = None + ) -> Dict[str, Any]: + """ + Phase 5D.2: Füllt Sections mit ContentParts. + """ + + # Sammle alle Sections für Kontext-Informationen (für alle Sections) + all_sections_list = [] + for doc in chapterStructure.get("documents", []): + for chapter in doc.get("chapters", []): + for section in chapter.get("sections", []): + all_sections_list.append(section) + + # Berechne Gesamtanzahl Chapters für Progress-Tracking + totalChapters = sum(len(doc.get("chapters", [])) for doc in chapterStructure.get("documents", [])) + fillOperationId = parentOperationId + + # Get concurrency limit for sections + maxConcurrent = self._getMaxConcurrentGeneration(options) + sectionSemaphore = asyncio.Semaphore(maxConcurrent) + + # Collect ALL sections from ALL chapters for fully parallel processing + # Each task carries: (docId, chapterId, chapterTitle, sectionIndex, section, docLanguage) + allSectionTasks = [] + totalSections = len(all_sections_list) + completedSections = [0] # Mutable counter for progress tracking + + for doc in chapterStructure.get("documents", []): + docId = doc.get("id", "unknown") + docLanguage = self._getDocumentLanguage(chapterStructure, docId) + docFormat = doc.get("outputFormat", "txt") # Get output format for this document + + for chapter in doc.get("chapters", []): + chapterId = chapter.get("id", "unknown") + chapterTitle = chapter.get("title", "Untitled Chapter") + sections = chapter.get("sections", []) + chapterSectionCount = len(sections) + + for sectionIndex, section in enumerate(sections): + allSectionTasks.append({ + "docId": docId, + "chapterId": chapterId, + "chapterTitle": chapterTitle, + "sectionIndex": sectionIndex, + "chapterSectionCount": chapterSectionCount, + "section": section, + "docLanguage": docLanguage, + "docFormat": docFormat # Include output format + }) + + logger.info(f"Starting FULLY PARALLEL section generation: {totalSections} sections across {totalChapters} chapters") + + # Create task wrapper for each section with progress tracking + async def processSectionWithSemaphore(taskInfo): + checkWorkflowStopped(self.services) + async with sectionSemaphore: + result = await self._processSingleSection( + section=taskInfo["section"], + sectionIndex=taskInfo["sectionIndex"], + totalSections=taskInfo["chapterSectionCount"], + chapterIndex=0, # Not used for sequential logic anymore + totalChapters=totalChapters, + chapterId=taskInfo["chapterId"], + chapterOperationId=fillOperationId, # Use fillOperationId as parent (no chapter-level ops in parallel mode) + fillOperationId=fillOperationId, + contentParts=contentParts, + userPrompt=userPrompt, + all_sections_list=all_sections_list, + language=taskInfo["docLanguage"], + outputFormat=taskInfo.get("docFormat", "txt"), # Pass output format + calculateOverallProgress=lambda *args: completedSections[0] / totalSections if totalSections > 0 else 1.0 + ) + + # Update progress after each section completes + completedSections[0] += 1 + overallProgress = completedSections[0] / totalSections if totalSections > 0 else 1.0 + sectionId = taskInfo["section"].get("id", "unknown") + self.services.chat.progressLogUpdate( + fillOperationId, + overallProgress, + f"Section {completedSections[0]}/{totalSections} completed: {sectionId}" + ) + + return result + + # Create all tasks + tasks = [processSectionWithSemaphore(taskInfo) for taskInfo in allSectionTasks] + + # Execute ALL sections in parallel with concurrency control + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Assign results back to sections + for taskInfo, result in zip(allSectionTasks, results): + section = taskInfo["section"] + if isinstance(result, Exception): + logger.error(f"Error processing section {section.get('id')}: {str(result)}") + section["elements"] = [{ + "type": "error", + "message": f"Error processing section: {str(result)}", + "sectionId": section.get("id") + }] + else: + section["elements"] = result if result is not None else [] + + logger.info(f"Completed FULLY PARALLEL section generation: {totalSections} sections") + + return chapterStructure + + def _addContentPartsMetadata( + self, + structure: Dict[str, Any], + contentParts: List[ContentPart] + ) -> Dict[str, Any]: + """ + Fügt ContentParts-Metadaten zur Struktur hinzu, wenn contentPartIds vorhanden sind. + Dies hilft der Validierung, den Kontext der ContentParts zu verstehen. + """ + # Erstelle Mapping von ContentPart-ID zu Metadaten + contentPartsMap = {} + for part in contentParts: + contentPartsMap[part.id] = { + "id": part.id, + "format": part.metadata.get("contentFormat", "unknown"), + "type": part.typeGroup, + "mimeType": part.mimeType, + "originalFileName": part.metadata.get("originalFileName"), + "usageHint": part.metadata.get("usageHint"), + "documentId": part.metadata.get("documentId"), + "dataSize": len(str(part.data)) if part.data else 0 + } + + # Füge Metadaten zu Sections hinzu, die contentPartIds haben + for doc in structure.get("documents", []): + # Prüfe ob Chapters vorhanden sind (neue Struktur) + if "chapters" in doc: + for chapter in doc.get("chapters", []): + # Füge Metadaten zu Chapter-Level contentPartIds hinzu + chapterContentPartIds, _ = self._extractContentPartInfo(chapter) + if chapterContentPartIds: + chapter["contentPartsMetadata"] = [] + for partId in chapterContentPartIds: + if partId in contentPartsMap: + chapter["contentPartsMetadata"].append(contentPartsMap[partId]) + + # Füge Metadaten zu Sections hinzu + for section in chapter.get("sections", []): + contentPartIds = section.get("contentPartIds", []) + if contentPartIds: + section["contentPartsMetadata"] = [] + for partId in contentPartIds: + if partId in contentPartsMap: + section["contentPartsMetadata"].append(contentPartsMap[partId]) + + return structure + + def _flattenChaptersToSections( + self, + chapterStructure: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Flattening: Konvertiert Chapters zu finaler Section-Struktur. + Jedes Chapter wird zu einer Heading-Section (Level 1) + dessen Sections. + + Chapters are the main structure elements (heading level 1). + All section headings with level < 2 are adjusted to level 2. + """ + result = { + "metadata": chapterStructure.get("metadata", {}), + "documents": [] + } + + for doc in chapterStructure.get("documents", []): + flattened_doc = { + "id": doc.get("id"), + "title": doc.get("title"), + "filename": doc.get("filename"), + "outputFormat": doc.get("outputFormat"), # Preserve from Phase 3 + "language": doc.get("language"), # Preserve from Phase 3 + "sections": [] + } + + for chapter in doc.get("chapters", []): + # 1. Vordefinierte Heading-Section für Chapter-Title (ALWAYS Level 1) + heading_section = { + "id": f"{chapter['id']}_heading", + "content_type": "heading", + "elements": [{ + "type": "heading", + "content": { + "text": chapter.get("title", ""), + "level": 1 # Chapters are always level 1 + } + }] + } + flattened_doc["sections"].append(heading_section) + + # 2. Generierte Sections - adjust heading levels + for section in chapter.get("sections", []): + # CRITICAL: Ensure elements are preserved when flattening + # _adjustSectionHeadingLevels uses deepcopy which should preserve elements, + # but verify that elements exist in the source section + adjusted_section = self._adjustSectionHeadingLevels(section) + # Ensure elements are preserved (deepcopy should handle this, but double-check) + if "elements" in section and "elements" not in adjusted_section: + adjusted_section["elements"] = section["elements"] + flattened_doc["sections"].append(adjusted_section) + + result["documents"].append(flattened_doc) + + return result + + def _adjustSectionHeadingLevels(self, section: Dict[str, Any]) -> Dict[str, Any]: + """ + Adjust heading levels in sections: sections with type heading and level < 2 are changed to level 2. + Only chapter headings have level 1. + """ + adjusted_section = copy.deepcopy(section) + + # Check if this is a heading section + if adjusted_section.get("content_type") == "heading": + elements = adjusted_section.get("elements", []) + for element in elements: + if isinstance(element, dict) and element.get("type") == "heading": + content = element.get("content", {}) + if isinstance(content, dict): + level = content.get("level", 1) + # If level < 2, change to level 2 (only chapters have level 1) + if level < 2: + content["level"] = 2 + + return adjusted_section + + def _buildChapterSectionsStructurePrompt( + self, + chapterId: str, + chapterLevel: int, + chapterTitle: str, + generationHint: str, + contentPartIds: List[str], + contentPartInstructions: Dict[str, Any], + contentParts: List[ContentPart], + userPrompt: str, + language: str = "en", + outputFormat: str = "txt" + ) -> str: + """Baue Prompt für Chapter-Sections-Struktur-Generierung, querying renderer for accepted section types.""" + # Baue ContentParts-Index (nur IDs, keine Previews!) + contentPartsIndex = "" + for partId in contentPartIds: + part = self._findContentPartById(partId, contentParts) + if not part: + # Part not found - try to show info from chapter structure + partInfo = contentPartInstructions.get(partId, {}) + if partInfo: + logger.warning(f"Chapter {chapterId}: ContentPart {partId} not found in contentParts list, but has chapter structure info.") + contentPartsIndex += f"\n- ContentPart ID: {partId}\n" + if "instruction" in partInfo: + contentPartsIndex += f" Instruction: {partInfo['instruction']}\n" + if "caption" in partInfo: + contentPartsIndex += f" Caption: {partInfo['caption']}\n" + contentPartsIndex += f" Note: ContentPart not found in contentParts list (ID may be from nested structure)\n" + continue + + contentFormat = part.metadata.get("contentFormat", "unknown") + partInfo = contentPartInstructions.get(partId, {}) + instruction = partInfo.get("instruction", "Use content as needed") + caption = partInfo.get("caption") + + contentPartsIndex += f"\n- ContentPart ID: {partId}\n" + contentPartsIndex += f" Format: {contentFormat}\n" + contentPartsIndex += f" Type: {part.typeGroup}\n" + if instruction and instruction != "Use content as needed": + contentPartsIndex += f" Instruction: {instruction}\n" + if caption: + contentPartsIndex += f" Caption: {caption}\n" + + if not contentPartsIndex: + contentPartsIndex = "\n(No content parts specified for this chapter)" + + # Query renderer for accepted section types + acceptedSectionTypes = self._getAcceptedSectionTypesForFormat(outputFormat) + + prompt = f"""TASK: Generate Chapter Sections Structure + +LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. + +CHAPTER: {chapterTitle} (Level {chapterLevel}, ID: {chapterId}) +GENERATION HINT: {generationHint} + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT sections: Focus on essential information only +- AVOID creating too many sections - combine related content where possible +- Each section should serve a clear purpose with meaningful data +- If no relevant data exists for a topic, do NOT create a section for it +- Prefer ONE comprehensive section over multiple sparse sections + +**CRITICAL**: The chapter's generationHint above describes what content this chapter should generate. If the generationHint references documents/images/data, then EACH section that generates content for this chapter MUST assign the relevant ContentParts from AVAILABLE CONTENT PARTS below. + +NOTE: Chapter already has a heading section. Do NOT generate a heading for the chapter title. + +## SECTION INDEPENDENCE +- Each section is independent and self-contained +- One section does NOT have information about another section +- Each section must provide its own context and be understandable alone + +AVAILABLE CONTENT PARTS: +{contentPartsIndex} + +## CONTENT ASSIGNMENT RULE - CRITICAL +If AVAILABLE CONTENT PARTS are listed above, then EVERY section that generates content related to those ContentParts MUST assign them explicitly. + +**Assignment logic:** +- If section generates text content ABOUT a ContentPart → assign "extracted" format ContentPart with appropriate instruction +- If section DISPLAYS a ContentPart → assign "object" format ContentPart +- If section's generationHint or purpose relates to a ContentPart listed above → it MUST have contentPartIds assigned +- If chapter's generationHint references documents/images/data AND section generates content for that chapter → section MUST assign relevant ContentParts +- Empty contentPartIds [] are only allowed if section generates content WITHOUT referencing any available ContentParts AND WITHOUT relating to chapter's generationHint + +## ACCEPTED CONTENT TYPES FOR THIS FORMAT +The document output format ({outputFormat}) accepts only the following content types: +{', '.join(acceptedSectionTypes)} + +**CRITICAL**: Only create sections with content types from this list. Other types will fail. + +useAiCall RULE (simple): +- useAiCall: true → Content needs AI processing (extract, transform, generate, filter, summarize) +- useAiCall: false → Content can be inserted directly without changes (Format is "object" or "reference") + +RETURN JSON: +{{ + "sections": [ + {{ + "id": "section_1", + "content_type": "{acceptedSectionTypes[0]}", + "contentPartIds": ["extracted_part_id"], + "generationHint": "Description of what to extract or generate", + "useAiCall": true, + "elements": [] + }} + ] +}} + +**MANDATORY CONTENT ASSIGNMENT CHECK:** +For each section, verify: +1. Are ContentParts listed in AVAILABLE CONTENT PARTS above? +2. Does this section's generationHint or purpose relate to those ContentParts? +3. If YES to both → section MUST have contentPartIds assigned (cannot be empty []) +4. Assign ContentPart IDs exactly as listed in AVAILABLE CONTENT PARTS above + +IMAGE SECTIONS: +- For image sections, always provide a "caption" field with a descriptive caption for the image. + +Return only valid JSON. Do not include any explanatory text outside the JSON. +""" + return prompt + + def _getContentStructureExample(self, contentType: str) -> str: + """Get the JSON structure example for a specific content type.""" + structures = { + "table": '{{"headers": ["Column1", "Column2"], "rows": [["Value1", "Value2"], ["Value3", "Value4"]]}}', + "bullet_list": '{{"items": ["Item 1", "Item 2", "Item 3"]}}', + "heading": '{{"text": "Section Title", "level": 2}}', + "paragraph": '{{"text": "This is paragraph text."}}', + "code_block": '{{"code": "function example() {{ return true; }}", "language": "javascript"}}', + "image": '{{"base64Data": "", "altText": "Description", "caption": "Optional caption"}}' + } + return structures.get(contentType, '{{"text": ""}}') + + def _buildSectionGenerationPrompt( + self, + section: Dict[str, Any], + contentParts: List[Optional[ContentPart]], + userPrompt: str, + generationHint: str, + allSections: Optional[List[Dict[str, Any]]] = None, + sectionIndex: Optional[int] = None, + isAggregation: bool = False, + language: str = "en", + outputFormat: str = "txt" + ) -> tuple[str, str]: + """Baue Prompt für Section-Generierung mit vollständigem Kontext.""" + # Filtere None-Werte + validParts = [p for p in contentParts if p is not None] + + # Section-Metadaten + sectionId = section.get("id", "unknown") + contentType = section.get("content_type", "paragraph") + + # Baue ContentParts-Beschreibung + contentPartsText = "" + if isAggregation: + # Aggregation: ContentParts werden als Parameter übergeben, keine IDs im Prompt nötig + # Keine ContentPart-Beschreibung nötig - Daten sind bereits im Context verfügbar + contentPartsText = "" + else: + # Einzelverarbeitung: Zeige Previews + for part in validParts: + contentFormat = part.metadata.get("contentFormat", "unknown") + contentPartsText += f"\n- ContentPart {part.id}:\n" + contentPartsText += f" Format: {contentFormat}\n" + contentPartsText += f" Type: {part.typeGroup}\n" + if part.metadata.get("originalFileName"): + contentPartsText += f" Source file: {part.metadata.get('originalFileName')}\n" + + if contentFormat == "extracted": + # CRITICAL: Check if this is binary/image data - NEVER include in text prompt! + isBinaryOrImage = ( + part.typeGroup == "image" or + part.typeGroup == "binary" or + (part.mimeType and ( + part.mimeType.startswith("image/") or + part.mimeType.startswith("video/") or + part.mimeType.startswith("audio/") or + self._isBinaryMimeType(part.mimeType) + )) or + # Heuristic check: if data looks like base64 (long string with base64 chars) + (part.data and isinstance(part.data, str) and + len(part.data) > 100 and + self._looksLikeBase64(part.data)) + ) + + if isBinaryOrImage: + # NEVER include binary/base64 data in text prompt - security risk and token explosion! + dataLength = len(part.data) if part.data else 0 + contentPartsText += f" Type: {part.typeGroup}\n" + contentPartsText += f" MIME type: {part.mimeType or 'unknown'}\n" + contentPartsText += f" Data size: {dataLength} chars (binary/base64 - not shown in prompt)\n" + if part.metadata.get("needsVisionExtraction"): + contentPartsText += f" Note: Will be processed with Vision AI\n" + if part.metadata.get("usageHint"): + contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" + else: + # Only for text data: Show preview + previewLength = 1000 + if part.data: + preview = part.data[:previewLength] + "..." if len(part.data) > previewLength else part.data + contentPartsText += f" Content preview:\n```\n{preview}\n```\n" + else: + contentPartsText += f" Content: (empty)\n" + elif contentFormat == "reference": + contentPartsText += f" Reference: {part.metadata.get('documentReference')}\n" + if part.metadata.get("usageHint"): + contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" + elif contentFormat == "object": + dataLength = len(part.data) if part.data else 0 + contentPartsText += f" Object type: {part.typeGroup}\n" + contentPartsText += f" MIME type: {part.mimeType}\n" + contentPartsText += f" Data size: {dataLength} chars (base64 encoded)\n" + if part.metadata.get("usageHint"): + contentPartsText += f" Usage hint: {part.metadata.get('usageHint')}\n" + + # Baue Section-Kontext (vorherige und nachfolgende Sections) + contextText = "" + if allSections and sectionIndex is not None: + prevSections = [] + nextSections = [] + + if sectionIndex > 0: + for i in range(max(0, sectionIndex - 2), sectionIndex): + prevSection = allSections[i] + prevSections.append({ + "id": prevSection.get("id"), + "content_type": prevSection.get("content_type"), + "generation_hint": prevSection.get("generation_hint", "")[:100] + }) + + if sectionIndex < len(allSections) - 1: + for i in range(sectionIndex + 1, min(len(allSections), sectionIndex + 3)): + nextSection = allSections[i] + nextSections.append({ + "id": nextSection.get("id"), + "content_type": nextSection.get("content_type"), + "generation_hint": nextSection.get("generation_hint", "")[:100] + }) + + if prevSections or nextSections: + contextText = "\n## DOCUMENT CONTEXT\n" + if prevSections: + contextText += "\nPrevious sections:\n" + for prev in prevSections: + contextText += f"- {prev['id']} ({prev['content_type']}): {prev['generation_hint']}\n" + if nextSections: + contextText += "\nFollowing sections:\n" + for next in nextSections: + contextText += f"- {next['id']} ({next['content_type']}): {next['generation_hint']}\n" + + # Get accepted section types for the output format + acceptedTypesAggr = self._getAcceptedSectionTypesForFormat(outputFormat) + + # CRITICAL: If the section's content_type is not supported by the output format, + # use the first accepted type instead. E.g., CSV only supports 'table', so + # even if section says 'code_block', we must output as 'table'. + effectiveContentType = contentType + if contentType not in acceptedTypesAggr and acceptedTypesAggr: + effectiveContentType = acceptedTypesAggr[0] + logger.debug(f"Section {sectionId}: Content type '{contentType}' not supported by format '{outputFormat}', using '{effectiveContentType}' instead") + + contentStructureExample = self._getContentStructureExample(effectiveContentType) + + # Build format note for the prompt - purely dynamic from renderer + # Always show what types are accepted for this format + formatNoteAggr = f"\n- Target Output Format: {outputFormat.upper()} (accepted content types: {', '.join(acceptedTypesAggr)})" + + # Create template structure explicitly (not extracted from prompt) + # This ensures exact identity between initial and continuation prompts + templateStructure = f"""{{ + "elements": [ + {{ + "type": "{effectiveContentType}", + "content": {contentStructureExample} + }} + ] +}}""" + + if isAggregation: + prompt = f"""# TASK: Generate Section Content (Aggregation) + +Return only valid JSON. No explanatory text, no comments, no markdown formatting outside JSON. +If ContentParts have no data, return: {{"elements": [{{"type": "{effectiveContentType}", "content": {{"headers": [], "rows": []}}}}]}} + +LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. + +## SECTION METADATA +- Section ID: {sectionId} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + +## INSTRUCTIONS +1. Extract all data from the context provided. Do not skip or omit any data. +2. Extract data only from the provided context. Never invent, create, or generate data that is not in the context. +3. If the context contains no data, return empty structures (empty rows array for tables). +4. Aggregate all data into one element (e.g., one table). +5. For table: Extract all rows from the context. Return {{"headers": [...], "rows": []}} only if no data exists. +6. Format based on content_type ({effectiveContentType}). +7. No HTML/styling: Plain text only, no markup. +8. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. + + +## OUTPUT FORMAT +Return a JSON object with this structure: + +{{ + "elements": [ + {{ + "type": "{effectiveContentType}", + "content": {contentStructureExample} + }} + ] +}} + +Output requirements: +- "content" must be an object (never a string) +- Return only valid JSON - no text before, no text after, no comments, no explanations +- No invented data: Return empty structures if ContentParts have no data +- Extract all data: Process every ContentPart completely and include all extracted data + +## USER REQUEST (for context) +``` +{userPrompt} +``` + +## CONTEXT +{contextText if contextText else ""} +""" + else: + # Determine if we have ContentParts or need to generate from scratch + hasContentParts = len(validParts) > 0 + + if hasContentParts: + # EXTRACT MODE: Extract data from provided ContentParts + prompt = f"""# TASK: Extract Section Content from Provided Data + +LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. + +## SECTION METADATA +- Section ID: {sectionId} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + +## AVAILABLE CONTENT FOR THIS SECTION +{contentPartsText} + +## INSTRUCTIONS +1. Extract data only from provided ContentParts. Never invent or generate data. +2. If ContentParts contain no data, return empty structures (empty rows array for tables). +3. Format based on content_type ({effectiveContentType}). +4. Return only valid JSON with "elements" array. +5. No HTML/styling: Plain text only, no markup. +6. CONTINUE UNTIL COMPLETE: Extract ALL data from the provided context. Do NOT stop early because you think the response might be too long. Do NOT truncate or abbreviate. Do not impose artificial limits on yourself. + +## OUTPUT FORMAT +Return a JSON object with this structure: + +{{ + "elements": [ + {{ + "type": "{effectiveContentType}", + "content": {contentStructureExample} + }} + ] +}} + +Output requirements: +- "content" must be an object (never a string) +- Return only valid JSON - no text before, no text after, no comments, no explanations, no markdown code fences +- Start with {{ and end with }} - return ONLY the JSON object itself +- No invented data: Return empty structures if ContentParts have no data + +## USER REQUEST +``` +{userPrompt} +``` + +## CONTEXT +{contextText if contextText else ""} +""" + else: + # GENERATE MODE: Generate content from scratch based on generationHint + prompt = f"""# TASK: Generate Section Content + +LANGUAGE: Generate all content in {language.upper()} language. All text, titles, headings, paragraphs, and content must be written in {language.upper()}. + +## SECTION METADATA +- Section ID: {sectionId} +- Content Type: {effectiveContentType} +- Generation Hint: {generationHint}{formatNoteAggr} + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential facts only +- AVOID verbose text, filler phrases, or redundant explanations +- Be CONCISE and direct - every word should add value +- NO introductory phrases like "This section describes..." or "Here we present..." +- Minimize output size for efficient processing + +## INSTRUCTIONS +1. Generate content based on the Generation Hint above. +2. Create appropriate content that matches the content_type ({effectiveContentType}). +3. The content should be relevant to the USER REQUEST and fit the context of surrounding sections. +4. Return only valid JSON with "elements" array. +5. No HTML/styling: Plain text only, no markup. +6. Keep content CONCISE - focus on substance, not length. + +## OUTPUT FORMAT +Return a JSON object with this structure: + +{{ + "elements": [ + {{ + "type": "{effectiveContentType}", + "content": {contentStructureExample} + }} + ] +}} + +Output requirements: +- "content" must be an object (never a string) +- Return only valid JSON - no text before, no text after, no comments, no explanations, no markdown code fences +- Start with {{ and end with }} - return ONLY the JSON object itself +- Generate meaningful content based on the Generation Hint + +## USER REQUEST +``` +{userPrompt} +``` + +## CONTEXT +{contextText if contextText else ""} +""" + return prompt, templateStructure + + async def buildSectionPromptWithContinuation( + self, + continuationContext: Any, + templateStructure: str, + basePrompt: str + ) -> str: + """Build section prompt with continuation context. Uses unified signature. + + Single unified implementation for all section content generation contexts. + + Note: All initial context (section, contentParts, userPrompt, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json + + # Generate both overlap context and hierarchy context using jsonContinuation + overlapContext = "" + unifiedContext = "" + if lastRawJson: + # Get contexts directly from jsonContinuation + from modules.shared.jsonContinuation import getContexts + contexts = getContexts(lastRawJson) + overlapContext = contexts.overlapContext + unifiedContext = contexts.hierarchyContextForPrompt + elif incompletePart: + unifiedContext = incompletePart + else: + unifiedContext = "Unable to extract context - response was completely broken" + + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} + +--- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. + +Context showing structure hierarchy with cut point: +``` +{unifiedContext} +``` + +Overlap Requirement: +To ensure proper merging, your response MUST start EXACTLY with the overlap context shown below, then continue with new content. + +Overlap context (start your response with this exact text): +```json +{overlapContext if overlapContext else "No overlap context available"} +``` + +TASK: +1. Start your response EXACTLY with the overlap context shown above (character by character) +2. Continue seamlessly from where the overlap context ends +3. Complete the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response MUST begin with the exact overlap context text (this enables automatic merging) +- Continue seamlessly after the overlap context with new content +- Your response must be valid JSON matching the structure template above""" + return continuationPrompt + + def _extractAndMergeMultipleJsonBlocks(self, responseText: str, contentType: str, sectionId: str) -> List[Dict[str, Any]]: + """ + Extract multiple JSON blocks from response and merge them appropriately. + For tables: Merge all rows into a single table. + For other types: Combine elements. + """ + from modules.shared.jsonUtils import tryParseJson, stripCodeFences, normalizeJsonText, extractFirstBalancedJson + + # Extract all JSON blocks, handling both --- separators and multiple ```json blocks + blocks = [] + + # Strategy: Extract all ```json blocks first (most reliable), then fall back to other methods + # This handles cases where --- separators and ```json blocks are mixed + if "```json" in responseText: + # Extract all ```json blocks regardless of --- separators + jsonParts = responseText.split("```json") + for jsonPart in jsonParts[1:]: # Skip first empty part + jsonPart = "```json" + jsonPart + # Extract just the JSON block (until closing ```) + closingFence = jsonPart.find("```", 7) # Find closing ``` after "```json" + if closingFence != -1: + jsonPart = jsonPart[:closingFence + 3] + jsonPart = jsonPart.strip() + if jsonPart: + blocks.append(jsonPart) + + # If no ```json blocks found, try splitting by --- and extracting JSON + if not blocks and "---" in responseText: + parts = responseText.split("---") + for part in parts: + part = part.strip() + if not part: + continue + + # Try to extract JSON directly from this part + normalized = normalizeJsonText(part) + normalized = stripCodeFences(normalized) + jsonBlock = extractFirstBalancedJson(normalized) + if jsonBlock: + blocks.append(jsonBlock) + elif responseText.count("```json") > 1: + # Split by ```json markers (no --- separator) + parts = responseText.split("```json") + for part in parts[1:]: # Skip first empty part + part = "```json" + part + part = part.strip() + if part: + blocks.append(part) + else: + # Try to find multiple JSON objects/arrays directly + normalized = normalizeJsonText(responseText) + normalized = stripCodeFences(normalized) + + # Find all JSON blocks + start = 0 + while start < len(normalized): + # Find next JSON start + brace = normalized.find('{', start) + bracket = normalized.find('[', start) + jsonStart = -1 + if brace != -1 and (bracket == -1 or brace < bracket): + jsonStart = brace + elif bracket != -1: + jsonStart = bracket + + if jsonStart == -1: + break + + # Extract balanced JSON + jsonBlock = extractFirstBalancedJson(normalized[jsonStart:]) + if jsonBlock: + blocks.append(jsonBlock) + start = jsonStart + len(jsonBlock) + else: + break + + if not blocks: + logger.warning(f"Section {sectionId}: Could not extract multiple JSON blocks") + return [] + + logger.info(f"Section {sectionId}: Extracted {len(blocks)} JSON blocks, merging for contentType={contentType}") + + # Parse all blocks + allElements = [] + for i, block in enumerate(blocks): + parsed, parseError, _ = tryParseJson(block) + if parseError: + logger.warning(f"Section {sectionId}: Failed to parse JSON block {i+1}: {str(parseError)}") + continue + + elementsFromBlock = [] + if isinstance(parsed, dict): + if "elements" in parsed: + elementsFromBlock = parsed["elements"] + allElements.extend(elementsFromBlock) + elif parsed.get("type"): + elementsFromBlock = [parsed] + allElements.append(parsed) + elif isinstance(parsed, list): + elementsFromBlock = parsed + allElements.extend(parsed) + + # Log row count for table elements + if contentType == "table": + tableCount = sum(1 for e in elementsFromBlock if isinstance(e, dict) and e.get("type") == "table") + rowCount = sum( + len(e.get("content", {}).get("rows", [])) + for e in elementsFromBlock + if isinstance(e, dict) and e.get("type") == "table" + ) + if tableCount > 0: + logger.info(f"Section {sectionId}: JSON block {i+1}: {tableCount} table(s) with {rowCount} total rows") + + # Merge elements based on contentType + if contentType == "table" and len(allElements) > 1: + # Find all table elements + tableElements = [e for e in allElements if isinstance(e, dict) and e.get("type") == "table"] + if len(tableElements) > 1: + # Check if tables can be merged (same column counts) + canMerge = self._canMergeTables(tableElements) + if canMerge: + logger.info(f"Section {sectionId}: Merging {len(tableElements)} tables into one") + mergedTable = self._mergeTableElements(tableElements) + # Replace all table elements with merged one + nonTableElements = [e for e in allElements if not (isinstance(e, dict) and e.get("type") == "table")] + return [mergedTable] + nonTableElements + else: + logger.warning(f"Section {sectionId}: Cannot merge {len(tableElements)} tables (incompatible headers/columns). Keeping tables separate.") + # Return all elements as-is (tables remain separate) + return allElements + + return allElements + + def _canMergeTables(self, tableElements: List[Dict[str, Any]]) -> bool: + """Check if tables can be safely merged (same column counts).""" + if len(tableElements) <= 1: + return True + + # Extract column counts from all tables + columnCounts = [] + for table in tableElements: + headers = [] + if isinstance(table.get("content"), dict): + headers = table["content"].get("headers", []) + elif isinstance(table.get("content"), list): + # Old format: content is list of rows + if table["content"] and isinstance(table["content"][0], list): + headers = table["content"][0] + columnCounts.append(len(headers)) + + # Check if all tables have the same column count + firstCount = columnCounts[0] if columnCounts else 0 + return all(count == firstCount for count in columnCounts) + + def _mergeTableElements(self, tableElements: List[Dict[str, Any]]) -> Dict[str, Any]: + """Merge multiple table elements into a single table. + Assumes tables have compatible column counts (checked by _canMergeTables). + """ + if not tableElements: + return {"type": "table", "content": {"headers": [], "rows": []}} + + if len(tableElements) == 1: + return tableElements[0] + + # Extract headers from all tables + allHeaders = [] + for table in tableElements: + headers = [] + if isinstance(table.get("content"), dict): + headers = table["content"].get("headers", []) + elif isinstance(table.get("content"), list): + # Old format: content is list of rows + if table["content"] and isinstance(table["content"][0], list): + headers = table["content"][0] + allHeaders.append(headers) + + # Check header compatibility (same headers or just same column count) + firstHeaders = allHeaders[0] + headersCompatible = all(headers == firstHeaders for headers in allHeaders) + + # If headers differ but column counts match, use first table's headers and log warning + if not headersCompatible: + logger.warning(f"Merging {len(tableElements)} tables with different headers but same column count. Using headers from first table.") + + # Use headers from first table + headers = firstHeaders + + # Collect all rows from all tables, validating column count + allRows = [] + for tableIdx, table in enumerate(tableElements): + rows = [] + if isinstance(table.get("content"), dict): + rows = table["content"].get("rows", []) + elif isinstance(table.get("content"), list): + # Old format: content is list of rows + if table["content"] and isinstance(table["content"][0], list): + rows = table["content"][1:] if len(table["content"]) > 1 else [] + + # Validate row column count matches header count + expectedColCount = len(headers) + validRows = [] + for rowIdx, row in enumerate(rows): + if isinstance(row, list): + if len(row) == expectedColCount: + validRows.append(row) + else: + logger.warning(f"Table {tableIdx+1}, row {rowIdx+1}: column count mismatch ({len(row)} vs {expectedColCount}), skipping row") + elif isinstance(row, dict): + # Convert dict row to list based on header order + rowList = [row.get(h, "") for h in headers] + validRows.append(rowList) + else: + logger.warning(f"Table {tableIdx+1}, row {rowIdx+1}: invalid row format, skipping") + + allRows.extend(validRows) + + # Keep all rows, including duplicates (duplicates may be intentional) + logger.info(f"Merged {len(tableElements)} tables: {len(allRows)} total rows (duplicates preserved)") + + return { + "type": "table", + "content": { + "headers": headers, + "rows": allRows + } + } + + def _isBinaryMimeType(self, mimeType: str) -> bool: + """Check if MIME type is binary.""" + binaryTypes = [ + "application/octet-stream", + "application/pdf", + "application/zip", + "application/x-zip-compressed" + ] + return mimeType in binaryTypes + + def _looksLikeBase64(self, data: str) -> bool: + """ + Heuristic check if string looks like base64-encoded data. + + Base64 contains only: A-Z, a-z, 0-9, +, /, =, and whitespace. + If >95% of characters are base64 chars and no normal text patterns, likely base64. + """ + if not data or len(data) < 100: + return False + + base64Chars = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r\t ") + sample = data[:500] # Check first 500 chars + if not sample: + return False + + base64Ratio = sum(1 for c in sample if c in base64Chars) / len(sample) + + # If >95% base64 chars and no normal text patterns (like spaces between words) → likely base64 + # Base64 typically has very long strings without spaces or punctuation + hasNormalTextPatterns = any( + c in sample[:200] for c in ".,!?;:()[]{}\"'" + ) or " " in sample[:200] # Double spaces suggest text + + return base64Ratio > 0.95 and not hasNormalTextPatterns + + def _findContentPartById(self, partId: str, contentParts: List[ContentPart]) -> Optional[ContentPart]: + """Finde ContentPart nach ID.""" + for part in contentParts: + if part.id == partId: + return part + return None + + def _needsAggregation( + self, + contentType: str, + contentPartCount: int + ) -> bool: + """ + Bestimmt ob mehrere ContentParts aggregiert werden müssen. + + Aggregation nötig wenn: + - content_type erfordert Aggregation (table, bullet_list) + - UND mehrere ContentParts vorhanden sind (> 1) + + Args: + contentType: Section content_type + contentPartCount: Anzahl der ContentParts in dieser Section + + Returns: + True wenn Aggregation nötig, False sonst + """ + aggregationTypes = ["table", "bullet_list"] + + if contentType in aggregationTypes and contentPartCount > 1: + return True + + # Optional: Auch für paragraph wenn mehrere Parts vorhanden + # (z.B. Vergleich mehrerer Dokumente) + # Standard: Keine Aggregation für paragraph + return False + + def _getAcceptedSectionTypesForFormat(self, outputFormat: str) -> List[str]: + """ + Get accepted section types for a given output format by querying the renderer. + + Args: + outputFormat: Format name (e.g., 'csv', 'json', 'pdf') + + Returns: + List of accepted section content types (e.g., ["table", "code_block"]) + + Raises: + ValueError: If renderer not found or doesn't provide accepted types + """ + from modules.serviceCenter.services.serviceGeneration.renderers.registry import getRenderer + + # Get document renderer for this format (structure filling is document generation path) + renderer = getRenderer(outputFormat, self.services, outputStyle='document') + + if not renderer: + raise ValueError(f"No renderer found for output format '{outputFormat}'. Check renderer registry.") + + if not hasattr(renderer, 'getAcceptedSectionTypes'): + raise ValueError(f"Renderer for '{outputFormat}' does not implement getAcceptedSectionTypes(). Add this method to the renderer.") + + acceptedTypes = renderer.getAcceptedSectionTypes(outputFormat) + + if not acceptedTypes: + raise ValueError(f"Renderer for '{outputFormat}' returned empty accepted types. Fix getAcceptedSectionTypes() in the renderer.") + + logger.debug(f"Renderer for '{outputFormat}' accepts: {acceptedTypes}") + return acceptedTypes + diff --git a/modules/serviceCenter/services/serviceAi/subStructureGeneration.py b/modules/serviceCenter/services/serviceAi/subStructureGeneration.py new file mode 100644 index 00000000..9795cf6f --- /dev/null +++ b/modules/serviceCenter/services/serviceAi/subStructureGeneration.py @@ -0,0 +1,508 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Structure Generation Module + +Handles document structure generation, including: +- Generating document structure with sections +- Building structure prompts +""" +import json +import logging +from typing import Dict, Any, List, Optional + +from modules.datamodels.datamodelExtraction import ContentPart +from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.workflows.processing.shared.stateTools import checkWorkflowStopped + +logger = logging.getLogger(__name__) + + +class StructureGenerator: + """Handles document structure generation.""" + + def __init__(self, services, aiService): + """Initialize StructureGenerator with service center and AI service access.""" + self.services = services + self.aiService = aiService + + def _getUserLanguage(self) -> str: + """Get user language for document generation""" + try: + if self.services: + # Prefer detected language if available (from user intention analysis) + if hasattr(self.services, 'currentUserLanguage') and self.services.currentUserLanguage: + return self.services.currentUserLanguage + # Fallback to user's preferred language + elif hasattr(self.services, 'user') and self.services.user and hasattr(self.services.user, 'language'): + return self.services.user.language + except Exception: + pass + return 'en' # Default fallback + + async def generateStructure( + self, + userPrompt: str, + contentParts: List[ContentPart], + outputFormat: Optional[str] = None, + parentOperationId: str = None + ) -> Dict[str, Any]: + """ + Phase 5C: Generiert Chapter-Struktur (Table of Contents). + Definiert für jedes Chapter: + - Level, Title + - contentParts (unified object with instruction and/or caption per part) + - generationHint + + Generate document structure with per-document format determination. + Multiple documents can be produced with different formats (e.g., one PDF, one HTML). + AI determines formats per-document from user prompt. The outputFormat parameter is + only a validation fallback - used if AI doesn't return format per document. + + Args: + userPrompt: User-Anfrage + contentParts: Alle vorbereiteten ContentParts mit Metadaten + outputFormat: Optional global format fallback. If omitted, formats are determined + from user prompt by AI. Used as validation fallback if AI doesn't + return format per document. Defaults to "txt" if not provided. + parentOperationId: Parent Operation-ID für ChatLog-Hierarchie + + Returns: + Struktur-Dict mit documents und chapters (nicht sections!) + """ + # If outputFormat not provided, use "txt" as fallback for validation + # AI will determine formats per document from user prompt + if not outputFormat: + outputFormat = "txt" + logger.debug("outputFormat not provided - using 'txt' as validation fallback, formats determined from prompt") + # Erstelle Operation-ID für Struktur-Generierung + structureOperationId = f"{parentOperationId}_structure_generation" + + # Starte ChatLog mit Parent-Referenz + formatDisplay = outputFormat if outputFormat else "auto-determined" + self.services.chat.progressLogStart( + structureOperationId, + "Chapter Structure Generation", + "Structure", + f"Generating chapter structure (format: {formatDisplay})", + parentOperationId=parentOperationId + ) + + try: + # Baue Chapter-Struktur-Prompt mit Content-Index + structurePrompt = self._buildChapterStructurePrompt( + userPrompt=userPrompt, + contentParts=contentParts, + outputFormat=outputFormat + ) + + # AI-Call für Chapter-Struktur-Generierung mit Looping-Unterstützung + # Use _callAiWithLooping instead of callAiPlanning to support continuation if response is cut + options = AiCallOptions( + operationType=OperationTypeEnum.DATA_GENERATE, + priority=PriorityEnum.QUALITY, + processingMode=ProcessingModeEnum.DETAILED, + compressPrompt=False, + compressContext=False, + resultFormat="json" + ) + + structurePrompt, templateStructure = self._buildChapterStructurePrompt( + userPrompt=userPrompt, + contentParts=contentParts, + outputFormat=outputFormat + ) + + # Create prompt builder for continuation support + async def buildChapterStructurePromptWithContinuation( + continuationContext: Any, + templateStructure: str, + basePrompt: str + ) -> str: + """Build chapter structure prompt with continuation context. Uses unified signature. + + Note: All initial context (userPrompt, contentParts, outputFormat, etc.) is already + contained in basePrompt. This function only adds continuation-specific instructions. + """ + # Extract continuation context fields (only what's needed for continuation) + incompletePart = continuationContext.incomplete_part + lastRawJson = continuationContext.last_raw_json + + # Generate both overlap context and hierarchy context using jsonContinuation + overlapContext = "" + unifiedContext = "" + if lastRawJson: + # Get contexts directly from jsonContinuation + from modules.shared.jsonContinuation import getContexts + contexts = getContexts(lastRawJson) + overlapContext = contexts.overlapContext + unifiedContext = contexts.hierarchyContextForPrompt + elif incompletePart: + unifiedContext = incompletePart + else: + unifiedContext = "Unable to extract context - response was completely broken" + + # Build unified continuation prompt format + continuationPrompt = f"""{basePrompt} + +--- CONTINUATION REQUEST --- +The previous JSON response was incomplete. Continue from where it stopped. + +Context showing structure hierarchy with cut point: +``` +{unifiedContext} +``` + +Overlap Requirement: +To ensure proper merging, your response MUST start EXACTLY with the overlap context shown below, then continue with new content. + +Overlap context (start your response with this exact text): +```json +{overlapContext if overlapContext else "No overlap context available"} +``` + +TASK: +1. Start your response EXACTLY with the overlap context shown above (character by character) +2. Continue seamlessly from where the overlap context ends +3. Complete the remaining content following the JSON structure template above +4. Return ONLY valid JSON following the structure template - no overlap/continuation wrapper objects + +CRITICAL: +- Your response MUST begin with the exact overlap context text (this enables automatic merging) +- Continue seamlessly after the overlap context with new content +- Your response must be valid JSON matching the structure template above""" + return continuationPrompt + + # Call AI with looping support + # NOTE: Do NOT pass contentParts here - we only need metadata for structure generation + # The contentParts metadata is already included in the prompt (contentPartsIndex) + # Actual content extraction happens later during section generation + checkWorkflowStopped(self.services) + aiResponseJson = await self.aiService.callAiWithLooping( + prompt=structurePrompt, + options=options, + debugPrefix="chapter_structure_generation", + promptBuilder=buildChapterStructurePromptWithContinuation, + promptArgs={ + "userPrompt": userPrompt, + "outputFormat": outputFormat, + "templateStructure": templateStructure, + "basePrompt": structurePrompt + }, + useCaseId="chapter_structure", # REQUIRED: Explicit use case ID + operationId=structureOperationId, + userPrompt=userPrompt, + contentParts=None # Do not pass ContentParts - only metadata needed, not content extraction + ) + + # Parse the complete JSON response (looping system already handles completion) + extractedJson = self.services.utils.jsonExtractString(aiResponseJson) + parsedJson, parseError, cleanedJson = self.services.utils.jsonTryParse(extractedJson) + + if parseError is not None: + # Even with looping, try repair as fallback + logger.warning(f"JSON parsing failed after looping: {str(parseError)}. Attempting repair...") + from modules.shared import jsonUtils + repairedJson = jsonUtils.repairBrokenJson(extractedJson) + if repairedJson: + parsedJson, parseError, _ = self.services.utils.jsonTryParse(json.dumps(repairedJson)) + if parseError is None: + logger.info("Successfully repaired and parsed JSON structure after looping") + structure = parsedJson + else: + logger.error(f"Failed to parse repaired JSON: {str(parseError)}") + raise ValueError(f"Failed to parse JSON structure after repair: {str(parseError)}") + else: + logger.error(f"Failed to repair JSON. Parse error: {str(parseError)}") + logger.error(f"Cleaned JSON preview (first 500 chars): {cleanedJson[:500]}") + raise ValueError(f"Failed to parse JSON structure: {str(parseError)}") + else: + structure = parsedJson + + # State 3 Validation: Validate and auto-fix structure + # Validation 3.1: Structure missing 'documents' field + if "documents" not in structure: + raise ValueError("Structure missing 'documents' field - cannot auto-fix") + + documents = structure["documents"] + + # Validation 3.2: Structure has no documents + if not isinstance(documents, list) or len(documents) == 0: + raise ValueError("Structure has no documents - cannot generate without documents") + + # Import renderer registry for format validation (existing infrastructure) + from modules.serviceCenter.services.serviceGeneration.renderers.registry import getRenderer + + # Validate and fix each document + for doc in documents: + # Validation 3.3 & 3.4: Document outputFormat + # outputFormat parameter is optional - if omitted, formats determined from prompt by AI + # Use as fallback only if AI doesn't return format per document + # Multiple documents can have different formats (e.g., one PDF, one HTML) + globalFormatFallback = outputFormat or "txt" # Fallback for validation + + if "outputFormat" not in doc or not doc["outputFormat"]: + # AI didn't return format or returned empty - use global fallback + doc["outputFormat"] = globalFormatFallback + logger.warning(f"Document {doc.get('id')} missing outputFormat - using fallback: {doc['outputFormat']}") + else: + # AI returned format - validate using existing renderer registry + formatName = str(doc["outputFormat"]).lower().strip() + renderer = getRenderer(formatName) # Uses existing infrastructure + + if not renderer: + # Format doesn't match any renderer - use txt (simple approach) + logger.warning(f"Document {doc.get('id')} has format without renderer: {formatName}, using 'txt'") + doc["outputFormat"] = "txt" + else: + # Valid format with renderer - normalize and keep AI result + doc["outputFormat"] = formatName + logger.debug(f"Document {doc.get('id')} using AI-determined format: {formatName}") + + # Validation 3.5 & 3.6: Document language + # Use validated currentUserLanguage (always valid, validated during user intention analysis) + # Access via _getUserLanguage() which uses self.services.currentUserLanguage + userPromptLanguage = self._getUserLanguage() # Uses validated currentUserLanguage infrastructure + + if "language" not in doc or not isinstance(doc["language"], str) or len(doc["language"]) != 2: + # AI didn't return language or invalid format - use validated currentUserLanguage + doc["language"] = userPromptLanguage + if "language" not in doc: + logger.warning(f"Document {doc.get('id')} missing language - using currentUserLanguage: {userPromptLanguage}") + else: + logger.warning(f"Document {doc.get('id')} has invalid language format from AI: {doc['language']}, using currentUserLanguage") + else: + # AI returned valid language format - normalize + doc["language"] = doc["language"].lower().strip()[:2] + logger.debug(f"Document {doc.get('id')} using AI-determined language: {doc['language']}") + + # Validation 3.7: Document missing 'chapters' field + if "chapters" not in doc: + raise ValueError(f"Document {doc.get('id')} missing 'chapters' field - cannot auto-fix") + + # Validation 3.8: Chapter missing 'contentParts' field + for chapter in doc["chapters"]: + if "contentParts" not in chapter: + raise ValueError(f"Chapter {chapter.get('id')} missing 'contentParts' field - cannot auto-fix") + + # ChatLog abschließen + self.services.chat.progressLogFinish(structureOperationId, True) + + return structure + + except Exception as e: + self.services.chat.progressLogFinish(structureOperationId, False) + logger.error(f"Error in generateStructure: {str(e)}") + raise + + def _buildChapterStructurePrompt( + self, + userPrompt: str, + contentParts: List[ContentPart], + outputFormat: str + ) -> tuple[str, str]: + """Baue Prompt für Chapter-Struktur-Generierung.""" + # Baue ContentParts-Index - filtere leere Parts heraus + contentPartsIndex = "" + validParts = [] + filteredParts = [] + + for part in contentParts: + contentFormat = part.metadata.get("contentFormat", "unknown") + + # WICHTIG: Reference Parts haben absichtlich leere Daten - immer einschließen + if contentFormat == "reference": + validParts.append(part) + logger.debug(f"Including reference ContentPart {part.id} (intentionally empty data)") + continue + + # Überspringe leere Parts (keine Daten oder nur Container ohne Inhalt) + # ABER: Reference Parts wurden bereits oben behandelt + if not part.data or (isinstance(part.data, str) and len(part.data.strip()) == 0): + # Überspringe Container-Parts ohne Daten + if part.typeGroup == "container" and not part.data: + filteredParts.append((part.id, "container without data")) + continue + # Überspringe andere leere Parts (aber nicht Reference, die wurden bereits behandelt) + if not part.data: + filteredParts.append((part.id, f"no data (format: {contentFormat})")) + continue + + validParts.append(part) + logger.debug(f"Including ContentPart {part.id}: format={contentFormat}, type={part.typeGroup}, dataLength={len(str(part.data)) if part.data else 0}") + + if filteredParts: + logger.debug(f"Filtered out {len(filteredParts)} empty ContentParts: {filteredParts}") + + logger.info(f"Building structure prompt with {len(validParts)} valid ContentParts (from {len(contentParts)} total)") + + # Baue Index nur für gültige Parts + for i, part in enumerate(validParts, 1): + contentFormat = part.metadata.get("contentFormat", "unknown") + originalFileName = part.metadata.get('originalFileName', 'N/A') + + contentPartsIndex += f"\n{i}. ContentPart ID: {part.id}\n" + contentPartsIndex += f" Format: {contentFormat}\n" + contentPartsIndex += f" Type: {part.typeGroup}\n" + contentPartsIndex += f" MIME Type: {part.mimeType or 'N/A'}\n" + contentPartsIndex += f" Source: {part.metadata.get('documentId', 'unknown')}\n" + contentPartsIndex += f" Original file name: {originalFileName}\n" + contentPartsIndex += f" Usage hint: {part.metadata.get('usageHint', 'N/A')}\n" + + if not contentPartsIndex: + contentPartsIndex = "\n(No content parts available)" + + # Get language from services (user intention analysis) + language = self._getUserLanguage() + logger.debug(f"Using language from services (user intention analysis) for structure generation: {language}") + + # Create template structure explicitly (not extracted from prompt) + # This ensures exact identity between initial and continuation prompts + templateStructure = f"""{{ + "metadata": {{ + "title": "Document Title", + "language": "{language}" + }}, + "documents": [{{ + "id": "doc_1", + "title": "Document Title", + "filename": "document.{outputFormat}", + "outputFormat": "{outputFormat}", + "language": "{language}", + "chapters": [ + {{ + "id": "chapter_1", + "level": 1, + "title": "Chapter Title", + "contentParts": {{ + "extracted_part_id": {{ + "instruction": "Use extracted content with ALL relevant details from user request" + }} + }}, + "generationHint": "Detailed description including ALL relevant details from user request for this chapter", + "sections": [] + }} + ] + }}] +}}""" + + prompt = f"""# TASK: Plan Document Structure (Documents + Chapters) + +This is a STRUCTURE PLANNING task. You define which documents to create and which chapters each document will have. +Chapter CONTENT will be generated in a later step - here you only plan the STRUCTURE and assign content references. +Return EXACTLY ONE complete JSON object. Do not generate multiple JSON objects, alternatives, or variations. Do not use separators like "---" between JSON objects. + +## USER REQUEST (for context) +``` +{userPrompt} +``` + +## AVAILABLE CONTENT PARTS +{contentPartsIndex} + +## CONTENT ASSIGNMENT RULE + +CRITICAL: Every chapter MUST have contentParts assigned if it relates to documents/images/data from the user request. +If the user request mentions documents/images/data, then EVERY chapter that generates content related to those references MUST assign the relevant ContentParts explicitly. + +Assignment logic: +- If chapter DISPLAYS a document/image → assign "object" format ContentPart with "caption" +- If chapter generates text content ABOUT a document/image/data → assign ContentPart with "instruction": + - Prefer "extracted" format if available (contains analyzed/extracted content) + - If only "object" format is available, use "object" format with "instruction" (to write ABOUT the image/document) +- If chapter's generationHint or purpose relates to a document/image/data mentioned in user request → it MUST have ContentParts assigned +- Multiple chapters might assign the same ContentPart (e.g., one chapter displays image, another writes about it) +- Use ContentPart IDs exactly as listed in AVAILABLE CONTENT PARTS above +- Empty contentParts are only allowed if chapter generates content WITHOUT referencing any documents/images/data from the user request + +CRITICAL RULE: If the user request mentions BOTH: + a) Documents/images/data (listed in AVAILABLE CONTENT PARTS above), AND + b) Generic content types (article text, main content, body text, etc.) +Then chapters that generate those generic content types MUST assign the relevant ContentParts, because the content should relate to or be based on the provided documents/images/data. + +## CONTENT EFFICIENCY PRINCIPLES +- Generate COMPACT content: Focus on essential information only +- AVOID verbose, lengthy, or repetitive text - be concise and direct +- Prioritize FACTS over filler text - no introductions like "In this chapter..." +- Minimize system resources: shorter content = faster processing +- Quality over quantity: precise, meaningful content rather than padding + +## CHAPTER STRUCTURE REQUIREMENTS +- Generate chapters based on USER REQUEST - analyze what structure the user wants +- Create ONLY the minimum chapters needed to cover the user's request - avoid over-structuring +- IMPORTANT: Each chapter MUST have ALL these fields: + - id: Unique identifier (e.g., "chapter_1") + - level: Heading level (1, 2, 3, etc.) + - title: Chapter title + - contentParts: Object mapping ContentPart IDs to usage instructions (MUST assign if chapter relates to documents/data from user request) + - generationHint: Description of what content to generate (including formatting/styling requirements) + - sections: Empty array [] (REQUIRED - sections are generated in next phase) +- contentParts: {{"partId": {{"instruction": "..."}} or {{"caption": "..."}} or both}} - Assign ContentParts as required by CONTENT ASSIGNMENT RULE above +- The "instruction" field for each ContentPart MUST contain ALL relevant details from the USER REQUEST that apply to content extraction for this specific chapter. Include all formatting rules, data requirements, constraints, and specifications mentioned in the user request that are relevant for processing this ContentPart in this chapter. +- generationHint: Keep CONCISE but include relevant details from the USER REQUEST. Focus on WHAT to generate, not HOW to phrase it verbosely. +- The number of chapters depends on the user request - create only what is requested. Do NOT create chapters for topics without available data. + +CRITICAL: Only create chapters for CONTENT sections, not for formatting/styling requirements. Formatting/styling requirements to be included in each generationHint if needed. + +## DOCUMENT STRUCTURE + +For each document, determine: +- outputFormat: From USER REQUEST (explicit mention or infer from purpose/content type). Default: "{outputFormat}". Multiple documents can have different formats. +- language: From USER REQUEST (map to ISO 639-1: de, en, fr, it...). Default: "{language}". Multiple documents can have different languages. +- chapters: Structure appropriately for the format (e.g., pptx=slides, docx=sections, xlsx=worksheets). Match format capabilities and constraints. + +Required JSON fields: +- metadata: {{"title": "...", "language": "..."}} +- documents: Array with id, title, filename, outputFormat, language, chapters[] +- chapters: Array with id, level, title, contentParts, generationHint, sections[] + +EXAMPLE STRUCTURE (for reference only - adapt to user request): +{{ + "metadata": {{ + "title": "Document Title", + "language": "{language}" + }}, + "documents": [{{ + "id": "doc_1", + "title": "Document Title", + "filename": "document.{outputFormat}", + "outputFormat": "{outputFormat}", + "language": "{language}", + "chapters": [ + {{ + "id": "chapter_1", + "level": 1, + "title": "Chapter Title", + "contentParts": {{ + "extracted_part_id": {{ + "instruction": "Use extracted content with ALL relevant details from user request" + }} + }}, + "generationHint": "Detailed description including ALL relevant details from user request for this chapter", + "sections": [] + }} + ] + }}] +}} + +CRITICAL INSTRUCTIONS: +- Generate chapters based on USER REQUEST, NOT based on the example above +- The example shows the JSON structure format, NOT the required chapters +- Create only the chapters that match the user's request +- Adapt chapter titles and structure to match the user's specific request +- Determine outputFormat and language for each document by analyzing the USER REQUEST above +- The example shows placeholders "{outputFormat}" and "{language}" - YOU MUST REPLACE THESE with actual values determined from the USER REQUEST + +MANDATORY CONTENT ASSIGNMENT CHECK: +For each chapter, verify: +1. Does the user request mention documents/images/data? (e.g., "photo", "image", "document", "data", "based on", "about") +2. Does this chapter's generationHint, title, or purpose relate to those documents/images/data mentioned in step 1? + - Examples: "article about the photo", "text describing the image", "analysis of the document", "content based on the data" + - Even if chapter doesn't explicitly say "about the image", if user request mentions both the image AND this chapter's content type → relate them +3. If YES to both → chapter MUST have contentParts assigned (cannot be empty {{}}) +4. If ContentPart is "object" format and chapter needs to write ABOUT it → assign with "instruction" field, not just "caption" + +OUTPUT FORMAT: Start with {{ and end with }}. Do NOT use markdown code fences (```json). Do NOT add explanatory text before or after the JSON. Return ONLY the JSON object itself. +""" + return prompt, templateStructure + diff --git a/modules/serviceCenter/services/serviceBilling/__init__.py b/modules/serviceCenter/services/serviceBilling/__init__.py new file mode 100644 index 00000000..55d95d1a --- /dev/null +++ b/modules/serviceCenter/services/serviceBilling/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Billing service.""" + +from .mainServiceBilling import BillingService, getService, InsufficientBalanceException, ProviderNotAllowedException, BillingContextError + +__all__ = ["BillingService", "getService", "InsufficientBalanceException", "ProviderNotAllowedException", "BillingContextError"] diff --git a/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py new file mode 100644 index 00000000..128a307e --- /dev/null +++ b/modules/serviceCenter/services/serviceBilling/mainServiceBilling.py @@ -0,0 +1,436 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Billing Service - Central service for billing operations. + +Handles: +- Balance checks before AI operations +- Cost recording after AI operations +- Provider permission checks via RBAC +- Price calculation with markup +""" + +import logging +from typing import Dict, Any, List, Optional +from datetime import datetime + +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelBilling import ( + BillingModelEnum, + BillingCheckResult, + TransactionTypeEnum, + ReferenceTypeEnum, + BillingTransaction, + BillingBalanceResponse, +) +from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface + +logger = logging.getLogger(__name__) + +# Markup percentage for internal pricing (+50% für Infrastruktur und Platform Service + 50% für Währungsrisiko ==> Faktor 2.0) +BILLING_MARKUP_PERCENT = 100 + +# Singleton cache +_billingServices: Dict[str, "BillingService"] = {} + + +def getService(currentUser: User, mandateId: str, featureInstanceId: str = None, featureCode: str = None) -> "BillingService": + """ + Factory function to get or create a BillingService instance. + + Args: + currentUser: Current user object + mandateId: Mandate ID for context + featureInstanceId: Optional feature instance ID + featureCode: Optional feature code (e.g., 'chatplayground', 'automation') + + Returns: + BillingService instance + """ + cacheKey = f"{currentUser.id}_{mandateId}_{featureInstanceId}" + + if cacheKey not in _billingServices: + _billingServices[cacheKey] = BillingService(currentUser, mandateId, featureInstanceId, featureCode) + else: + _billingServices[cacheKey].setContext(currentUser, mandateId, featureInstanceId, featureCode) + + return _billingServices[cacheKey] + + +def _get_feature_code_from_context(context) -> Optional[str]: + """Extract featureCode from ServiceCenterContext.""" + if context.workflow and hasattr(context.workflow, "feature") and context.workflow.feature: + return getattr(context.workflow.feature, "code", None) + return getattr(context.workflow, "featureCode", None) if context.workflow else None + + +class BillingService: + """ + Central billing service for AI operations. + + Responsibilities: + - Check balance before operations + - Record usage costs + - Apply pricing markup + - Check provider permissions via RBAC + + Supports both service center (context, get_service) and legacy (user, mandateId, ...) initialization. + """ + + def __init__(self, context_or_user, mandateId=None, featureInstanceId=None, featureCode=None, get_service=None): + """ + Initialize the billing service. + + Service center: (context, get_service) - resolver passes exactly these two args + Legacy: (currentUser, mandateId, featureInstanceId, featureCode) from getService() factory + """ + # Detect service center: second arg is callable (get_service) + if mandateId is not None and callable(mandateId): + ctx = context_or_user + get_service = mandateId + self.currentUser = ctx.user + self.mandateId = ctx.mandate_id or "" + self.featureInstanceId = ctx.feature_instance_id + self.featureCode = _get_feature_code_from_context(ctx) + elif get_service is not None and hasattr(context_or_user, "user"): + ctx = context_or_user + self.currentUser = ctx.user + self.mandateId = ctx.mandate_id or "" + self.featureInstanceId = ctx.feature_instance_id + self.featureCode = _get_feature_code_from_context(ctx) + else: + self.currentUser = context_or_user + self.mandateId = mandateId or "" + self.featureInstanceId = featureInstanceId + self.featureCode = featureCode + + self._billingInterface = getBillingInterface(self.currentUser, self.mandateId) + self._settingsCache = None + + def setContext( + self, + currentUser: User, + mandateId: str, + featureInstanceId: str = None, + featureCode: str = None + ): + """Update service context.""" + self.currentUser = currentUser + self.mandateId = mandateId + self.featureInstanceId = featureInstanceId + self.featureCode = featureCode + self._billingInterface = getBillingInterface(currentUser, mandateId) + self._settingsCache = None + + def _getSettings(self) -> Optional[Dict[str, Any]]: + """Get billing settings with caching.""" + if self._settingsCache is None: + self._settingsCache = self._billingInterface.getSettings(self.mandateId) + return self._settingsCache + + # ========================================================================= + # Price Calculation + # ========================================================================= + + def calculatePriceWithMarkup(self, basePriceCHF: float) -> float: + """ + Calculate final price with markup. + + The AICore plugins return prices in their original currency (USD). + This method applies the configured markup percentage. + + Args: + basePriceCHF: Base price from AI model (actually USD from provider) + + Returns: + Final price in CHF with markup applied + """ + if basePriceCHF <= 0: + return 0.0 + + # Apply markup (50% = multiply by 1.5) + markup_multiplier = 1 + (BILLING_MARKUP_PERCENT / 100) + return round(basePriceCHF * markup_multiplier, 6) + + # ========================================================================= + # Balance Operations + # ========================================================================= + + def checkBalance(self, estimatedCost: float = 0.0) -> BillingCheckResult: + """ + Check if the current user/mandate has sufficient balance. + + Args: + estimatedCost: Estimated cost of the operation (with markup applied) + + Returns: + BillingCheckResult indicating if operation is allowed + """ + return self._billingInterface.checkBalance( + self.mandateId, + self.currentUser.id, + estimatedCost + ) + + def hasBalance(self, estimatedCost: float = 0.0) -> bool: + """ + Quick check if balance is sufficient. + + Args: + estimatedCost: Estimated cost with markup + + Returns: + True if operation is allowed + """ + result = self.checkBalance(estimatedCost) + return result.allowed + + def getCurrentBalance(self) -> float: + """ + Get current balance for the user/mandate. + + Returns: + Current balance in CHF + """ + result = self.checkBalance(0.0) + return result.currentBalance or 0.0 + + # ========================================================================= + # Usage Recording + # ========================================================================= + + def recordUsage( + self, + priceCHF: float, + workflowId: str = None, + aicoreProvider: str = None, + aicoreModel: str = None, + description: str = None + ) -> Optional[Dict[str, Any]]: + """ + Record AI usage cost as a billing transaction. + + This method: + 1. Applies the pricing markup + 2. Creates a DEBIT transaction + 3. Updates the account balance + + Args: + priceCHF: Base price from AI model (before markup) + workflowId: Optional workflow ID + aicoreProvider: AICore provider name (e.g., 'anthropic', 'openai') + aicoreModel: AICore model name (e.g., 'claude-4-sonnet', 'gpt-4o') + description: Optional description + + Returns: + Created transaction dict or None if not recorded + """ + if priceCHF <= 0: + return None + + # Apply markup + finalPrice = self.calculatePriceWithMarkup(priceCHF) + + if finalPrice <= 0: + return None + + # Build description + if not description: + description = f"AI Usage: {aicoreModel or aicoreProvider or 'unknown'}" + + return self._billingInterface.recordUsage( + mandateId=self.mandateId, + userId=self.currentUser.id, + priceCHF=finalPrice, + workflowId=workflowId, + featureInstanceId=self.featureInstanceId, + featureCode=self.featureCode, + aicoreProvider=aicoreProvider, + aicoreModel=aicoreModel, + description=description + ) + + # ========================================================================= + # Provider Permission Check (via RBAC) + # ========================================================================= + + def isProviderAllowed(self, provider: str) -> bool: + """ + Check if the user has permission to use an AICore provider. + + Uses RBAC to check for resource permission: + resource.aicore.{provider} + + Args: + provider: Provider name (e.g., 'anthropic', 'openai') + + Returns: + True if provider is allowed + """ + try: + from modules.security.rbac import RbacClass + from modules.datamodels.datamodelRbac import AccessRuleContext + from modules.security.rootAccess import getRootDbAppConnector + + # Get database connector via established pattern + dbApp = getRootDbAppConnector() + + rbac = RbacClass(dbApp, dbApp) + resourceKey = f"resource.aicore.{provider}" + + # Check if user has view permission for this resource (view = use for RESOURCE context) + permissions = rbac.getUserPermissions( + self.currentUser, + AccessRuleContext.RESOURCE, + resourceKey, + mandateId=self.mandateId + ) + + return permissions.view + except Exception as e: + logger.warning(f"Error checking provider permission: {e}") + # Default to allowed if RBAC check fails + return True + + def getallowedProviders(self) -> List[str]: + """ + Get list of AICore providers the user is allowed to use. + + Returns: + List of allowed provider names + """ + try: + from modules.aicore.aicoreModelRegistry import modelRegistry + + # Get all available providers + connectors = modelRegistry.discoverConnectors() + allProviders = [c.getConnectorType() for c in connectors] + + # Filter by RBAC permissions + return [p for p in allProviders if self.isProviderAllowed(p)] + except Exception as e: + logger.warning(f"Error getting allowed providers: {e}") + return [] + + # ========================================================================= + # Admin Operations + # ========================================================================= + + def addCredit( + self, + amount: float, + description: str = "Manual credit", + referenceType: ReferenceTypeEnum = ReferenceTypeEnum.ADMIN + ) -> Optional[Dict[str, Any]]: + """ + Add credit to the account (admin operation). + + Args: + amount: Amount to credit (positive) + description: Transaction description + referenceType: Reference type (ADMIN, PAYMENT, SYSTEM) + + Returns: + Created transaction dict or None + """ + if amount <= 0: + return None + + settings = self._getSettings() + if not settings: + logger.warning(f"No billing settings for mandate {self.mandateId}") + return None + + billingModel = BillingModelEnum(settings.get("billingModel", BillingModelEnum.UNLIMITED.value)) + + # Get or create account + if billingModel == BillingModelEnum.PREPAY_USER: + account = self._billingInterface.getOrCreateUserAccount( + self.mandateId, + self.currentUser.id, + initialBalance=0.0 + ) + else: + account = self._billingInterface.getOrCreateMandateAccount( + self.mandateId, + initialBalance=0.0 + ) + + # Create credit transaction + transaction = BillingTransaction( + accountId=account["id"], + transactionType=TransactionTypeEnum.CREDIT, + amount=amount, + description=description, + referenceType=referenceType + ) + + return self._billingInterface.createTransaction(transaction) + + # ========================================================================= + # Statistics & Reporting + # ========================================================================= + + def getBalancesForUser(self) -> List[BillingBalanceResponse]: + """ + Get all billing balances for the current user. + + Returns: + List of balance responses for each mandate + """ + return self._billingInterface.getBalancesForUser(self.currentUser.id) + + def getTransactionHistory(self, limit: int = 100) -> List[Dict[str, Any]]: + """ + Get transaction history for the user across all mandates. + + Args: + limit: Maximum number of transactions + + Returns: + List of transactions + """ + return self._billingInterface.getTransactionsForUser(self.currentUser.id, limit=limit) + + +# ============================================================================ +# Exception Classes +# ============================================================================ + +class InsufficientBalanceException(Exception): + """Raised when there's insufficient balance for an operation.""" + + def __init__(self, currentBalance: float, requiredAmount: float, message: str = None): + self.currentBalance = currentBalance + self.requiredAmount = requiredAmount + self.message = message or f"Insufficient balance. Current: {currentBalance:.2f} CHF, Required: {requiredAmount:.2f} CHF" + super().__init__(self.message) + + +class ProviderNotAllowedException(Exception): + """Raised when a user doesn't have permission to use an AI provider.""" + + def __init__(self, provider: str, message: str = None): + self.provider = provider + self.message = message or f"Provider '{provider}' is not allowed for your role" + super().__init__(self.message) + + +class BillingContextError(Exception): + """Raised when billing context is incomplete (missing mandateId, user, etc.). + + This is a FAIL-SAFE error: AI calls MUST NOT proceed without valid billing context. + Acts like a 0 CHF credit card pre-authorization check - validates that billing + CAN be recorded before any expensive AI operation starts. + """ + + def __init__(self, message: str = None): + self.message = message or "Billing context incomplete - AI call blocked" + super().__init__(self.message) + + +# Expose exception classes on BillingService so consumers can use service.InsufficientBalanceException +# instead of importing from this module +BillingService.InsufficientBalanceException = InsufficientBalanceException +BillingService.ProviderNotAllowedException = ProviderNotAllowedException +BillingService.BillingContextError = BillingContextError diff --git a/modules/serviceCenter/services/serviceBilling/stripeCheckout.py b/modules/serviceCenter/services/serviceBilling/stripeCheckout.py new file mode 100644 index 00000000..692e5087 --- /dev/null +++ b/modules/serviceCenter/services/serviceBilling/stripeCheckout.py @@ -0,0 +1,104 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Stripe Checkout service for billing credit top-ups. +Creates Checkout Sessions for redirect-based payment flow. +""" + +import logging +from typing import Optional + +from modules.shared.configuration import APP_CONFIG + +logger = logging.getLogger(__name__) + +# Server-side allowed amounts in CHF - never trust client +ALLOWED_AMOUNTS_CHF = [10, 25, 50, 100, 250, 500] + + +def create_checkout_session( + mandate_id: str, + user_id: Optional[str], + amount_chf: float +) -> str: + """ + Create a Stripe Checkout Session for credit top-up. + + Amount and currency are validated server-side. The client-provided amount + must match an allowed preset. + + Args: + mandate_id: Target mandate ID + user_id: Target user ID (for PREPAY_USER) or None (for mandate pool) + amount_chf: Amount in CHF (must be in ALLOWED_AMOUNTS_CHF) + + Returns: + Stripe Checkout Session URL for redirect + + Raises: + ValueError: If amount is invalid + """ + import stripe + + # Validate amount server-side + if amount_chf not in ALLOWED_AMOUNTS_CHF: + raise ValueError( + f"Invalid amount {amount_chf} CHF. Allowed: {ALLOWED_AMOUNTS_CHF}" + ) + + # Pin API version from config (match Stripe Dashboard) + api_version = APP_CONFIG.get("STRIPE_API_VERSION") + if api_version: + stripe.api_version = api_version + + # Get secrets + secret_key = APP_CONFIG.get("STRIPE_SECRET_KEY_SECRET") or APP_CONFIG.get("STRIPE_SECRET_KEY") + if not secret_key: + raise ValueError("STRIPE_SECRET_KEY_SECRET not configured") + + stripe.api_key = secret_key + + frontend_url = APP_CONFIG.get("APP_FRONTEND_URL", "https://nyla-int.poweron-center.net") + base_path = "/admin/billing" + success_url = f"{frontend_url.rstrip('/')}{base_path}?success=true&session_id={{CHECKOUT_SESSION_ID}}" + cancel_url = f"{frontend_url.rstrip('/')}{base_path}?canceled=true" + + # Amount in cents for Stripe (CHF uses 2 decimal places) + amount_cents = int(round(amount_chf * 100)) + + metadata = { + "mandateId": mandate_id, + "amountChf": str(amount_chf), + } + if user_id: + metadata["userId"] = user_id + + session = stripe.checkout.Session.create( + mode="payment", + line_items=[ + { + "price_data": { + "currency": "chf", + "unit_amount": amount_cents, + "product_data": { + "name": "Guthaben aufladen", + "description": "AI Service Guthaben (CHF)", + }, + }, + "quantity": 1, + } + ], + success_url=success_url, + cancel_url=cancel_url, + metadata=metadata, + ) + + if not session or not session.url: + raise ValueError("Stripe Checkout Session creation failed") + + logger.info( + f"Created Stripe Checkout Session {session.id} for mandate {mandate_id}, " + f"amount {amount_chf} CHF" + ) + + return session.url diff --git a/modules/serviceCenter/services/serviceChat/__init__.py b/modules/serviceCenter/services/serviceChat/__init__.py new file mode 100644 index 00000000..a776b886 --- /dev/null +++ b/modules/serviceCenter/services/serviceChat/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chat service.""" + +from .mainServiceChat import ChatService + +__all__ = ["ChatService"] diff --git a/modules/serviceCenter/services/serviceChat/mainServiceChat.py b/modules/serviceCenter/services/serviceChat/mainServiceChat.py new file mode 100644 index 00000000..6182f397 --- /dev/null +++ b/modules/serviceCenter/services/serviceChat/mainServiceChat.py @@ -0,0 +1,1086 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Chat service for document processing, chat operations, and workflow management.""" +import logging +from typing import Dict, Any, List, Optional, Callable +from modules.datamodels.datamodelUam import User, UserConnection +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog +from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum +from modules.shared.progressLogger import ProgressLogger + +logger = logging.getLogger(__name__) + + +class ChatService: + """Service class containing methods for document processing, chat operations, and workflow management.""" + + def __init__(self, context, get_service: Callable[[str], Any]): + """Initialize with ServiceCenterContext and service resolver.""" + self._context = context + self._get_service = get_service + self.user = context.user + from modules.interfaces.interfaceDbApp import getInterface as getAppInterface + from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface + from modules.interfaces.interfaceDbChat import getInterface as getChatInterface + self.interfaceDbApp = getAppInterface(context.user, mandateId=context.mandate_id) + self.interfaceDbComponent = getComponentInterface(context.user, mandateId=context.mandate_id) + self.interfaceDbChat = getChatInterface( + context.user, + mandateId=context.mandate_id, + featureInstanceId=context.feature_instance_id, + ) + self._progressLogger = None + + @property + def _workflow(self): + """Workflow from context (stable during workflow execution).""" + return self._context.workflow + + def getChatDocumentsFromDocumentList(self, documentList) -> List[ChatDocument]: + """Get ChatDocuments from a DocumentReferenceList. + + Args: + documentList: DocumentReferenceList (required) + + Returns: + List[ChatDocument]: List of ChatDocument objects + """ + from modules.datamodels.datamodelDocref import DocumentReferenceList + + if not isinstance(documentList, DocumentReferenceList): + logger.error(f"getChatDocumentsFromDocumentList: Invalid documentList type: {type(documentList)}. Expected DocumentReferenceList.") + return [] + + # Convert to string list for processing + stringRefs = documentList.to_string_list() + + try: + # Use self._workflow which is the ChatWorkflow object (stable during workflow execution) + workflow = self._workflow + if not workflow: + logger.error("getChatDocumentsFromDocumentList: No workflow available (self._workflow is not set)") + return [] + + workflowId = workflow.id if hasattr(workflow, 'id') else 'NO_ID' + workflowObjId = id(workflow) + logger.debug(f"getChatDocumentsFromDocumentList: input documentList = {stringRefs}") + logger.debug(f"getChatDocumentsFromDocumentList: using workflow.id = {workflowId}, workflow object id = {workflowObjId}") + + # Root cause analysis: Verify workflow.messages integrity and detect workflow changes + self._verifyWorkflowMessagesIntegrity(workflow, workflowId) + + # Debug: list available messages with their labels and document names (filtered by workflowId) + try: + if workflow and hasattr(workflow, 'messages') and workflow.messages: + msgLines = [] + messagesFromOtherWorkflows = [] + for message in workflow.messages: + msgWorkflowId = getattr(message, 'workflowId', None) + # Only include messages that belong to this workflow + if msgWorkflowId and msgWorkflowId != workflowId: + messagesFromOtherWorkflows.append(f"id={getattr(message, 'id', None)}, label={getattr(message, 'documentsLabel', None)}, workflowId={msgWorkflowId}") + continue + # Also skip messages without workflowId (shouldn't happen, but be safe) + if not msgWorkflowId: + messagesFromOtherWorkflows.append(f"id={getattr(message, 'id', None)}, label={getattr(message, 'documentsLabel', None)}, workflowId=Missing") + continue + + label = getattr(message, 'documentsLabel', None) + docNames = [] + if getattr(message, 'documents', None): + for doc in message.documents: + name = getattr(doc, 'fileName', None) or getattr(doc, 'documentName', None) or 'Unnamed' + docNames.append(name) + msgLines.append( + f"- id={getattr(message, 'id', None)}, label={label}, workflowId={msgWorkflowId}, docs={docNames}" + ) + if msgLines: + logger.debug("getChatDocumentsFromDocumentList: available messages (filtered for workflow):\n" + "\n".join(msgLines)) + if messagesFromOtherWorkflows: + logger.warning(f"getChatDocumentsFromDocumentList: Found {len(messagesFromOtherWorkflows)} messages from other workflows in workflow.messages list:\n" + "\n".join(messagesFromOtherWorkflows)) + else: + logger.debug("getChatDocumentsFromDocumentList: no messages available on current workflow") + except Exception as e: + logger.debug(f"getChatDocumentsFromDocumentList: unable to enumerate messages for debug: {e}") + + allDocuments = [] + for docRef in stringRefs: + if docRef.startswith("docItem:"): + # docItem:: or docItem: (filename is optional) + # ALWAYS try to match by documentId first (parts[1] is always the documentId when format is correct) + # Both formats are supported: docItem: and docItem:: + parts = docRef.split(':') + if len(parts) >= 2: + docId = parts[1] # This should be the documentId (UUID) + docFound = False + + # ALWAYS try to match by documentId first (regardless of number of parts) + # This handles both formats: + # - docItem: (without filename - still works) + # - docItem:: (with filename - preferred) + for message in workflow.messages: + # Validate message belongs to this workflow + msgWorkflowId = getattr(message, 'workflowId', None) + if not msgWorkflowId or msgWorkflowId != workflowId: + continue + + if message.documents: + for doc in message.documents: + if doc.id == docId: + allDocuments.append(doc) + docFound = True + logger.debug(f"Matched document reference '{docRef}' to document {doc.id} (fileName: {getattr(doc, 'fileName', 'unknown')}) by documentId") + break + if docFound: + break + + # Fallback: If not found by documentId and it looks like a filename (has file extension), try filename matching + # This handles cases where AI incorrectly generates docItem:filename.docx + if not docFound and '.' in docId and len(parts) == 2: + # Format: docItem:filename (AI generated wrong format) - try to match by filename + filename = parts[1] + logger.warning(f"Document reference '{docRef}' not found by documentId, attempting to match by filename: {filename}") + + for message in workflow.messages: + # Validate message belongs to this workflow + msgWorkflowId = getattr(message, 'workflowId', None) + if not msgWorkflowId or msgWorkflowId != workflowId: + continue + + if message.documents: + for doc in message.documents: + docFileName = getattr(doc, 'fileName', '') + # Match filename exactly or by base name (without path) + if docFileName == filename or docFileName.endswith(filename): + allDocuments.append(doc) + docFound = True + logger.info(f"Matched document reference '{docRef}' to document {doc.id} by filename {docFileName}") + break + if docFound: + break + + if not docFound: + logger.error(f"Could not resolve document reference '{docRef}' - no document found with filename '{filename}'") + elif not docFound: + logger.error(f"Could not resolve document reference '{docRef}' - no document found with documentId '{docId}'") + elif docRef.startswith("docList:"): + # docList::