# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
import json
import logging
import re
import time
import base64
from typing import Dict, Any, List, Optional, Tuple, Callable
from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument, WorkflowModeEnum
from modules.datamodels.datamodelAi import AiCallRequest, AiCallResponse, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
from modules.datamodels.datamodelExtraction import ContentPart, DocumentIntent
from modules.datamodels.datamodelWorkflow import AiResponse, AiResponseMetadata, DocumentData
from modules.datamodels.datamodelDocument import RenderedDocument
from modules.interfaces.interfaceAiObjects import AiObjects
from modules.shared.jsonUtils import (
    parseJsonWithModel
)
from .subJsonResponseHandling import JsonResponseHandler
from modules.datamodels.datamodelAi import JsonAccumulationState
from modules.serviceCenter.services.serviceBilling.billingExhaustedNotify import (
    maybeEmailMandatePoolExhausted,
)
from modules.serviceCenter.services.serviceBilling.mainServiceBilling import (
    getService as getBillingService,
    InsufficientBalanceException,
    ProviderNotAllowedException,
    BillingContextError
)
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import (
    SubscriptionInactiveException,
    SUBSCRIPTION_REASONS,
)

logger = logging.getLogger(__name__)

# Rebuild the model to resolve forward references
AiCallRequest.model_rebuild()


class _ServicesAdapter:
    """Adapter providing Services-like interface from (context, get_service).
    Workflow is read from context dynamically so propagation updates are visible."""
    def __init__(self, context, get_service: Callable[[str], Any]):
        self._context = context
        self._get_service = get_service
        self.user = context.user
        self.mandateId = context.mandate_id
        self.featureInstanceId = context.feature_instance_id

    @property
    def workflow(self):
        return self._context.workflow

    @property
    def chat(self):
        return self._get_service("chat")

    @property
    def extraction(self):
        return self._get_service("extraction")

    @property
    def utils(self):
        return self._get_service("utils")

    @property
    def ai(self):
        return self._get_service("ai")

    @property
    def interfaceDbChat(self):
        return self._get_service("chat").interfaceDbChat

    @property
    def interfaceDbComponent(self):
        return self._get_service("chat").interfaceDbComponent

    @property
    def featureCode(self) -> Optional[str]:
        fc = getattr(self._context, "feature_code", None)
        if fc and str(fc).strip():
            return str(fc).strip()
        w = self.workflow
        if w and hasattr(w, "feature") and w.feature:
            return getattr(w.feature, "code", None)
        return getattr(w, "featureCode", None) if w else None

    def __getattr__(self, name: str):
        if name in ("allowedProviders", "preferredProviders", "currentUserLanguage"):
            return getattr(self.workflow, name, None) if self.workflow else None
        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")


class AiService:
    """AI service with core operations integrated."""

    def __init__(self, context, get_service: Callable[[str], Any]) -> None:
        """Initialize with ServiceCenterContext and service resolver.
        
        Args:
            context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow
            get_service: Callable to resolve dependency services by key
        """
        self.services = _ServicesAdapter(context, get_service)
        self._get_service = get_service
        self.aiObjects = None
        self.extractionService = None

    def _initializeSubmodules(self):
        """Initialize all submodules after aiObjects is ready."""
        if self.aiObjects is None:
            raise RuntimeError("aiObjects must be initialized before initializing submodules")
        
        if self.extractionService is None:
            logger.info("Initializing ExtractionService via service center...")
            self.extractionService = self._get_service("extraction")

        # Initialize new submodules
        from .subResponseParsing import ResponseParser
        from .subDocumentIntents import DocumentIntentAnalyzer
        from .subContentExtraction import ContentExtractor
        from .subStructureGeneration import StructureGenerator
        from .subStructureFilling import StructureFiller
        from .subAiCallLooping import AiCallLooper
        
        if not hasattr(self, 'responseParser'):
            logger.info("Initializing ResponseParser...")
            self.responseParser = ResponseParser(self.services)
        
        if not hasattr(self, 'intentAnalyzer'):
            logger.info("Initializing DocumentIntentAnalyzer...")
            self.intentAnalyzer = DocumentIntentAnalyzer(self.services, self)
        
        if not hasattr(self, 'contentExtractor'):
            logger.info("Initializing ContentExtractor...")
            self.contentExtractor = ContentExtractor(self.services, self, self.intentAnalyzer)
        
        if not hasattr(self, 'structureGenerator'):
            logger.info("Initializing StructureGenerator...")
            self.structureGenerator = StructureGenerator(self.services, self)
        
        if not hasattr(self, 'structureFiller'):
            logger.info("Initializing StructureFiller...")
            self.structureFiller = StructureFiller(self.services, self)
        
        if not hasattr(self, 'aiCallLooper'):
            logger.info("Initializing AiCallLooper...")
            self.aiCallLooper = AiCallLooper(self.services, self, self.responseParser)

    async def callAi(self, request: AiCallRequest, progressCallback=None):
        """Router: handles content parts via extractionService, text context via interface.
        
        FAIL-SAFE BILLING at the source:
        1. Pre-flight check: validates billing context is complete (RAISES if not)
        2. Balance & provider check before AI call
        3. billingCallback on aiObjects: records one billing transaction per model call
           with exact provider + model name (set before AI call, invoked by _callWithModel)
        
        NEUTRALIZATION: If enabled, prompt text is neutralized before the AI call
        and placeholders in the response are rehydrated afterwards.
        """
        await self.ensureAiObjectsInitialized()

        # SPEECH_TEAMS: Dedicated pipeline, bypasses standard model selection
        if request.options and request.options.operationType == OperationTypeEnum.SPEECH_TEAMS:
            return await self._handleSpeechTeams(request)
        
        # FAIL-SAFE: Pre-flight billing validation (like 0 CHF credit card check)
        self._preflightBillingCheck()
        
        # Balance & provider permission checks
        await self._checkBillingBeforeAiCall()
        
        # Calculate effective allowedProviders: RBAC ∩ Workflow
        effectiveProviders = self._calculateEffectiveProviders()
        if effectiveProviders and request.options:
            request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})
            logger.debug(f"Effective allowedProviders for AI request: {effectiveProviders}")
        
        # Neutralize prompt if enabled (before AI call)
        _wasNeutralized = False
        _excludedDocs: List[str] = []
        if self._shouldNeutralize(request):
            request, _wasNeutralized, _excludedDocs = await self._neutralizeRequest(request)
            if _excludedDocs:
                logger.warning(f"Neutralization partial failures (continuing): {_excludedDocs}")
        
        logger.debug("callAi: neutralization phase done, starting main AI call")
        self.aiObjects.billingCallback = self._createBillingCallback()
        
        try:
            if hasattr(request, 'contentParts') and request.contentParts:
                response = await self.extractionService.processContentPartsWithAi(
                    request, self.aiObjects, progressCallback
                )
            else:
                response = await self.aiObjects.callWithTextContext(request)
        finally:
            self.aiObjects.billingCallback = None
        
        # Attach neutralization exclusion metadata if any parts failed
        if _excludedDocs and response:
            if not hasattr(response, 'metadata') or response.metadata is None:
                response.metadata = {}
            if isinstance(response.metadata, dict):
                response.metadata["neutralizationExcluded"] = _excludedDocs
            elif hasattr(response.metadata, '__dict__'):
                response.metadata.neutralizationExcluded = _excludedDocs
        
        return response

    async def callAiStream(self, request: AiCallRequest):
        """Streaming variant of callAi. Yields str deltas during generation, then final AiCallResponse.
        
        NEUTRALIZATION: If enabled, prompt text is neutralized before streaming.
        Rehydration happens on the final AiCallResponse (not on individual str deltas).
        """
        await self.ensureAiObjectsInitialized()
        self._preflightBillingCheck()
        await self._checkBillingBeforeAiCall()

        effectiveProviders = self._calculateEffectiveProviders()
        if effectiveProviders and request.options:
            request.options = request.options.model_copy(update={'allowedProviders': effectiveProviders})

        # Neutralize prompt if enabled (before streaming)
        _wasNeutralized = False
        _excludedDocs: List[str] = []
        if self._shouldNeutralize(request):
            request, _wasNeutralized, _excludedDocs = await self._neutralizeRequest(request)
            if _excludedDocs:
                logger.warning(f"Neutralization partial failures in stream (continuing): {_excludedDocs}")

        logger.debug("callAiStream: neutralization phase done, starting main AI stream")
        self.aiObjects.billingCallback = self._createBillingCallback()
        try:
            async for chunk in self.aiObjects.callWithTextContextStream(request):
                if not isinstance(chunk, str):
                    if _excludedDocs:
                        if not hasattr(chunk, 'metadata') or chunk.metadata is None:
                            chunk.metadata = {}
                        if isinstance(chunk.metadata, dict):
                            chunk.metadata["neutralizationExcluded"] = _excludedDocs
                        elif hasattr(chunk.metadata, '__dict__'):
                            chunk.metadata.neutralizationExcluded = _excludedDocs
                yield chunk
        finally:
            self.aiObjects.billingCallback = None

    async def callEmbedding(self, texts: List[str]) -> AiCallResponse:
        """Generate embeddings while respecting allowedProviders."""
        await self.ensureAiObjectsInitialized()
        options = AiCallOptions(operationType=OperationTypeEnum.EMBEDDING)
        effectiveProviders = self._calculateEffectiveProviders()
        if effectiveProviders:
            options.allowedProviders = effectiveProviders
        self.aiObjects.billingCallback = self._createBillingCallback()
        try:
            return await self.aiObjects.callEmbedding(texts, options)
        finally:
            self.aiObjects.billingCallback = None

    # =========================================================================
    # SPEECH_TEAMS: Dedicated handler for Teams Meeting AI analysis
    # Bypasses standard model selection. Uses a fixed fast model.
    # =========================================================================

    async def _handleSpeechTeams(self, request: AiCallRequest):
        """
        Dedicated handler for SPEECH_TEAMS operation type.
        Bypasses standard model selection and uses a fixed fast model optimized
        for low-latency meeting transcript analysis.
        
        The handler:
        1. Selects a fixed fast model (no model selector)
        2. Builds a specialized system prompt for meeting analysis
        3. Calls the model with structured JSON output
        4. Returns a SpeechTeamsResponse wrapped in AiCallResponse
        
        Args:
            request: AiCallRequest with:
                - prompt: User-configured system prompt (from FeatureInstance.config.aiSystemPrompt)
                - context: Accumulated transcript segments to analyze
                - options.metadata: Optional dict with "botName" key
        
        Returns:
            AiCallResponse with content as JSON string (SpeechTeamsResponse format)
        """
        from modules.datamodels.datamodelAi import AiCallResponse, AiModelCall, AiCallOptions, PriorityEnum

        startTime = time.time()

        # Billing pre-flight (SPEECH_TEAMS also needs billing)
        self._preflightBillingCheck()
        await self._checkBillingBeforeAiCall()

        # 1. Select a fixed fast model (bypass model selector)
        model = self._getSpeechTeamsModel()
        if not model:
            return AiCallResponse(
                content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": "No suitable model available for SPEECH_TEAMS", "detectedIntent": "none"}),
                modelName="error",
                provider="unknown",
                priceCHF=0.0,
                processingTime=0.0,
                bytesSent=0,
                bytesReceived=0,
                errorCount=1
            )

        # 2. Build specialized system prompt
        metadata = {}
        if hasattr(request.options, 'allowedProviders') and request.options.allowedProviders:
            # Reuse allowedProviders field as metadata carrier if set (backward compat)
            pass
        botName = metadata.get("botName", "AI Assistant")
        
        # Extract botName from context if embedded as header
        contextText = request.context or ""
        if contextText.startswith("BOT_NAME:"):
            lines = contextText.split("\n", 1)
            botName = lines[0].replace("BOT_NAME:", "").strip()
            contextText = lines[1] if len(lines) > 1 else ""

        userSystemPrompt = request.prompt or ""
        systemPrompt = self._buildSpeechTeamsSystemPrompt(userSystemPrompt, botName)

        # 3. Build messages
        messages = [
            {"role": "system", "content": systemPrompt},
            {"role": "user", "content": contextText}
        ]

        # 4. Call model directly (no failover loop -- single fast model)
        modelCall = AiModelCall(
            messages=messages,
            model=model,
            options=AiCallOptions(
                operationType=OperationTypeEnum.SPEECH_TEAMS,
                priority=PriorityEnum.SPEED,
                temperature=0.3,
                resultFormat="json"
            )
        )

        # Set billing callback
        self.aiObjects.billingCallback = self._createBillingCallback()
        
        try:
            inputBytes = len((systemPrompt + contextText).encode('utf-8'))
            modelResponse = await model.functionCall(modelCall)

            if not modelResponse.success:
                raise ValueError(f"SPEECH_TEAMS model call failed: {modelResponse.error}")

            content = modelResponse.content
            outputBytes = len(content.encode('utf-8'))
            processingTime = time.time() - startTime
            priceCHF = model.calculatepriceCHF(processingTime, inputBytes, outputBytes)

            response = AiCallResponse(
                content=content,
                modelName=model.name,
                provider=model.connectorType,
                priceCHF=priceCHF,
                processingTime=processingTime,
                bytesSent=inputBytes,
                bytesReceived=outputBytes,
                errorCount=0
            )

            # Record billing
            if self.aiObjects.billingCallback:
                try:
                    self.aiObjects.billingCallback(response)
                except Exception as e:
                    logger.error(f"BILLING: Failed to record billing for SPEECH_TEAMS: {e}")

            logger.info(f"SPEECH_TEAMS call completed: model={model.name}, time={processingTime:.2f}s, cost={priceCHF:.4f} CHF")
            return response

        except Exception as e:
            processingTime = time.time() - startTime
            logger.error(f"SPEECH_TEAMS call failed: {e}")
            return AiCallResponse(
                content=json.dumps({"shouldRespond": False, "responseText": None, "reasoning": f"Error: {str(e)}", "detectedIntent": "none"}),
                modelName=model.name if model else "error",
                provider=model.connectorType if model else "unknown",
                priceCHF=0.0,
                processingTime=processingTime,
                bytesSent=0,
                bytesReceived=0,
                errorCount=1
            )
        finally:
            self.aiObjects.billingCallback = None

    def _getSpeechTeamsModel(self):
        """
        Get the fixed fast model for SPEECH_TEAMS.
        Prioritizes: GPT-4o-mini > Claude Haiku > any fast model with DATA_ANALYSE capability.
        Returns the AiModel instance or None.
        """
        from modules.aicore.aicoreModelRegistry import modelRegistry

        availableModels = modelRegistry.getAvailableModels()
        if not availableModels:
            logger.error("SPEECH_TEAMS: No models available in registry")
            return None

        # Priority list of preferred models for SPEECH_TEAMS (fast + cheap)
        _preferredModelNames = [
            "gpt-4o-mini",          # OpenAI: fast, cheap, good at JSON
            "claude-3-5-haiku",     # Anthropic: fast, cheap
            "gpt-4o",              # OpenAI: fallback to quality model
            "claude-sonnet-4-5",   # Anthropic: fallback
        ]

        # Try preferred models in order
        for preferredName in _preferredModelNames:
            for model in availableModels:
                if preferredName in model.name.lower() and model.functionCall and model.isAvailable:
                    logger.info(f"SPEECH_TEAMS: Selected preferred model '{model.name}' ({model.displayName})")
                    return model

        # Fallback: pick fastest available model with DATA_ANALYSE capability
        _dataAnalyseModels = []
        for model in availableModels:
            if not model.functionCall or not model.isAvailable:
                continue
            for opRating in model.operationTypes:
                if opRating.operationType == OperationTypeEnum.DATA_ANALYSE:
                    _dataAnalyseModels.append((model, opRating.rating))
                    break

        if _dataAnalyseModels:
            # Sort by speed rating (descending) then cost (ascending)
            _dataAnalyseModels.sort(key=lambda x: (-x[0].speedRating, x[0].costPer1kTokensInput))
            bestModel = _dataAnalyseModels[0][0]
            logger.info(f"SPEECH_TEAMS: Selected fallback model '{bestModel.name}' (speed={bestModel.speedRating})")
            return bestModel

        # Last resort: first available model
        for model in availableModels:
            if model.functionCall and model.isAvailable:
                logger.warning(f"SPEECH_TEAMS: Using last-resort model '{model.name}'")
                return model

        return None

    def _buildSpeechTeamsSystemPrompt(self, userSystemPrompt: str, botName: str) -> str:
        """
        Build the specialized system prompt for SPEECH_TEAMS meeting analysis.
        Combines a fixed base prompt with user-configurable instructions.
        """
        # Extract first name for examples (e.g. "Nyla" from "Nyla Larsson")
        botFirstName = botName.split()[0] if " " in botName else botName

        basePrompt = f"""Du bist "{botName}", ein AI-Teilnehmer in einem Microsoft Teams Meeting.
Analysiere das folgende Transkript und entscheide, ob du antworten sollst.

SPRACHE: Das Transkript kann in verschiedenen Sprachen sein. Antworte immer in der Sprache des letzten Sprechers der dich angesprochen hat. Wenn jemand sagt "let's talk German" oder "sprich deutsch", wechsle die Sprache entsprechend.

WICHTIG - SPRACHERKENNUNG: Das Transkript stammt aus einer automatischen Spracherkennung (Live Captions).
Dein Name "{botFirstName}" kann VERZERRT transkribiert werden, z.B. als aehnlich klingende Varianten
(z.B. "{botFirstName}" koennte als "Naila", "Neela", "Nila", "Sheila" etc. erscheinen).
Wenn ein Wort im Transkript PHONETISCH AEHNLICH zu "{botFirstName}" klingt und im Kontext einer Anrede steht, bist du gemeint.

WANN ANTWORTEN:

REGEL 1 (HOECHSTE PRIORITAET - NUR wenn direkt angesprochen):
Antworte NUR wenn dein Name "{botFirstName}" (oder phonetisch aehnliche Varianten durch Spracherkennung) DIREKT im aktuellsten Transkript-Segment vorkommt.
Beispiele wo du antworten MUSST: "{botFirstName}, was denkst du?", "Hey {botFirstName}", "{botFirstName} please introduce yourself"
Beispiele wo du NICHT antworten darfst: Jemand spricht ueber ein Thema ohne dich zu adressieren.

REGEL 2 (NUR bei direkter Frage an dich):
Wenn jemand eine Frage DIREKT AN DICH stellt (mit deinem Namen), beantworte sie.
Antworte NICHT auf allgemeine Fragen in der Runde, die nicht an dich gerichtet sind.

REGEL 3 (NICHT ANTWORTEN - sehr wichtig):
- Wenn Teilnehmer miteinander sprechen ohne dich zu adressieren: NICHT antworten
- Wenn die Konversation nicht an dich gerichtet ist: NICHT antworten
- Wenn du bereits auf dieselbe Frage geantwortet hast: NICHT nochmal antworten
- Wenn du nicht sicher bist ob du gemeint bist: NICHT antworten
- Im Zweifel: shouldRespond = false

ANTWORT-STIL (wenn du antwortest):
- Direkt und konkret antworten, KEINE Floskeln
- NICHT mit "Hallo [Name]" anfangen wenn du bereits begruessst hast
- NICHT "Ich bin {botName} und ich bin hier um zu helfen" wiederholen
- NICHT frueheres wiederholen das du schon gesagt hast
- Max 1-2 Saetze, praezise auf den Punkt
- Sieh dir an was du (markiert als [YOU]) bereits gesagt hast und wiederhole es NICHT
- KEINE reinen Absichtssaetze wie "Ich werde ...", "Ich kann ...", "Gerne ...".
  Liefere direkt den eigentlichen Inhalt in der gleichen Antwort.

WENN DER USER DICH BITTET ETWAS VORZULESEN / ZUSAMMENZUFASSEN:
- Gib IMMER sofort die Zusammenfassung aus (nicht nur ankündigen).
- Falls Vorlesen gewuenscht ist, setze zusaetzlich ein "readAloud"-Kommando mit dem Text.

KANAL-AUSWAHL (Voice vs Chat) - Je nach Anfrage unterschiedlich antworten:
- Du kannst pro Anfrage festlegen, ob deine Antwort per Voice (TTS), per Chat, oder beides erfolgt.
- Wenn jemand sagt "schreib das in den Chat", "schreib die Zusammenfassung in den Chat", "poste das im Chat": 
  - responseChannels: ["voice", "chat"]
  - responseTextForVoice: Kurze Bestaetigung (z.B. "Ich schreibe die Zusammenfassung jetzt in den Chat")
  - responseTextForChat: Der eigentliche Inhalt (z.B. die vollstaendige Zusammenfassung)
- Wenn jemand sagt "sag mir das", "lies das vor", "sprich das aus": 
  - responseChannels: ["voice"] oder ["voice","chat"] je nach Kontext
  - responseTextForVoice: Der zu sprechende Text
- Wenn jemand sagt "nur im Chat", "schreib nur": responseChannels: ["chat"]
- Wenn keine Kanal-Praeferenz erkennbar: responseChannels weglassen (Config entscheidet), responseText verwenden.

STOP-ERKENNUNG:
Wenn jemand dich bittet aufzuhoeren, still zu sein, zu stoppen, oder nicht mehr zu reden
(in JEDER Sprache, z.B. "{botFirstName} stop", "{botFirstName} sei still", "{botFirstName} halt", "{botFirstName} be quiet",
"{botFirstName} shut up", "{botFirstName} arrete", etc.), dann setze detectedIntent auf "stop" und
shouldRespond auf false. Du musst NICHT antworten wenn jemand dich stoppt."""

        # Append user-configured instructions if provided
        if userSystemPrompt and userSystemPrompt.strip():
            basePrompt += f"\n\nZUSAETZLICHE ANWEISUNGEN:\n{userSystemPrompt.strip()}"

        basePrompt += f"""

KOMMANDOS: Du kannst optionale Aktions-Kommandos ausfuehren lassen.
Verfuegbare Kommandos (im "commands" Array):
- {{"action": "toggleTranscript", "params": {{"enable": true/false}}}} — Transkription ein-/ausschalten
- {{"action": "sendChat", "params": {{"text": "Nachricht"}}}} — Zusaetzliche Nachricht in den Chat schreiben (unabhaengig von responseText)
- {{"action": "readAloud", "params": {{"text": "Text zum Vorlesen"}}}} — Einen bestimmten Text vorlesen (unabhaengig von responseText)
- {{"action": "changeLanguage", "params": {{"language": "en-US"}}}} — Kommunikationssprache aendern (z.B. "de-DE", "en-US", "fr-FR")

Verwende Kommandos NUR wenn explizit darum gebeten wird (z.B. "schalte die Transkription ein", "schreib das in den Chat", "lies das vor", "sprich Englisch").

WICHTIG: Antworte IMMER als valides JSON in exakt diesem Format:
{{
    "shouldRespond": true/false,
    "responseText": "Deine Antwort hier" oder null (Standard fuer beide Kanäle),
    "responseTextForVoice": optional - Text nur fuer TTS/Voice (z.B. kurze Bestaetigung),
    "responseTextForChat": optional - Text nur fuer Chat (z.B. lange Zusammenfassung),
    "responseChannels": optional - ["voice"], ["chat"] oder ["voice","chat"] je nach User-Anfrage,
    "reasoning": "Kurze Begruendung deiner Entscheidung",
    "detectedIntent": "addressed" | "question" | "proactive" | "stop" | "none",
    "commands": [] oder null
}}

detectedIntent-Werte:
- "addressed": {botName} wurde direkt angesprochen
- "question": Eine allgemeine Frage wurde gestellt
- "proactive": Du hast einen wertvollen proaktiven Beitrag
- "stop": Der User bittet {botName} aufzuhoeren/still zu sein (in jeder Sprache)
- "none": Kein Handlungsbedarf"""

        return basePrompt
    
    # =========================================================================
    # NEUTRALIZATION: Centralized prompt neutralization / response rehydration
    # =========================================================================

    async def _hasNeutralizationModel(self) -> bool:
        """Fast check: is at least one model available for NEUTRALIZATION_TEXT
        given the current effective provider list?  No AI call is made."""
        try:
            from modules.aicore.aicoreModelRegistry import modelRegistry
            from modules.aicore.aicoreModelSelector import modelSelector as _modSel
            from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum

            _models = modelRegistry.getAvailableModels()
            _providers = self._calculateEffectiveProviders()
            if _providers:
                _models = [m for m in _models if m.connectorType in _providers]
            _opts = AiCallOptions(operationType=OperationTypeEnum.NEUTRALIZATION_TEXT)
            _failover = _modSel.getFailoverModelList("x", "", _opts, _models)
            return bool(_failover)
        except Exception as _e:
            logger.warning(f"_hasNeutralizationModel check failed: {_e}")
            return True

    def _shouldNeutralize(self, request: AiCallRequest) -> bool:
        """Check if this AI request should have neutralization applied.

        OR-logic across three sources (any True → neutralize):
        1. Feature-Instance config (NeutralizationConfig.enabled)
        2. Workflow/Session (context.requireNeutralization)
        3. Per-request (request.requireNeutralization)

        No source can override another's True with False.
        Neutralization calls themselves (NEUTRALIZATION_TEXT / NEUTRALIZATION_IMAGE)
        are never re-neutralized (recursion guard).
        """
        try:
            if not request.prompt and not request.messages and not request.context:
                return False

            _opType = request.options.operationType if request.options else None
            if _opType in (OperationTypeEnum.NEUTRALIZATION_TEXT, OperationTypeEnum.NEUTRALIZATION_IMAGE):
                return False

            _sources = []

            # Source 1: Feature-Instance config
            _neutralSvc = self._get_service("neutralization")
            if _neutralSvc and hasattr(_neutralSvc, 'getConfig'):
                _config = _neutralSvc.getConfig()
                if _config and getattr(_config, 'enabled', False):
                    _sources.append("featureInstance")

            # Source 2: Workflow / Session context
            _ctx = getattr(self.services, '_context', None)
            _ctxFlag = getattr(_ctx, "requireNeutralization", None) if _ctx else None
            if _ctxFlag is True:
                _sources.append("context")

            # Source 3: Per-request flag
            if request.requireNeutralization is True:
                _sources.append("request")

            if _sources:
                logger.debug(f"Neutralization required by: {', '.join(_sources)}")
                request.requireNeutralization = True
                return True

            return False
        except Exception as e:
            logger.error(f"_shouldNeutralize check failed: {e} — defaulting to False")
            return False

    async def _neutralizeRequest(self, request: AiCallRequest) -> Tuple[AiCallRequest, bool, List[str]]:
        """Neutralize the prompt text and messages in an AiCallRequest (async).

        Returns (modifiedRequest, wasNeutralized, excludedDocs).

        Uses ``processTextAsync`` which calls AI with NEUTRALIZATION_TEXT
        to identify PII, protected logic and names — then applies regex as
        supplementary pass.

        FAILSAFE behaviour when ``requireNeutralization is True`` (explicit):
        - Service unavailable → raises (caller must not send raw data to AI).
        - Prompt neutralization fails → raises.
        - Individual message neutralization fails → message is **removed**
          (not kept in original form) and noted in excludedDocs.

        When neutralization is only config-driven (requireNeutralization is
        None) the behaviour is softer: failures are logged and originals are
        kept — but a warning is emitted.
        """
        _hardMode = request.requireNeutralization is True
        excludedDocs: List[str] = []

        neutralSvc = self._get_service("neutralization")
        if not neutralSvc or not hasattr(neutralSvc, 'processTextAsync'):
            if _hardMode:
                raise RuntimeError("Neutralization explicitly required but service unavailable — AI call BLOCKED")
            logger.warning("Neutralization required by config but service unavailable — continuing without neutralization")
            excludedDocs.append("Neutralization service unavailable; prompt sent un-neutralized")
            return request, False, excludedDocs

        _wasNeutralized = False
        _snapshots: list = []

        if _hardMode:
            _hasNeutModel = await self._hasNeutralizationModel()
            if not _hasNeutModel:
                raise RuntimeError(
                    "Neutralisierung ist aktiviert, aber es ist kein AI-Modell für "
                    "NEUTRALIZATION_TEXT verfügbar. Bitte ein Modell für Neutralisierung "
                    "freigeben oder die Neutralisierung deaktivieren."
                )

        if request.prompt:
            logger.debug(f"_neutralizeRequest: neutralizing prompt ({len(request.prompt)} chars)")
            try:
                result = await neutralSvc.processTextAsync(request.prompt)
                if result and result.get("neutralized_text"):
                    request.prompt = result["neutralized_text"]
                    _wasNeutralized = True
                    _snapshots.append(("Prompt", result["neutralized_text"], len(result.get("mapping", {}))))
                    logger.debug("Neutralized prompt in AiCallRequest")
                else:
                    if _hardMode:
                        raise RuntimeError(f"Prompt neutralization returned empty — AI call BLOCKED (hard mode)")
                    logger.warning("Neutralization of prompt returned no neutralized_text — sending original prompt")
                    excludedDocs.append("Prompt neutralization failed; original prompt used")
            except RuntimeError:
                raise
            except Exception as e:
                if _hardMode:
                    raise RuntimeError(f"Prompt neutralization failed — AI call BLOCKED: {e}") from e
                logger.warning(f"Neutralization of prompt failed: {e} — sending original prompt")
                excludedDocs.append(f"Prompt neutralization error: {e}")

        if request.context:
            logger.debug(f"_neutralizeRequest: neutralizing context ({len(request.context)} chars)")
            try:
                result = await neutralSvc.processTextAsync(request.context)
                if result and result.get("neutralized_text"):
                    request.context = result["neutralized_text"]
                    _wasNeutralized = True
                    _snapshots.append(("Kontext", result["neutralized_text"], len(result.get("mapping", {}))))
                    logger.debug("Neutralized context in AiCallRequest")
                else:
                    if _hardMode:
                        raise RuntimeError("Context neutralization returned empty — AI call BLOCKED (hard mode)")
                    logger.warning("Neutralization of context returned no neutralized_text — sending original context")
                    excludedDocs.append("Context neutralization failed; original context used")
            except RuntimeError:
                raise
            except Exception as e:
                if _hardMode:
                    raise RuntimeError(f"Context neutralization failed — AI call BLOCKED: {e}") from e
                logger.warning(f"Neutralization of context failed: {e} — sending original context")
                excludedDocs.append(f"Context neutralization error: {e}")

        _msgCount = len(request.messages) if request.messages and isinstance(request.messages, list) else 0
        if _msgCount:
            logger.debug(f"_neutralizeRequest: neutralizing {_msgCount} message(s)")
        if request.messages and isinstance(request.messages, list):
            cleanMessages = []
            for idx, msg in enumerate(request.messages):
                content = msg.get("content") if isinstance(msg, dict) else None
                if content is None:
                    cleanMessages.append(msg)
                    continue
                if isinstance(content, str):
                    if not content:
                        cleanMessages.append(msg)
                        continue
                    try:
                        result = await neutralSvc.processTextAsync(content)
                        if result and result.get("neutralized_text"):
                            msg["content"] = result["neutralized_text"]
                            _wasNeutralized = True
                            _role = msg.get("role", "?")
                            _snapshots.append((f"Nachricht {idx+1} ({_role})", result["neutralized_text"], len(result.get("mapping", {}))))
                            cleanMessages.append(msg)
                        else:
                            if _hardMode:
                                raise RuntimeError(
                                    f"Neutralisierung von Nachricht {idx+1}/{_msgCount} schlug fehl "
                                    f"(leere Antwort). Konversation kann nicht sicher gesendet werden."
                                )
                            logger.warning(f"Neutralization of message[{idx}] returned no neutralized_text — keeping original")
                            excludedDocs.append(f"Message[{idx}] neutralization failed; original kept")
                            cleanMessages.append(msg)
                    except RuntimeError:
                        raise
                    except Exception as e:
                        if _hardMode:
                            raise RuntimeError(
                                f"Neutralisierung von Nachricht {idx+1}/{_msgCount} schlug fehl: {e}. "
                                f"Konversation kann nicht sicher gesendet werden."
                            ) from e
                        logger.warning(f"Neutralization of message[{idx}] failed: {e} — keeping original")
                        excludedDocs.append(f"Message[{idx}] neutralization error: {e}")
                        cleanMessages.append(msg)
                elif isinstance(content, list):
                    _cleanParts = []
                    for _partIdx, _part in enumerate(content):
                        if not isinstance(_part, dict):
                            _cleanParts.append(_part)
                            continue
                        _partType = _part.get("type", "")
                        if _partType == "text" and _part.get("text"):
                            try:
                                _result = await neutralSvc.processTextAsync(_part["text"])
                                if _result and _result.get("neutralized_text"):
                                    _part["text"] = _result["neutralized_text"]
                                    _wasNeutralized = True
                                    _role = msg.get("role", "?")
                                    _snapshots.append((f"Nachricht {idx+1}.{_partIdx+1} ({_role})", _result["neutralized_text"], len(_result.get("mapping", {}))))
                                    _cleanParts.append(_part)
                                else:
                                    if _hardMode:
                                        raise RuntimeError(
                                            f"Neutralisierung von Nachricht {idx+1}, Teil {_partIdx+1} "
                                            f"schlug fehl (leere Antwort)."
                                        )
                                    _cleanParts.append(_part)
                            except RuntimeError:
                                raise
                            except Exception as e:
                                if _hardMode:
                                    raise RuntimeError(
                                        f"Neutralisierung von Nachricht {idx+1}, Teil {_partIdx+1} "
                                        f"schlug fehl: {e}"
                                    ) from e
                                _cleanParts.append(_part)
                        elif _partType == "image_url":
                            if _hardMode:
                                logger.warning(f"Message[{idx}].content[{_partIdx}] image_url — REMOVING (neutralization active)")
                                excludedDocs.append(f"Message[{idx}].content[{_partIdx}] image removed (neutralization)")
                            else:
                                _cleanParts.append(_part)
                        else:
                            _cleanParts.append(_part)
                    if _cleanParts:
                        msg["content"] = _cleanParts
                        cleanMessages.append(msg)
                    else:
                        cleanMessages.append(msg)
                else:
                    cleanMessages.append(msg)
            request.messages = cleanMessages
            logger.debug(f"_neutralizeRequest: messages done, {len(cleanMessages)} kept of {_msgCount}")

        if hasattr(request, 'contentParts') and request.contentParts:
            _cleanParts = []
            for _cpIdx, _cp in enumerate(request.contentParts):
                _tg = getattr(_cp, 'typeGroup', '') or ''
                _data = getattr(_cp, 'data', '') or ''
                if _tg in ('text', 'table') and _data:
                    try:
                        _result = await neutralSvc.processTextAsync(str(_data))
                        if _result and _result.get("neutralized_text"):
                            _cp.data = _result["neutralized_text"]
                            _wasNeutralized = True
                            _snapshots.append((f"Inhalt {_cpIdx+1} ({_tg})", _result["neutralized_text"], len(_result.get("mapping", {}))))
                            _cleanParts.append(_cp)
                        else:
                            if _hardMode:
                                logger.warning(f"ContentPart[{_cpIdx}] neutralization empty — REMOVING")
                                excludedDocs.append(f"ContentPart[{_cpIdx}] removed")
                            else:
                                _cleanParts.append(_cp)
                    except Exception as e:
                        if _hardMode:
                            logger.warning(f"ContentPart[{_cpIdx}] neutralization error — REMOVING: {e}")
                            excludedDocs.append(f"ContentPart[{_cpIdx}] error: {e}")
                        else:
                            _cleanParts.append(_cp)
                elif _tg == 'image':
                    if _hardMode:
                        logger.warning(f"ContentPart[{_cpIdx}] image — REMOVING (neutralization active)")
                        excludedDocs.append(f"ContentPart[{_cpIdx}] image removed")
                    else:
                        _cleanParts.append(_cp)
                else:
                    _cleanParts.append(_cp)
            request.contentParts = _cleanParts
            logger.debug(f"_neutralizeRequest: contentParts done, {len(_cleanParts)} kept")

        if _snapshots and _wasNeutralized:
            try:
                neutralSvc.clearSnapshots()
                for _label, _text, _phCount in _snapshots:
                    neutralSvc.saveSnapshot(_label, _text, _phCount)
                logger.debug(f"_neutralizeRequest: saved {len(_snapshots)} snapshot(s)")
            except Exception as _snapErr:
                logger.warning(f"_neutralizeRequest: could not save snapshots: {_snapErr}")

        logger.info(f"_neutralizeRequest complete: neutralized={_wasNeutralized}, excluded={len(excludedDocs)}")
        return request, _wasNeutralized, excludedDocs

    def _rehydrateResponse(self, responseText: str) -> str:
        """Replace neutralization placeholders with original values in AI response."""
        if not responseText:
            return responseText
        try:
            neutralSvc = self._get_service("neutralization")
            if not neutralSvc or not hasattr(neutralSvc, 'resolveText'):
                return responseText
            resolved = neutralSvc.resolveText(responseText)
            return resolved if resolved else responseText
        except Exception as e:
            logger.warning(f"Response rehydration failed: {e}")
            return responseText

    def _preflightBillingCheck(self) -> None:
        """
        Pre-flight billing validation - like a 0 CHF credit card authorization check.
        
        Validates that ALL required billing context is present and that a billing
        transaction CAN be recorded. This dry-run check catches missing context
        BEFORE an expensive AI call starts.
        
        FAIL-SAFE: This method RAISES if billing context is incomplete.
        An AI call without billing context MUST NOT proceed.
        
        Raises:
            BillingContextError: If billing context is incomplete or invalid
        """
        if not self.services:
            raise BillingContextError("No service context available - cannot bill AI call")
        
        user = getattr(self.services, 'user', None)
        if not user:
            raise BillingContextError("No user context - cannot bill AI call")
        
        mandateId = getattr(self.services, 'mandateId', None)
        if not mandateId:
            raise BillingContextError(
                f"No mandateId in service context for user {user.id} - cannot bill AI call. "
                "Every AI call MUST have a mandate context for billing."
            )
        
        # Validate billing service can be created
        featureInstanceId = getattr(self.services, 'featureInstanceId', None)
        featureCode = getattr(self.services, 'featureCode', None)
        
        try:
            billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
        except Exception as e:
            raise BillingContextError(
                f"Cannot create billing service for user {user.id}, mandate {mandateId}: {e}"
            )
        
        # Dry-run: verify billing service can check balance (DB accessible)
        try:
            billingService.checkBalance(0.0)
        except Exception as e:
            raise BillingContextError(
                f"Billing system not accessible for mandate {mandateId}: {e}"
            )
        
        logger.debug(
            f"Pre-flight billing check PASSED: user={user.id}, mandate={mandateId}, "
            f"feature={featureCode or 'none'}, instance={featureInstanceId or 'none'}"
        )

    async def _checkBillingBeforeAiCall(self) -> None:
        """
        Check billing status before making an AI call.
        
        FAIL-SAFE: Context validation is done in _preflightBillingCheck() which is
        called first. This method handles balance and provider permission checks.
        
        Verifies:
        1. User has sufficient balance (for prepay models)
        2. Provider is allowed for the user (via RBAC)
        
        Raises:
            InsufficientBalanceException: If balance is insufficient
            ProviderNotAllowedException: If provider is not allowed
            BillingContextError: If billing check fails unexpectedly
        """
        # Context is already validated by _preflightBillingCheck()
        user = self.services.user
        mandateId = self.services.mandateId
        featureInstanceId = getattr(self.services, 'featureInstanceId', None)
        featureCode = getattr(self.services, 'featureCode', None)
        
        try:
            # Get billing service
            billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
            
            # Check balance (estimate typical AI call cost)
            estimatedCost = 0.01  # ~1 cent CHF minimum
            balanceCheck = billingService.checkBalance(estimatedCost)
            
            if not balanceCheck.allowed:
                reason = balanceCheck.reason or ""

                if reason in SUBSCRIPTION_REASONS:
                    from modules.datamodels.datamodelSubscription import SubscriptionStatusEnum
                    statusMap = {
                        "SUBSCRIPTION_PAYMENT_REQUIRED": SubscriptionStatusEnum.PAST_DUE,
                        "SUBSCRIPTION_EXPIRED": SubscriptionStatusEnum.EXPIRED,
                        "SUBSCRIPTION_INACTIVE": SubscriptionStatusEnum.EXPIRED,
                    }
                    raise SubscriptionInactiveException(
                        status=statusMap.get(reason, SubscriptionStatusEnum.EXPIRED),
                        mandateId=str(mandateId),
                    )

                balance_str = f"{(balanceCheck.currentBalance or 0):.2f}"
                logger.warning(
                    f"AI billing check failed (mandate pool): mandate={mandateId} user={user.id} "
                    f"poolBalance={balance_str} CHF required~={estimatedCost:.4f} CHF reason={reason}"
                )
                ulabel = (getattr(user, "email", None) or getattr(user, "username", None) or str(user.id))
                maybeEmailMandatePoolExhausted(
                    str(mandateId),
                    str(user.id),
                    str(ulabel),
                    float(balanceCheck.currentBalance or 0.0),
                    float(estimatedCost),
                )
                raise InsufficientBalanceException.fromBalanceCheck(
                    balanceCheck,
                    str(mandateId),
                    float(estimatedCost),
                )

            balance_str = f"{(balanceCheck.currentBalance or 0):.2f}"
            logger.debug(f"Billing check passed: Balance {balance_str} CHF")
            
            # Check if at least one provider is allowed (RBAC check)
            rbacAllowedProviders = billingService.getallowedProviders()
            if not rbacAllowedProviders:
                logger.warning(f"No AI providers allowed for user {user.id} in mandate {mandateId}")
                raise ProviderNotAllowedException(
                    provider="any",
                    message="Keine AI-Provider fuer Ihre Rolle freigegeben. Kontaktieren Sie Ihren Administrator."
                )
            
            # Check automation-level allowedProviders restriction
            automationAllowedProviders = getattr(self.services, 'allowedProviders', None)
            if automationAllowedProviders:
                effectiveProviders = [p for p in automationAllowedProviders if p in rbacAllowedProviders]
                if not effectiveProviders:
                    logger.warning(f"No providers available after automation restriction. "
                                   f"Automation allows: {automationAllowedProviders}, "
                                   f"RBAC allows: {rbacAllowedProviders}")
                    raise ProviderNotAllowedException(
                        provider="any",
                        message="Die konfigurierten AI-Provider dieser Automation sind fuer Ihre Rolle nicht freigegeben."
                    )
                logger.debug(f"Automation provider check passed: {effectiveProviders}")
            
            # Check if preferred providers (from UI multiselect) are allowed
            preferredProviders = getattr(self.services, 'preferredProviders', None)
            if preferredProviders:
                for provider in preferredProviders:
                    if provider not in rbacAllowedProviders:
                        logger.warning(f"Preferred provider {provider} not allowed for user {user.id}")
                        raise ProviderNotAllowedException(
                            provider=provider,
                            message=f"Der gewaehlte Provider '{provider}' ist fuer Ihre Rolle nicht freigegeben."
                        )
                logger.debug(f"All preferred providers are allowed: {preferredProviders}")
            
            logger.debug(f"Provider check passed: {len(rbacAllowedProviders)} providers allowed")
            
        except SubscriptionInactiveException:
            raise
        except InsufficientBalanceException:
            raise
        except ProviderNotAllowedException:
            raise
        except BillingContextError:
            raise
        except Exception as e:
            logger.error(f"BILLING FAIL-SAFE: Billing check failed with unexpected error: {e}")
            raise BillingContextError(f"Billing check failed: {e}")

    def _createBillingCallback(self):
        """
        Create a billing callback for interfaceAiObjects._callWithModel().
        
        Returns a function that records one billing transaction per individual model call.
        Each transaction contains the exact provider name AND model name.
        Also writes an AI audit log entry for compliance tracking.
        
        For a 200 MB document processed with N parallel AI calls (possibly different models),
        this creates N separate billing transactions - one per model call.
        """
        user = self.services.user
        mandateId = self.services.mandateId
        featureInstanceId = getattr(self.services, 'featureInstanceId', None)
        featureCode = getattr(self.services, 'featureCode', None)
        
        workflowId = None
        workflow = getattr(self.services, 'workflow', None)
        if workflow and hasattr(workflow, 'id'):
            workflowId = workflow.id
        
        billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
        
        def _billingCallback(response) -> None:
            """Record billing transaction + AI audit entry."""
            if not response:
                return

            provider = getattr(response, 'provider', None) or 'unknown'
            modelName = getattr(response, 'modelName', None) or 'unknown'
            basePriceCHF = getattr(response, 'priceCHF', 0.0)
            hasError = getattr(response, 'errorCount', 0) > 0
            processingTime = getattr(response, 'processingTime', None)
            bytesSent = getattr(response, 'bytesSent', None)
            bytesReceived = getattr(response, 'bytesReceived', None)

            if not hasError and basePriceCHF and basePriceCHF > 0:
                try:
                    billingService.recordUsage(
                        priceCHF=basePriceCHF,
                        workflowId=workflowId,
                        aicoreProvider=provider,
                        aicoreModel=modelName,
                        description=f"AI: {modelName}",
                        processingTime=processingTime,
                        bytesSent=bytesSent,
                        bytesReceived=bytesReceived,
                        errorCount=getattr(response, 'errorCount', None)
                    )
                    logger.debug(
                        f"Billed model call: {basePriceCHF:.4f} CHF, "
                        f"provider={provider}, model={modelName}, mandate={mandateId}"
                    )
                except Exception as e:
                    logger.error(
                        f"BILLING: Failed to record transaction! "
                        f"Cost={basePriceCHF:.4f} CHF, user={user.id}, mandate={mandateId}, "
                        f"provider={provider}, model={modelName}, error={e}"
                    )

            try:
                from modules.shared.aiAuditLogger import aiAuditLogger

                contentOut = getattr(response, 'content', None)
                metadata = getattr(response, 'metadata', None) or {}
                tokensUsed = metadata.get('tokensUsed') if isinstance(metadata, dict) else None

                aiAuditLogger.logAiCall(
                    userId=user.id,
                    mandateId=mandateId or "",
                    aiProvider=provider,
                    aiModel=modelName,
                    username=getattr(user, 'username', None),
                    featureInstanceId=featureInstanceId,
                    featureCode=featureCode,
                    operationType=metadata.get('operationType') if isinstance(metadata, dict) else None,
                    tokensInput=tokensUsed.get('input') if isinstance(tokensUsed, dict) else None,
                    tokensOutput=tokensUsed.get('output') if isinstance(tokensUsed, dict) else None,
                    processingTimeMs=int(processingTime * 1000) if processingTime else None,
                    priceCHF=basePriceCHF if basePriceCHF else None,
                    contentOutput=str(contentOut)[:500] if contentOut else None,
                    success=not hasError,
                    errorMessage=str(getattr(response, 'errorMessage', None)) if hasError else None,
                )
            except Exception as e:
                logger.warning(f"AI audit log failed (non-critical): {e}")
        
        return _billingCallback

    def _calculateEffectiveProviders(self) -> Optional[List[str]]:
        """
        Calculate effective allowed providers: RBAC ∩ Workflow.
        
        RBAC is master - only RBAC-permitted providers can ever be used.
        If workflow specifies allowedProviders, intersect with RBAC.
        If no workflow providers, use all RBAC-permitted providers.
        
        Returns:
            List of effective allowed providers, or None if no filtering needed
        """
        try:
            user = getattr(self.services, 'user', None)
            mandateId = getattr(self.services, 'mandateId', None)
            
            if not user or not mandateId:
                return None
            
            # Get RBAC-permitted providers (master list)
            # Note: getBillingService is imported at module level from mainServiceBilling
            featureInstanceId = getattr(self.services, 'featureInstanceId', None)
            featureCode = getattr(self.services, 'featureCode', None)
            billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
            rbacProviders = billingService.getallowedProviders()
            
            if not rbacProviders:
                logger.warning("No RBAC-permitted providers found")
                return None
            
            # Get workflow-specified providers (optional filter)
            workflowProviders = getattr(self.services, 'allowedProviders', None)
            
            if workflowProviders:
                # Intersect: only providers that are both RBAC-permitted AND workflow-allowed
                effectiveProviders = [p for p in workflowProviders if p in rbacProviders]
                logger.debug(f"Provider filter: RBAC={rbacProviders}, Workflow={workflowProviders}, Effective={effectiveProviders}")
            else:
                # No workflow filter - use all RBAC-permitted providers
                effectiveProviders = rbacProviders
                logger.debug(f"Provider filter: RBAC={rbacProviders} (no workflow filter)")
            
            return effectiveProviders if effectiveProviders else None
            
        except Exception as e:
            logger.warning(f"Error calculating effective providers: {e}")
            return None

    async def ensureAiObjectsInitialized(self):
        """Ensure aiObjects is initialized and submodules are ready."""
        if self.aiObjects is None:
            logger.info("Lazy initializing AiObjects...")
            self.aiObjects = await AiObjects.create()
            logger.info("AiObjects initialization completed")
        self._initializeSubmodules()

    @classmethod
    async def create(cls, servicesHub) -> "AiService":
        """Create AiService from a ServiceHub instance."""
        from modules.serviceCenter import getService
        from modules.serviceCenter.context import ServiceCenterContext
        ctx = ServiceCenterContext(
            user=servicesHub.user,
            mandate_id=servicesHub.mandateId,
            feature_instance_id=servicesHub.featureInstanceId,
            workflow=getattr(servicesHub, "workflow", None),
        )
        return getService("ai", ctx)

    # Helper methods
    
    def _buildPromptWithPlaceholders(self, prompt: str, placeholders: Optional[Dict[str, str]]) -> str:
        """
        Build full prompt by replacing placeholders with their content.
        Uses the new {{KEY:placeholder}} format.
        
        Args:
            prompt: The base prompt template
            placeholders: Dictionary of placeholder key-value pairs
            
        Returns:
            Prompt with placeholders replaced
        """
        if not placeholders:
            return prompt
        
        full_prompt = prompt
        for placeholder, content in placeholders.items():
            # Skip if content is None or empty
            if content is None:
                continue
            # Replace {{KEY:placeholder}}
            full_prompt = full_prompt.replace(f"{{{{KEY:{placeholder}}}}}", str(content))
        
        return full_prompt
    
    async def _analyzePromptAndCreateOptions(self, prompt: str) -> AiCallOptions:
        """Analyze prompt to determine appropriate AiCallOptions parameters."""
        try:
            # Get dynamic enum values from Pydantic models
            operationTypes = [e.value for e in OperationTypeEnum]
            priorities = [e.value for e in PriorityEnum]
            processingModes = [e.value for e in ProcessingModeEnum]
            
            # Create analysis prompt for AI to determine operation type and parameters
            analysisPrompt = f"""
You are an AI operation analyzer. Analyze the following prompt and determine the most appropriate operation type and parameters.

PROMPT TO ANALYZE:
{self.services.utils.sanitizePromptContent(prompt, 'userinput')}

Based on the prompt content, determine:
1. operationType: Choose the most appropriate from: {', '.join(operationTypes)}
2. priority: Choose from: {', '.join(priorities)}
3. processingMode: Choose from: {', '.join(processingModes)}
4. compressPrompt: true/false (true for story-like prompts, false for structured prompts with JSON/schemas)
5. compressContext: true/false (true to summarize context, false to process fully)

Respond with ONLY a JSON object in this exact format:
{{
    "operationType": "dataAnalyse",
    "priority": "balanced", 
    "processingMode": "basic",
    "compressPrompt": true,
    "compressContext": true
}}
"""

            # Use AI to analyze the prompt
            request = AiCallRequest(
                prompt=analysisPrompt,
                options=AiCallOptions(
                    operationType=OperationTypeEnum.DATA_ANALYSE,
                    priority=PriorityEnum.SPEED,
                    processingMode=ProcessingModeEnum.BASIC,
                    compressPrompt=True,
                    compressContext=False
                )
            )
            
            response = await self.callAi(request)
            
            # Parse AI response using structured parsing with AiCallOptions model
            try:
                # Use parseJsonWithModel to parse response into AiCallOptions (handles enum conversion automatically)
                analysis = parseJsonWithModel(response.content, AiCallOptions)
                return analysis
            except Exception as e:
                logger.warning(f"Failed to parse AI analysis response: {e}")
                
        except Exception as e:
            logger.warning(f"Prompt analysis failed: {e}")
        
        # Fallback to default options
        return AiCallOptions(
            operationType=OperationTypeEnum.DATA_ANALYSE,
            priority=PriorityEnum.BALANCED,
            processingMode=ProcessingModeEnum.BASIC
        )
    
    async def callAiWithLooping(
        self,
        prompt: str,
        options: AiCallOptions,
        debugPrefix: str = "ai_call",
        promptBuilder: Optional[callable] = None,
        promptArgs: Optional[Dict[str, Any]] = None,
        operationId: Optional[str] = None,
        userPrompt: Optional[str] = None,
        contentParts: Optional[List[ContentPart]] = None,  # ARCHITECTURE: Support ContentParts for large content
        useCaseId: Optional[str] = None  # REQUIRED: Explicit use case ID for generic looping system
    ) -> str:
        """Public method: Delegate to AiCallLooper for AI calls with looping support."""
        return await self.aiCallLooper.callAiWithLooping(
            prompt, options, debugPrefix, promptBuilder, promptArgs, operationId, userPrompt, contentParts, useCaseId
        )
    
    # JSON merging logic moved to subJsonResponseHandling.py
    
    def _extractSectionsFromResponse(
        self,
        result: str,
        iteration: int,
        debugPrefix: str,
        allSections: List[Dict[str, Any]] = None,
        accumulationState: Optional[JsonAccumulationState] = None
    ) -> Tuple[List[Dict[str, Any]], bool, Optional[Dict[str, Any]], Optional[JsonAccumulationState]]:
        """Delegate to ResponseParser."""
        return self.responseParser.extractSectionsFromResponse(
            result, iteration, debugPrefix, allSections, accumulationState
        )
    
    def _shouldContinueGeneration(
        self,
        allSections: List[Dict[str, Any]],
        iteration: int,
        wasJsonComplete: bool,
        rawResponse: str = None
    ) -> bool:
        """Delegate to ResponseParser."""
        return self.responseParser.shouldContinueGeneration(
            allSections, iteration, wasJsonComplete, rawResponse
        )
    
    def _extractDocumentMetadata(
        self,
        parsedResult: Dict[str, Any]
    ) -> Optional[Dict[str, Any]]:
        """Delegate to ResponseParser."""
        return self.responseParser.extractDocumentMetadata(parsedResult)
    
    def _buildFinalResultFromSections(
        self,
        allSections: List[Dict[str, Any]],
        documentMetadata: Optional[Dict[str, Any]] = None
    ) -> str:
        """Delegate to ResponseParser."""
        return self.responseParser.buildFinalResultFromSections(allSections, documentMetadata)

    # Public API Methods

    # Planning AI Call
    async def callAiPlanning(
        self,
        prompt: str,
        placeholders: Optional[List[PromptPlaceholder]] = None,
        debugType: Optional[str] = None
        ) -> str:
        """
        Planning AI call for task planning, action planning, action selection, etc.
        Always uses static parameters optimized for planning tasks.
        
        Args:
            prompt: The planning prompt
            placeholders: Optional list of placeholder replacements
            debugType: Optional debug file type identifier (e.g., 'taskplan', 'dynamic', 'intentanalysis')
                      If not provided, defaults to 'plan'
            
        Returns:
            Planning JSON response
        """
        await self.ensureAiObjectsInitialized()
        
        # Planning calls always use static parameters
        options = AiCallOptions(
            operationType=OperationTypeEnum.PLAN,
            priority=PriorityEnum.QUALITY,
            processingMode=ProcessingModeEnum.DETAILED,
            compressPrompt=False,
            compressContext=False
        )
        
        # Build full prompt with placeholders
        if placeholders:
            placeholdersDict = {p.label: p.content for p in placeholders}
            fullPrompt = self._buildPromptWithPlaceholders(prompt, placeholdersDict)
        else:
            fullPrompt = prompt
        
        # Root-cause fix: planning must return raw single-shot JSON, not section-based output
        request = AiCallRequest(
            prompt=fullPrompt,
            context="",
            options=options
        )
        
        # Debug: persist prompt/response for analysis with context-specific naming
        debugPrefix = debugType if debugType else "plan"
        self.services.utils.writeDebugFile(fullPrompt, f"{debugPrefix}_prompt")
        response = await self.callAi(request)  # Use callAi to ensure stats are stored
        result = response.content or ""
        self.services.utils.writeDebugFile(result, f"{debugPrefix}_response")
        return result

    # Helper methods for callAiContent refactoring
    
    async def _handleImageGeneration(
        self,
        prompt: str,
        options: AiCallOptions,
        title: Optional[str],
        parentOperationId: Optional[str]
    ) -> AiResponse:
        """Handle IMAGE_GENERATE operation type using image generation path."""
        from modules.serviceCenter.services.serviceGeneration.paths.imagePath import ImageGenerationPath
        
        imagePath = ImageGenerationPath(self.services)
        
        # Extract format from options
        format = options.resultFormat or "png"
        
        return await imagePath.generateImages(
            userPrompt=prompt,
            format=format,
            title=title,
            parentOperationId=parentOperationId
        )
    
    async def _handleWebOperation(
        self,
        prompt: str,
        options: AiCallOptions,
        opType: OperationTypeEnum,
        aiOperationId: str
    ) -> AiResponse:
        """Handle WEB_SEARCH_DATA and WEB_CRAWL operation types."""
        self.services.chat.progressLogUpdate(aiOperationId, 0.4, f"Calling AI for {opType.name}")
        
        request = AiCallRequest(
            prompt=prompt,  # Raw JSON prompt - connector will parse it
            context="",
            options=options
        )
        
        response = await self.callAi(request)
        
        if not response.content:
            errorMsg = f"No content returned from {opType.name}: {response.content}"
            logger.error(f"Error in {opType.name}: {errorMsg}")
            self.services.chat.progressLogFinish(aiOperationId, False)
            raise ValueError(errorMsg)
        
        metadata = AiResponseMetadata(
            operationType=opType.value
        )
        
        # Note: Stats are now stored centrally in callAi() - no need to duplicate here
        
        self.services.chat.progressLogUpdate(aiOperationId, 0.9, f"{opType.name} completed")
        self.services.chat.progressLogFinish(aiOperationId, True)
        
        # Preserve metadata from response if available (e.g., results_with_content from Tavily)
        # Check if response has metadata attribute (AiCallResponse from callAi)
        if hasattr(response, 'metadata') and response.metadata:
            # If metadata is a dict, store it in additionalData
            if isinstance(response.metadata, dict):
                if not metadata.additionalData:
                    metadata.additionalData = {}
                metadata.additionalData.update(response.metadata)
            # If metadata is an object with attributes, extract them
            elif hasattr(response.metadata, '__dict__'):
                if not metadata.additionalData:
                    metadata.additionalData = {}
                for key, value in response.metadata.__dict__.items():
                    if not key.startswith('_'):
                        metadata.additionalData[key] = value
        
        return AiResponse(
            content=response.content,
            metadata=metadata
        )
    
    def _getIntentForDocument(
        self,
        docId: str,
        intents: Optional[List[DocumentIntent]]
    ) -> Optional[DocumentIntent]:
        """Find DocumentIntent for given documentId."""
        if not intents:
            return None
        for intent in intents:
            if intent.documentId == docId:
                return intent
        return None
    
    async def clarifyDocumentIntents(
        self,
        documents: List[ChatDocument],
        userPrompt: str,
        actionParameters: Dict[str, Any],
        parentOperationId: str
    ) -> List[DocumentIntent]:
        """Public method: Delegate to DocumentIntentAnalyzer."""
        return await self.intentAnalyzer.clarifyDocumentIntents(
            documents, userPrompt, actionParameters, parentOperationId
        )
    
    async def extractAndPrepareContent(
        self,
        documents: List[ChatDocument],
        documentIntents: List[DocumentIntent],
        parentOperationId: str
    ) -> List[ContentPart]:
        """Public method: Delegate to ContentExtractor."""
        return await self.contentExtractor.extractAndPrepareContent(
            documents, documentIntents, parentOperationId, self._getIntentForDocument
        )
    
    async def generateStructure(
        self,
        userPrompt: str,
        contentParts: List[ContentPart],
        outputFormat: Optional[str] = None,
        parentOperationId: str = None
    ) -> Dict[str, Any]:
        """Public method: Delegate to StructureGenerator."""
        return await self.structureGenerator.generateStructure(
            userPrompt, contentParts, outputFormat, parentOperationId
        )
    
    async def fillStructure(
        self,
        structure: Dict[str, Any],
        contentParts: List[ContentPart],
        userPrompt: str,
        parentOperationId: str
    ) -> Dict[str, Any]:
        """Public method: Delegate to StructureFiller."""
        return await self.structureFiller.fillStructure(
            structure, contentParts, userPrompt, parentOperationId
        )
    
    async def renderResult(
        self,
        filledStructure: Dict[str, Any],
        outputFormat: str,
        language: str,
        title: str,
        userPrompt: str,
        parentOperationId: str
    ) -> List[RenderedDocument]:
        """
        Phase 5E: Rendert gefüllte Struktur zum Ziel-Format.
        Jedes Dokument wird einzeln gerendert, jeder Renderer kann 1..n Dokumente zurückgeben.
        
        Render filled structure to documents.
        Per-document format and language are extracted from structure (validated in State 3).
        The outputFormat and language parameters are only used as global fallbacks.
        Multiple documents can have different formats and languages.
        
        Args:
            filledStructure: Gefüllte Struktur mit elements
            outputFormat: Ziel-Format (pdf, docx, html, etc.) - Global fallback
            language: Language (global fallback) - Per-document language extracted from structure
            title: Dokument-Titel
            userPrompt: User-Anfrage
            parentOperationId: Parent Operation-ID für ChatLog-Hierarchie
        
        Returns:
            List of RenderedDocument objects.
            Jedes RenderedDocument repräsentiert ein gerendertes Dokument (Hauptdokument oder unterstützende Datei)
        """
        # Language comes from structure (per-document), validated in State 3
        # This parameter is only used as global fallback if structure validation fails
        # Use validated currentUserLanguage as fallback (always valid)
        if not language:
            language = self._getUserLanguage() if hasattr(self, '_getUserLanguage') else (self.services.currentUserLanguage if hasattr(self.services, 'currentUserLanguage') else 'en')
        # Erstelle Operation-ID für Rendering
        renderOperationId = f"{parentOperationId}_rendering"
        
        # Starte ChatLog mit Parent-Referenz
        self.services.chat.progressLogStart(
            renderOperationId,
            "Content Rendering",
            "Rendering",
            f"Rendering to {outputFormat} format",
            parentOperationId=parentOperationId
        )
        
        try:
            generationService = self._get_service("generation")
            
            # renderReport verarbeitet jetzt jedes Dokument einzeln
            # und gibt Liste von (documentData, mimeType, filename) zurück
            renderedDocuments = await generationService.renderReport(
                filledStructure,
                outputFormat,
                language,  # Pass language (global fallback, per-document extracted in renderReport)
                title,
                userPrompt,
                self,
                parentOperationId=renderOperationId  # Parent-Referenz für ChatLog-Hierarchie
            )
            
            # ChatLog abschließen
            self.services.chat.progressLogFinish(renderOperationId, True)
            
            return renderedDocuments
            
        except Exception as e:
            self.services.chat.progressLogFinish(renderOperationId, False)
            logger.error(f"Error in _renderResult: {str(e)}")
            raise
    
    def _shouldSkipContentPart(
        self,
        part: ContentPart
    ) -> bool:
        """Check if ContentPart should be skipped (already structured JSON)."""
        if part.typeGroup == "structure" and part.mimeType == "application/json":
            if part.metadata.get("skipExtraction", False):
                logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (skipExtraction=True)")
                return True
            try:
                if isinstance(part.data, str):
                    jsonData = json.loads(part.data)
                    if isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData):
                        logger.debug(f"Skipping already-structured JSON ContentPart {part.id} (contains documents/sections)")
                        return True
            except Exception:
                pass  # Not JSON, continue processing
        return False

    async def callAiContent(
        self,
        prompt: str,
        options: AiCallOptions,
        contentParts: Optional[List[ContentPart]] = None,
        documentList: Optional[Any] = None,  # DocumentReferenceList
        documentIntents: Optional[List[DocumentIntent]] = None,
        outputFormat: Optional[str] = None,
        title: Optional[str] = None,
        parentOperationId: Optional[str] = None,
        generationIntent: Optional[str] = None  # NEW: Explicit intent from action (skips detection)
    ) -> AiResponse:
        """
        Unified AI content generation with explicit intent requirement.
        
        All AI-Actions (ai.process, ai.generateDocument, etc.) route through here.
        They differ only in parameters, not in logic.
        
        Args:
            prompt: The main prompt for the AI call
            options: AI call configuration options (REQUIRED - operationType must be set)
            contentParts: Optional list of already-extracted content parts (preferred)
            documentList: Optional DocumentReferenceList (wird zu ChatDocuments konvertiert)
            documentIntents: Optional list of DocumentIntent objects (wird erstellt wenn nicht vorhanden)
            outputFormat: Optional output format for document generation (e.g., 'pdf', 'docx', 'xlsx')
            title: Optional title for generated documents
            parentOperationId: Optional parent operation ID for hierarchical logging
            generationIntent: REQUIRED explicit intent ("document" | "code" | "image") from action.
                             NO auto-detection - actions must explicitly specify intent.
        
        Returns:
            AiResponse with content, metadata, and optional documents
        """
        await self.ensureAiObjectsInitialized()
        
        # Erstelle Operation-ID
        workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
        aiOperationId = f"ai_content_{workflowId}_{int(time.time())}"
        
        # Starte Progress-Tracking mit Parent-Referenz
        formatDisplay = outputFormat if outputFormat else "auto-determined"
        self.services.chat.progressLogStart(
            aiOperationId,
            "AI content processing",
            "Content Processing",
            f"Format: {formatDisplay}",
            parentOperationId=parentOperationId
        )
        
        try:
            # outputFormat is optional - if None, formats determined from prompt by AI
            # No default fallback here - let AI service handle it
            
            opType = getattr(options, "operationType", None)
            if not opType:
                options.operationType = OperationTypeEnum.DATA_GENERATE
                opType = OperationTypeEnum.DATA_GENERATE
            
            # Route zu Operation-spezifischen Handlern
            if opType == OperationTypeEnum.IMAGE_GENERATE:
                # Image generation - route to image path
                return await self._handleImageGeneration(prompt, options, title, parentOperationId)
            
            if opType == OperationTypeEnum.WEB_SEARCH_DATA or opType == OperationTypeEnum.WEB_CRAWL:
                return await self._handleWebOperation(prompt, options, opType, aiOperationId)
            
            # Data generation - REQUIRES explicit generationIntent
            if opType == OperationTypeEnum.DATA_GENERATE:
                if not generationIntent:
                    errorMsg = (
                        "generationIntent is required for DATA_GENERATE operation. "
                        "Actions must explicitly specify 'document' or 'code' intent. "
                        "No auto-detection - use qualified actions (ai.generateDocument, ai.generateCode)."
                    )
                    logger.error(errorMsg)
                    self.services.chat.progressLogFinish(aiOperationId, False)
                    raise ValueError(errorMsg)
                
                # Route based on explicit intent (no auto-detection, no fallback)
                if generationIntent == "code":
                    # Route to code generation path
                    return await self._handleCodeGeneration(
                        prompt=prompt,
                        options=options,
                        contentParts=contentParts,
                        outputFormat=outputFormat,
                        title=title,
                        parentOperationId=parentOperationId
                    )
                else:
                    # Route to document generation path (existing behavior)
                    return await self._handleDocumentGeneration(
                        prompt=prompt,
                        options=options,
                        documentList=documentList,
                        documentIntents=documentIntents,
                        contentParts=contentParts,
                        outputFormat=outputFormat,
                        title=title,
                        parentOperationId=parentOperationId
                    )
            
            # DATA_EXTRACT: Extract content from documents and process with AI (no structure generation)
            if opType == OperationTypeEnum.DATA_EXTRACT:
                return await self._handleDataExtraction(
                    prompt=prompt,
                    options=options,
                    documentList=documentList,
                    documentIntents=documentIntents,
                    contentParts=contentParts,
                    outputFormat=outputFormat,
                    title=title,
                    parentOperationId=parentOperationId
                )
            
            # Other operation types (DATA_ANALYSE, etc.) - not supported
            errorMsg = f"Unsupported operation type: {opType}. Supported types: IMAGE_GENERATE, DATA_GENERATE, DATA_EXTRACT"
            logger.error(errorMsg)
            self.services.chat.progressLogFinish(aiOperationId, False)
            raise ValueError(errorMsg)
            
        except Exception as e:
            logger.error(f"Error in callAiContent: {str(e)}")
            self.services.chat.progressLogFinish(aiOperationId, False)
            raise
    
    async def _handleDataExtraction(
        self,
        prompt: str,
        options: AiCallOptions,
        documentList: Optional[Any],
        documentIntents: Optional[List[DocumentIntent]],
        contentParts: Optional[List[ContentPart]],
        outputFormat: str,
        title: str,
        parentOperationId: Optional[str]
    ) -> AiResponse:
        """
        Handle DATA_EXTRACT: Extract content from documents, then process with AI.

        - AUTOMATION mode: No intent analysis. The passed prompt is used as extractionPrompt
          for every document and for the final AI call (exact prompt preserved).
        - DYNAMIC mode: Intent analysis (clarifyDocumentIntents) runs first; extraction and
          processing use the intents and AI-derived extractionPrompt.
        """
        import time
        
        # Create operation ID
        workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
        extractOperationId = f"data_extract_{workflowId}_{int(time.time())}"
        
        # Start progress tracking
        self.services.chat.progressLogStart(
            extractOperationId,
            "Data Extraction",
            "Extraction",
            f"Format: {outputFormat}",
            parentOperationId=parentOperationId
        )
        
        try:
            # Step 1: Get documents from documentList
            documents = []
            if documentList:
                documents = self.services.chat.getChatDocumentsFromDocumentList(documentList)
            
            # Filter: Remove original documents if already covered by pre-extracted JSONs
            # (to prevent duplicate ContentParts - pre-extracted JSONs contain already extracted ContentParts)
            if documents:
                # Step 1: Identify all original document IDs covered by pre-extracted JSONs
                originalDocIdsCoveredByPreExtracted = set()
                for doc in documents:
                    preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc)
                    if preExtracted:
                        originalDocId = preExtracted["originalDocument"]["id"]
                        originalDocIdsCoveredByPreExtracted.add(originalDocId)
                        logger.debug(f"Found pre-extracted JSON {doc.id} covering original document {originalDocId}")
                
                # Step 2: Filter documents - remove originals covered by pre-extracted JSONs
                filteredDocuments = []
                for doc in documents:
                    preExtracted = self.intentAnalyzer.resolvePreExtractedDocument(doc)
                    if preExtracted:
                        filteredDocuments.append(doc)  # Keep pre-extracted JSON
                    elif doc.id in originalDocIdsCoveredByPreExtracted:
                        logger.info(f"Skipping original document {doc.id} ({doc.fileName}) - already covered by pre-extracted JSON")
                    else:
                        filteredDocuments.append(doc)  # Keep regular document
                
                documents = filteredDocuments  # Use filtered list
            
            # Step 2: Document intents – AUTOMATION uses exact prompt; DYNAMIC uses intent analysis
            if not documentIntents and documents:
                workflowMode = getattr(self.services.workflow, "workflowMode", None) if self.services.workflow else None
                if workflowMode == WorkflowModeEnum.WORKFLOW_AUTOMATION:
                    # Automation: no intent AI call – use the given prompt as extractionPrompt for every document
                    documentIntents = [
                        DocumentIntent(
                            documentId=doc.id,
                            intents=["extract"],
                            extractionPrompt=prompt,
                            reasoning="Automation mode: use exact prompt from action",
                        )
                        for doc in documents
                    ]
                    logger.debug("DATA_EXTRACT in AUTOMATION mode: using exact prompt for all documents (no intent analysis)")
                else:
                    documentIntents = await self.clarifyDocumentIntents(
                        documents,
                        prompt,
                        {"outputFormat": outputFormat},
                        extractOperationId
                    )
            
            # Step 3: Extract and prepare content (NO AI - pure extraction) - REQUIRED for all documents
            if documents:
                preparedContentParts = await self.extractAndPrepareContent(
                    documents,
                    documentIntents or [],
                    extractOperationId
                )
                
                # Merge with provided contentParts (if any)
                if contentParts:
                    for part in contentParts:
                        if part.metadata.get("skipExtraction", False):
                            part.metadata.setdefault("contentFormat", "extracted")
                            part.metadata.setdefault("isPreExtracted", True)
                    preparedContentParts.extend(contentParts)
                
                contentParts = preparedContentParts
            
            # Step 4: Process contentParts with AI via ExtractionService
            # Always use processContentPartsWithAi – it handles text vs image parts correctly:
            # - Text parts → text models (with chunking if needed)
            # - Image parts → Vision AI (proper image_url content blocks)
            # No manual contentText concatenation or token estimation needed.
            if not contentParts:
                raise ValueError("No content extracted from documents")
            
            # Filter out empty content parts (e.g. PDF container with 0 bytes) that would
            # produce garbage AI responses and pollute the merged result.
            nonEmptyParts = [p for p in contentParts if p.data and len(p.data.strip()) > 0]
            if not nonEmptyParts:
                raise ValueError("No non-empty content parts to process")
            
            self.services.utils.writeDebugFile(prompt, "data_extract_prompt")
            extractionService = self.services.extraction
            aiRequest = AiCallRequest(
                prompt=prompt,
                context="",
                options=options,
                contentParts=nonEmptyParts,
            )
            aiResponse = await extractionService.processContentPartsWithAi(
                aiRequest, self.aiObjects
            )
            _respText = aiResponse.content if isinstance(aiResponse.content, str) else (aiResponse.content.decode("utf-8", errors="replace") if aiResponse.content else "")
            self.services.utils.writeDebugFile(_respText, "data_extract_response")
            
            # Create response document
            resultDocument = DocumentData(
                documentName=f"{title or 'extracted_data'}.{outputFormat}",
                documentData=aiResponse.content.encode('utf-8') if isinstance(aiResponse.content, str) else aiResponse.content,
                mimeType=f"text/{outputFormat}" if outputFormat in ["txt", "json", "csv"] else "application/octet-stream"
            )
            
            metadata = AiResponseMetadata(
                title=title or "Extracted Data",
                operationType=OperationTypeEnum.DATA_EXTRACT.value
            )
            
            self.services.chat.progressLogFinish(extractOperationId, True)
            
            return AiResponse(
                content=aiResponse.content if isinstance(aiResponse.content, str) else aiResponse.content.decode('utf-8', errors='replace'),
                metadata=metadata,
                documents=[resultDocument]
            )
            
        except Exception as e:
            logger.error(f"Error in data extraction: {str(e)}")
            self.services.chat.progressLogFinish(extractOperationId, False)
            raise
    
    async def _handleCodeGeneration(
        self,
        prompt: str,
        options: AiCallOptions,
        contentParts: Optional[List[ContentPart]],
        outputFormat: str,
        title: str,
        parentOperationId: Optional[str]
    ) -> AiResponse:
        """Handle code generation using code generation path."""
        from modules.serviceCenter.services.serviceGeneration.paths.codePath import CodeGenerationPath
        
        codePath = CodeGenerationPath(self.services)
        return await codePath.generateCode(
            userPrompt=prompt,
            outputFormat=outputFormat,
            contentParts=contentParts,
            title=title or "Generated Code",
            parentOperationId=parentOperationId
        )
    
    async def _handleDocumentGeneration(
        self,
        prompt: str,
        options: AiCallOptions,
        documentList: Optional[Any],
        documentIntents: Optional[List[DocumentIntent]],
        contentParts: Optional[List[ContentPart]],
        outputFormat: str,
        title: str,
        parentOperationId: Optional[str]
    ) -> AiResponse:
        """Handle document generation using document generation path."""
        from modules.serviceCenter.services.serviceGeneration.paths.documentPath import DocumentGenerationPath
        
        # Set compression options for document generation
        options.compressPrompt = False
        options.compressContext = False
        
        documentPath = DocumentGenerationPath(self.services)
        return await documentPath.generateDocument(
            userPrompt=prompt,
            documentList=documentList,
            documentIntents=documentIntents,
            contentParts=contentParts,
            outputFormat=outputFormat,
            title=title or "Generated Document",
            parentOperationId=parentOperationId
        )
            
    
    def _determineDocumentName(
        self,
        filledStructure: Dict[str, Any],
        outputFormat: str,
        title: Optional[str]
    ) -> str:
        """Bestimme Dokument-Namen aus Struktur oder Titel."""
        # Versuche aus Struktur zu extrahieren
        if isinstance(filledStructure, dict) and "documents" in filledStructure:
            docs = filledStructure["documents"]
            if isinstance(docs, list) and len(docs) > 0:
                firstDoc = docs[0]
                if isinstance(firstDoc, dict) and firstDoc.get("filename"):
                    return firstDoc["filename"]
        
        # Fallback zu Titel
        if title:
            sanitized = re.sub(r"[^a-zA-Z0-9._-]", "_", title)
            sanitized = re.sub(r"_+", "_", sanitized).strip("_")
            if sanitized:
                if not sanitized.lower().endswith(f".{outputFormat}"):
                    return f"{sanitized}.{outputFormat}"
                return sanitized
        
        return f"generated.{outputFormat}"

