From b33444e8919af3971371231712354f251a277b21 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 24 Mar 2026 16:39:25 +0100 Subject: [PATCH] unified data completed implementation --- .../features/commcoach/datamodelCommcoach.py | 24 -- .../commcoach/interfaceFeatureCommcoach.py | 28 -- modules/features/commcoach/mainCommcoach.py | 7 +- .../commcoach/routeFeatureCommcoach.py | 274 +-------------- .../features/commcoach/serviceCommcoach.py | 215 ++++++++---- .../serviceCommcoachContextRetrieval.py | 50 ++- .../commcoach/tests/test_datamodel.py | 1 - modules/interfaces/interfaceBootstrap.py | 7 + modules/interfaces/interfaceVoiceObjects.py | 119 ------- modules/migration/migrateVoiceAndDocuments.py | 316 ++++++++++++++++++ modules/routes/routeSecurityLocal.py | 134 +++++--- modules/routes/routeVoiceGoogle.py | 139 +++----- .../services/serviceAgent/mainServiceAgent.py | 84 ++--- .../services/serviceAi/mainServiceAi.py | 31 +- 14 files changed, 698 insertions(+), 731 deletions(-) create mode 100644 modules/migration/migrateVoiceAndDocuments.py diff --git a/modules/features/commcoach/datamodelCommcoach.py b/modules/features/commcoach/datamodelCommcoach.py index 090640c6..bd94f173 100644 --- a/modules/features/commcoach/datamodelCommcoach.py +++ b/modules/features/commcoach/datamodelCommcoach.py @@ -170,8 +170,6 @@ class CoachingUserProfile(BaseModel): userId: str = Field(description="Owner user ID") mandateId: str = Field(description="Mandate ID") instanceId: str = Field(description="Feature instance ID") - preferredLanguage: str = Field(default="de-DE") - preferredVoice: Optional[str] = Field(default=None, description="Google TTS voice name") dailyReminderTime: Optional[str] = Field(default=None, description="HH:MM format") dailyReminderEnabled: bool = Field(default=False) emailSummaryEnabled: bool = Field(default=True) @@ -205,26 +203,6 @@ class CoachingPersona(BaseModel): updatedAt: Optional[str] = Field(default=None) -# ============================================================================ -# Iteration 2: Documents -# ============================================================================ - -class CoachingDocument(BaseModel): - """A document attached to a coaching context.""" - id: str = Field(default_factory=lambda: str(uuid.uuid4())) - contextId: str = Field(description="FK to CoachingContext") - userId: str = Field(description="Owner user ID") - mandateId: str = Field(description="Mandate ID") - instanceId: Optional[str] = Field(default=None) - fileName: str = Field(description="Original file name") - mimeType: str = Field(default="application/octet-stream") - fileSize: int = Field(default=0) - extractedText: Optional[str] = Field(default=None, description="Text content extracted from file") - summary: Optional[str] = Field(default=None, description="AI-generated summary") - fileRef: Optional[str] = Field(default=None, description="Reference to file in storage") - createdAt: Optional[str] = Field(default=None) - - # ============================================================================ # Iteration 2: Badges / Gamification # ============================================================================ @@ -282,8 +260,6 @@ class UpdateTaskStatusRequest(BaseModel): class UpdateProfileRequest(BaseModel): - preferredLanguage: Optional[str] = None - preferredVoice: Optional[str] = None dailyReminderTime: Optional[str] = None dailyReminderEnabled: Optional[bool] = None emailSummaryEnabled: Optional[bool] = None diff --git a/modules/features/commcoach/interfaceFeatureCommcoach.py b/modules/features/commcoach/interfaceFeatureCommcoach.py index e612c6ba..825fca5d 100644 --- a/modules/features/commcoach/interfaceFeatureCommcoach.py +++ b/modules/features/commcoach/interfaceFeatureCommcoach.py @@ -269,34 +269,6 @@ class CommcoachObjects: from .datamodelCommcoach import CoachingPersona return self.db.recordDelete(CoachingPersona, personaId) - # ========================================================================= - # Documents - # ========================================================================= - - def getDocuments(self, contextId: str, userId: str) -> List[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - records = self.db.getRecordset(CoachingDocument, recordFilter={"contextId": contextId, "userId": userId}) - records.sort(key=lambda r: r.get("createdAt") or "", reverse=True) - return records - - def getDocument(self, documentId: str) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - records = self.db.getRecordset(CoachingDocument, recordFilter={"id": documentId}) - return records[0] if records else None - - def createDocument(self, data: Dict[str, Any]) -> Dict[str, Any]: - from .datamodelCommcoach import CoachingDocument - data["createdAt"] = getIsoTimestamp() - return self.db.recordCreate(CoachingDocument, data) - - def updateDocument(self, documentId: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]: - from .datamodelCommcoach import CoachingDocument - return self.db.recordModify(CoachingDocument, documentId, updates) - - def deleteDocument(self, documentId: str) -> bool: - from .datamodelCommcoach import CoachingDocument - return self.db.recordDelete(CoachingDocument, documentId) - # ========================================================================= # Badges # ========================================================================= diff --git a/modules/features/commcoach/mainCommcoach.py b/modules/features/commcoach/mainCommcoach.py index 69ac6b1c..e8abcee8 100644 --- a/modules/features/commcoach/mainCommcoach.py +++ b/modules/features/commcoach/mainCommcoach.py @@ -61,18 +61,13 @@ DATA_OBJECTS = [ { "objectKey": "data.feature.commcoach.CoachingUserProfile", "label": {"en": "User Profile", "de": "Benutzerprofil", "fr": "Profil utilisateur"}, - "meta": {"table": "CoachingUserProfile", "fields": ["id", "userId", "preferredLanguage"]} + "meta": {"table": "CoachingUserProfile", "fields": ["id", "userId", "dailyReminderEnabled"]} }, { "objectKey": "data.feature.commcoach.CoachingPersona", "label": {"en": "Coaching Persona", "de": "Coaching-Persona", "fr": "Persona coaching"}, "meta": {"table": "CoachingPersona", "fields": ["id", "key", "label", "gender"]} }, - { - "objectKey": "data.feature.commcoach.CoachingDocument", - "label": {"en": "Coaching Document", "de": "Coaching-Dokument", "fr": "Document coaching"}, - "meta": {"table": "CoachingDocument", "fields": ["id", "contextId", "fileName"]} - }, { "objectKey": "data.feature.commcoach.CoachingBadge", "label": {"en": "Coaching Badge", "de": "Coaching-Auszeichnung", "fr": "Badge coaching"}, diff --git a/modules/features/commcoach/routeFeatureCommcoach.py b/modules/features/commcoach/routeFeatureCommcoach.py index 9074d2ba..6d6eb44f 100644 --- a/modules/features/commcoach/routeFeatureCommcoach.py +++ b/modules/features/commcoach/routeFeatureCommcoach.py @@ -26,7 +26,7 @@ from .datamodelCommcoach import ( CoachingContext, CoachingContextStatus, CoachingSession, CoachingSessionStatus, CoachingMessage, CoachingMessageRole, CoachingMessageContentType, CoachingTask, CoachingTaskStatus, - CoachingPersona, CoachingDocument, CoachingBadge, + CoachingPersona, CoachingBadge, CreateContextRequest, UpdateContextRequest, SendMessageRequest, CreateTaskRequest, UpdateTaskRequest, UpdateTaskStatusRequest, UpdateProfileRequest, @@ -334,9 +334,8 @@ async def startSession( try: from modules.interfaces.interfaceVoiceObjects import getVoiceInterface voiceInterface = getVoiceInterface(context.user, mandateId) - profile = interface.getProfile(userId, instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" - voiceName = profile.get("preferredVoice") if profile else None + from .serviceCommcoach import _getUserVoicePrefs + language, voiceName = _getUserVoicePrefs(userId, mandateId) from .serviceCommcoach import _stripMarkdownForTts ttsResult = await voiceInterface.textToSpeech( text=_stripMarkdownForTts(greetingText), @@ -574,8 +573,8 @@ async def sendAudioStream( if not audioBody: raise HTTPException(status_code=400, detail="No audio data received") - profile = interface.getProfile(str(context.user.id), instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" + from .serviceCommcoach import _getUserVoicePrefs + language, _ = _getUserVoicePrefs(str(context.user.id), mandateId) contextId = session.get("contextId") service = CommcoachService(context.user, mandateId, instanceId) @@ -839,73 +838,6 @@ async def updateProfile( return {"profile": updated} -# ========================================================================= -# Voice Endpoints -# ========================================================================= - -@router.get("/{instanceId}/voice/languages") -@limiter.limit("30/minute") -async def getVoiceLanguages( - request: Request, - instanceId: str, - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - languagesResult = await voiceInterface.getAvailableLanguages() - languageList = languagesResult.get("languages", []) if isinstance(languagesResult, dict) else languagesResult - return {"languages": languageList} - - -@router.get("/{instanceId}/voice/voices") -@limiter.limit("30/minute") -async def getVoiceVoices( - request: Request, - instanceId: str, - language: str = "de-DE", - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - voicesResult = await voiceInterface.getAvailableVoices(language) - voiceList = voicesResult.get("voices", []) if isinstance(voicesResult, dict) else voicesResult - return {"voices": voiceList} - - -@router.post("/{instanceId}/voice/tts") -@limiter.limit("10/minute") -async def testVoice( - request: Request, - instanceId: str, - context: RequestContext = Depends(getRequestContext), -): - """TTS preview / voice test.""" - mandateId = _validateInstanceAccess(instanceId, context) - body = await request.json() - text = body.get("text", "Hallo, ich bin dein Coaching-Assistent.") - language = body.get("language", "de-DE") - voiceId = body.get("voiceId") - - from modules.interfaces.interfaceVoiceObjects import getVoiceInterface - voiceInterface = getVoiceInterface(context.user, mandateId) - - try: - result = await voiceInterface.textToSpeech(text=text, languageCode=language, voiceName=voiceId) - if result and isinstance(result, dict): - audioContent = result.get("audioContent") - if audioContent: - audioB64 = base64.b64encode( - audioContent if isinstance(audioContent, bytes) else audioContent.encode() - ).decode() - return {"success": True, "audio": audioB64, "format": "mp3", "text": text} - return {"success": False, "error": "TTS returned no audio"} - except Exception as e: - logger.error(f"Voice test failed: {e}") - raise HTTPException(status_code=500, detail=f"TTS test failed: {str(e)}") - - # ========================================================================= # Export Endpoints (Iteration 2) # ========================================================================= @@ -1074,202 +1006,6 @@ async def deletePersonaRoute( return {"deleted": True} -# ========================================================================= -# Document Endpoints (Iteration 2) -# ========================================================================= - -@router.get("/{instanceId}/contexts/{contextId}/documents") -@limiter.limit("60/minute") -async def listDocuments( - request: Request, - instanceId: str, - contextId: str, - context: RequestContext = Depends(getRequestContext), -): - _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - userId = str(context.user.id) - docs = interface.getDocuments(contextId, userId) - return {"documents": docs} - - -@router.post("/{instanceId}/contexts/{contextId}/documents") -@limiter.limit("10/minute") -async def uploadDocument( - request: Request, - instanceId: str, - contextId: str, - context: RequestContext = Depends(getRequestContext), -): - """Upload a document and bind it to a context. Stores file in Management DB.""" - mandateId = _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - userId = str(context.user.id) - - ctx = interface.getContext(contextId) - if not ctx: - raise HTTPException(status_code=404, detail="Context not found") - _validateOwnership(ctx, context) - - form = await request.form() - file = form.get("file") - if not file or not hasattr(file, "read"): - raise HTTPException(status_code=400, detail="No file uploaded") - - content = await file.read() - fileName = getattr(file, "filename", "document") - mimeType = getattr(file, "content_type", "application/octet-stream") - fileSize = len(content) - - if not content: - raise HTTPException(status_code=400, detail="Leere Datei hochgeladen") - - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface(currentUser=context.user) - fileItem, _dupType = mgmtInterface.saveUploadedFile(content, fileName) - fileRef = fileItem.id - - extractedText = _extractText(content, mimeType, fileName) - summary = None - if extractedText and len(extractedText.strip()) > 50: - try: - from .serviceCommcoach import CommcoachService - service = CommcoachService(context.user, mandateId, instanceId) - aiResp = await service._callAi( - "Du fasst Dokumente in 2-3 Saetzen zusammen.", - f"Fasse folgendes Dokument zusammen:\n\n{extractedText[:3000]}" - ) - if aiResp and aiResp.errorCount == 0 and aiResp.content: - summary = aiResp.content.strip() - except Exception as e: - logger.warning(f"Document summary failed: {e}") - - docData = CoachingDocument( - contextId=contextId, - userId=userId, - mandateId=mandateId, - instanceId=instanceId, - fileName=fileName, - mimeType=mimeType, - fileSize=fileSize, - extractedText=extractedText[:10000] if extractedText else None, - summary=summary, - fileRef=fileRef, - ).model_dump() - created = interface.createDocument(docData) - return {"document": created} - - -@router.delete("/{instanceId}/documents/{documentId}") -@limiter.limit("10/minute") -async def deleteDocumentRoute( - request: Request, - instanceId: str, - documentId: str, - context: RequestContext = Depends(getRequestContext), -): - mandateId = _validateInstanceAccess(instanceId, context) - interface = _getInterface(context, instanceId) - - doc = interface.getDocument(documentId) - if not doc: - raise HTTPException(status_code=404, detail="Document not found") - _validateOwnership(doc, context) - - fileRef = doc.get("fileRef") - if fileRef: - try: - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface( - currentUser=context.user, mandateId=mandateId, featureInstanceId=instanceId - ) - mgmtInterface.deleteFile(fileRef) - except Exception as e: - logger.warning(f"Failed to delete file {fileRef}: {e}") - - interface.deleteDocument(documentId) - return {"deleted": True} - - -def _extractText(content: bytes, mimeType: str, fileName: str) -> Optional[str]: - """Extract text from uploaded file content (TXT, MD, HTML, PDF, DOCX, XLSX, PPTX).""" - import io - - lowerName = fileName.lower() - try: - if mimeType in ("text/plain",) or lowerName.endswith(".txt"): - return content.decode("utf-8", errors="replace") - - if mimeType in ("text/markdown",) or lowerName.endswith(".md"): - return content.decode("utf-8", errors="replace") - - if mimeType in ("text/html",) or lowerName.endswith((".html", ".htm")): - from html.parser import HTMLParser - class _Strip(HTMLParser): - def __init__(self): - super().__init__() - self._parts: list[str] = [] - def handle_data(self, d): - self._parts.append(d) - def result(self): - return " ".join(self._parts) - parser = _Strip() - parser.feed(content.decode("utf-8", errors="replace")) - return parser.result() - - if "pdf" in mimeType or lowerName.endswith(".pdf"): - try: - from PyPDF2 import PdfReader - reader = PdfReader(io.BytesIO(content)) - return "".join(page.extract_text() or "" for page in reader.pages) - except ImportError: - logger.warning("PyPDF2 not installed, cannot extract PDF text") - return None - - if "wordprocessingml" in mimeType or lowerName.endswith(".docx"): - try: - from docx import Document - doc = Document(io.BytesIO(content)) - return "\n".join(p.text for p in doc.paragraphs if p.text) - except ImportError: - logger.warning("python-docx not installed, cannot extract DOCX text") - return None - - if "spreadsheetml" in mimeType or lowerName.endswith(".xlsx"): - try: - from openpyxl import load_workbook - wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True) - parts: list[str] = [] - for ws in wb.worksheets: - for row in ws.iter_rows(values_only=True): - cells = [str(c) for c in row if c is not None] - if cells: - parts.append("\t".join(cells)) - return "\n".join(parts) - except ImportError: - logger.warning("openpyxl not installed, cannot extract XLSX text") - return None - - if "presentationml" in mimeType or lowerName.endswith(".pptx"): - try: - from pptx import Presentation - prs = Presentation(io.BytesIO(content)) - parts = [] - for slide in prs.slides: - for shape in slide.shapes: - if shape.has_text_frame: - parts.append(shape.text_frame.text) - return "\n".join(parts) - except ImportError: - logger.warning("python-pptx not installed, cannot extract PPTX text") - return None - - logger.info(f"No text extractor for {fileName} (mime={mimeType})") - except Exception as e: - logger.warning(f"Text extraction failed for {fileName}: {e}") - return None - - # ========================================================================= # Badge + Score History Endpoints (Iteration 2) # ========================================================================= diff --git a/modules/features/commcoach/serviceCommcoach.py b/modules/features/commcoach/serviceCommcoach.py index bf5ec281..36fc6e16 100644 --- a/modules/features/commcoach/serviceCommcoach.py +++ b/modules/features/commcoach/serviceCommcoach.py @@ -42,6 +42,30 @@ from .serviceCommcoachContextRetrieval import ( logger = logging.getLogger(__name__) +def _getUserVoicePrefs(userId: str, mandateId: Optional[str] = None) -> tuple: + """Load voice language and voiceName from central UserVoicePreferences. + Returns (language, voiceName) tuple.""" + try: + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.security.rootAccess import getRootInterface + rootIf = getRootInterface() + prefs = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId, "mandateId": mandateId} + ) + if not prefs and mandateId: + prefs = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId} + ) + if prefs: + p = prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump() + return (p.get("ttsLanguage") or p.get("sttLanguage") or "de-DE", p.get("ttsVoice")) + except Exception as e: + logger.warning(f"Failed to load UserVoicePreferences for user={userId}: {e}") + return ("de-DE", None) + + def _stripMarkdownForTts(text: str) -> str: """Strip markdown formatting so TTS reads clean speech text.""" t = text @@ -159,9 +183,7 @@ async def _generateAndEmitTts(sessionId: str, speechText: str, currentUser, mand from modules.interfaces.interfaceVoiceObjects import getVoiceInterface import base64 voiceInterface = getVoiceInterface(currentUser, mandateId) - profile = interface.getProfile(str(currentUser.id), instanceId) - language = profile.get("preferredLanguage", "de-DE") if profile else "de-DE" - voiceName = profile.get("preferredVoice") if profile else None + language, voiceName = _getUserVoicePrefs(str(currentUser.id), mandateId) ttsResult = await voiceInterface.textToSpeech( text=_stripMarkdownForTts(speechText), languageCode=language, @@ -196,60 +218,36 @@ def _resolveFileNameAndMime(title: str) -> tuple: async def _saveOrUpdateDocument(doc: Dict[str, Any], contextId: str, userId: str, mandateId: str, instanceId: str, interface, sessionId: str, user=None): - """Save a new document or update an existing one. Stores file in Management DB.""" - from .datamodelCommcoach import CoachingDocument + """Save a document as platform FileItem (no CoachingDocument).""" try: - docId = doc.get("id") title = doc.get("title", "Dokument") content = doc.get("content", "") contentBytes = content.encode("utf-8") fileName, mimeType = _resolveFileNameAndMime(title) - fileRef = None - try: - import modules.interfaces.interfaceDbManagement as interfaceDbManagement - mgmtInterface = interfaceDbManagement.getInterface( - currentUser=user, mandateId=mandateId, featureInstanceId=instanceId - ) - fileItem = mgmtInterface.createFile(name=fileName, mimeType=mimeType, content=contentBytes) - mgmtInterface.createFileData(fileItem.id, contentBytes) - fileRef = fileItem.id - except Exception as e: - logger.warning(f"Failed to store document in file DB: {e}") + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + mgmtInterface = interfaceDbManagement.getInterface( + currentUser=user, mandateId=mandateId, featureInstanceId=instanceId + ) + fileItem = mgmtInterface.createFile(name=fileName, mimeType=mimeType, content=contentBytes) + mgmtInterface.createFileData(fileItem.id, contentBytes) + + from modules.datamodels.datamodelFiles import FileItem as FileItemModel + mgmtInterface.db.recordModify(FileItemModel, fileItem.id, { + "scope": "featureInstance", + "featureInstanceId": instanceId, + "mandateId": mandateId, + }) + + await emitSessionEvent(sessionId, "documentCreated", { + "id": fileItem.id, "fileName": fileName, "fileSize": len(contentBytes), + }) + logger.info(f"Document saved as platform FileItem: {fileItem.id} ({title})") - if docId: - updates = { - "fileName": fileName, - "mimeType": mimeType, - "extractedText": content, - "summary": title, - "fileSize": len(contentBytes), - } - if fileRef: - updates["fileRef"] = fileRef - updated = interface.updateDocument(docId, updates) - if updated: - await emitSessionEvent(sessionId, "documentUpdated", updated) - logger.info(f"Document updated: {docId} ({title})") - else: - logger.warning(f"Document update failed, id not found: {docId}") - else: - docData = CoachingDocument( - contextId=contextId, - userId=userId, - mandateId=mandateId, - instanceId=instanceId, - fileName=fileName, - mimeType=mimeType, - fileSize=len(contentBytes), - extractedText=content, - summary=title, - fileRef=fileRef, - ).model_dump() - created = interface.createDocument(docData) - await emitSessionEvent(sessionId, "documentCreated", created) except Exception as e: - logger.warning(f"Failed to save/update document: {e}") + logger.warning(f"Failed to save document as FileItem: {e}") + + async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, Any]], callAiFn) -> Dict[str, Any]: @@ -269,17 +267,60 @@ async def _resolveDocumentIntent(combinedUserPrompt: str, docs: List[Dict[str, A return {"read": [], "update": [], "create": [], "noDocumentAction": True} -def _loadDocumentContents(docIds: List[str], interface) -> List[Dict[str, Any]]: - """Load full extractedText for the given document IDs.""" - results = [] - for docId in docIds[:DOC_INTENT_MAX_DOCS]: - doc = interface.getDocument(docId) - if doc and doc.get("extractedText"): - results.append({ - "id": doc.get("id", ""), - "title": doc.get("summary") or doc.get("fileName", ""), - "content": doc.get("extractedText", "")[:DOC_CONTENT_MAX_CHARS], +def _getPlatformFileList(mandateId: str = None, instanceId: str = None) -> List[Dict[str, Any]]: + """Get list of platform FileItems for this feature instance (for doc intent detection).""" + try: + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + records = mgmtIf.db.getRecordset( + FileItem, recordFilter={"featureInstanceId": instanceId} + ) if instanceId else [] + result = [] + for r in records: + d = r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else {} + result.append({ + "id": d.get("id", ""), + "fileName": d.get("fileName") or d.get("name") or "Dokument", + "summary": d.get("fileName") or "", }) + return result + except Exception as e: + logger.warning(f"Failed to load platform file list: {e}") + return [] + + +def _loadDocumentContents(docIds: List[str], interface, mandateId: str = None, instanceId: str = None) -> List[Dict[str, Any]]: + """Load file content for given IDs from platform FileItem store.""" + results = [] + try: + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + for fId in docIds[:DOC_INTENT_MAX_DOCS]: + fileRecords = mgmtIf.db.getRecordset(FileItem, recordFilter={"id": fId}) + if fileRecords: + f = fileRecords[0] if isinstance(fileRecords[0], dict) else fileRecords[0].model_dump() + content = "" + try: + from modules.datamodels.datamodelKnowledge import FileContentIndex + idxRecords = mgmtIf.db.getRecordset(FileContentIndex, recordFilter={"fileId": fId}) + if idxRecords: + idx = idxRecords[0] if isinstance(idxRecords[0], dict) else idxRecords[0].model_dump() + content = (idx.get("extractedText") or "")[:DOC_CONTENT_MAX_CHARS] + except Exception: + pass + results.append({ + "id": fId, + "title": f.get("fileName") or f.get("name") or "Dokument", + "content": content, + }) + except Exception as e: + logger.warning(f"Failed to load document contents from platform: {e}") return results @@ -319,20 +360,42 @@ def _resolvePersona(session: Optional[Dict[str, Any]], interface) -> Optional[Di return None -def _getDocumentSummaries(contextId: str, userId: str, interface) -> Optional[List[str]]: - """Get document summaries for context to include in the AI prompt.""" +def _getDocumentSummaries(contextId: str, userId: str, interface, + mandateId: str = None, instanceId: str = None) -> Optional[List[str]]: + """Get document summaries from platform FileItems (UDL) for the coaching instance.""" try: - docs = interface.getDocuments(contextId, userId) + import modules.interfaces.interfaceDbManagement as interfaceDbManagement + from modules.datamodels.datamodelFiles import FileItem + mgmtIf = interfaceDbManagement.getInterface( + currentUser=None, mandateId=mandateId, featureInstanceId=instanceId + ) + files = mgmtIf.db.getRecordset( + FileItem, recordFilter={"featureInstanceId": instanceId} + ) if instanceId else [] summaries = [] - for doc in docs[:5]: - summary = doc.get("summary") - if summary: - summaries.append(f"[{doc.get('fileName', 'Dokument')}] {summary}") - elif doc.get("extractedText"): - summaries.append(f"[{doc.get('fileName', 'Dokument')}] {doc['extractedText'][:200]}...") + for f in files[:10]: + fData = f if isinstance(f, dict) else f.model_dump() if hasattr(f, "model_dump") else {} + name = fData.get("fileName") or fData.get("name") or "Dokument" + fId = fData.get("id") + snippet = None + if fId: + try: + from modules.datamodels.datamodelKnowledge import FileContentIndex + idxRecords = mgmtIf.db.getRecordset( + FileContentIndex, recordFilter={"fileId": fId} + ) + if idxRecords: + idx = idxRecords[0] if isinstance(idxRecords[0], dict) else idxRecords[0].model_dump() + snippet = (idx.get("extractedText") or "")[:200] + except Exception: + pass + if snippet: + summaries.append(f"[{name}] {snippet}...") + else: + summaries.append(f"[{name}]") return summaries if summaries else None except Exception as e: - logger.warning(f"Failed to load document summaries for context {contextId}: {e}") + logger.warning(f"Failed to load platform file summaries for instance {instanceId}: {e}") return None @@ -427,18 +490,22 @@ class CommcoachService: ) persona = _resolvePersona(session, interface) - documentSummaries = _getDocumentSummaries(contextId, self.userId, interface) + documentSummaries = _getDocumentSummaries( + contextId, self.userId, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) # Document intent detection (pre-AI-call) referencedDocumentContents = None - allDocs = interface.getDocuments(contextId, self.userId) if documentSummaries else [] + allDocs = _getPlatformFileList(self.mandateId, self.instanceId) if documentSummaries else [] if allDocs: await emitSessionEvent(sessionId, "status", {"label": "Dokumente werden geprueft..."}) docIntent = await _resolveDocumentIntent(combinedUserPrompt, allDocs, self._callAi) if not docIntent.get("noDocumentAction"): docIdsToLoad = list(set((docIntent.get("read") or []) + (docIntent.get("update") or []))) if docIdsToLoad: - referencedDocumentContents = _loadDocumentContents(docIdsToLoad, interface) + referencedDocumentContents = _loadDocumentContents( + docIdsToLoad, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) systemPrompt = aiPrompts.buildCoachingSystemPrompt( context, @@ -536,7 +603,9 @@ class CommcoachService: session = interface.getSession(sessionId) persona = _resolvePersona(session, interface) - documentSummaries = _getDocumentSummaries(contextId, self.userId, interface) + documentSummaries = _getDocumentSummaries( + contextId, self.userId, interface, mandateId=self.mandateId, instanceId=self.instanceId + ) systemPrompt = aiPrompts.buildCoachingSystemPrompt( context, previousMessages, tasks, diff --git a/modules/features/commcoach/serviceCommcoachContextRetrieval.py b/modules/features/commcoach/serviceCommcoachContextRetrieval.py index d673b04a..f1ccb9a3 100644 --- a/modules/features/commcoach/serviceCommcoachContextRetrieval.py +++ b/modules/features/commcoach/serviceCommcoachContextRetrieval.py @@ -172,20 +172,48 @@ def searchSessionsByTopic( def searchSessionsByTopicRag( - sessions: List[Dict[str, Any]], query: str, - maxResults: int = TOPIC_SEARCH_MAX_RESULTS, - embeddingProvider: Optional[Any] = None, + userId: str, + instanceId: str, + mandateId: str = None, + queryVector: List[float] = None, ) -> List[Dict[str, Any]]: + """Search using platform RAG (semantic search across mandate-wide knowledge data). + + Requires a pre-computed queryVector (embedding). The caller is responsible + for generating the embedding via AiService.callEmbedding before invoking this. """ - Phase 7 RAG: Semantic search via embeddings. - When embeddingProvider is None, falls back to keyword search. - Future: Pass embeddingProvider that has embed(text) -> vector and similarity search. - """ - if embeddingProvider is None: - return searchSessionsByTopic(sessions, query, maxResults) - # TODO: When embedding API exists: embed query, embed session summaries, cosine similarity - return searchSessionsByTopic(sessions, query, maxResults) + if not queryVector: + logger.warning("searchSessionsByTopicRag called without queryVector, skipping RAG search") + return [] + try: + from modules.interfaces.interfaceDbKnowledge import getInterface as _getKnowledgeInterface + + knowledgeDb = _getKnowledgeInterface() + + results = knowledgeDb.semanticSearch( + queryVector=queryVector, + userId=userId, + featureInstanceId=instanceId, + mandateId=mandateId, + isSysAdmin=False, + limit=TOPIC_SEARCH_MAX_RESULTS, + ) + + formatted = [] + for r in (results or []): + rData = r if isinstance(r, dict) else r.model_dump() if hasattr(r, "model_dump") else {} + contextRef = rData.get("contextRef") or {} + formatted.append({ + "source": "rag", + "content": rData.get("data") or rData.get("summary") or "", + "fileName": contextRef.get("containerPath") or "RAG-Ergebnis", + "score": rData.get("_score") or 0, + }) + return formatted + except Exception as e: + logger.warning(f"RAG search failed for query '{query[:50]}': {e}") + return [] def buildSessionSummariesForPrompt( diff --git a/modules/features/commcoach/tests/test_datamodel.py b/modules/features/commcoach/tests/test_datamodel.py index fb39ba34..05d174c5 100644 --- a/modules/features/commcoach/tests/test_datamodel.py +++ b/modules/features/commcoach/tests/test_datamodel.py @@ -136,7 +136,6 @@ class TestCoachingUserProfile: profile = CoachingUserProfile( userId="u1", mandateId="m1", instanceId="i1", ) - assert profile.preferredLanguage == "de-DE" assert profile.dailyReminderEnabled is False assert profile.emailSummaryEnabled is True assert profile.streakDays == 0 diff --git a/modules/interfaces/interfaceBootstrap.py b/modules/interfaces/interfaceBootstrap.py index 80607268..e2a0dfa4 100644 --- a/modules/interfaces/interfaceBootstrap.py +++ b/modules/interfaces/interfaceBootstrap.py @@ -108,6 +108,13 @@ def initBootstrap(db: DatabaseConnector) -> None: except Exception as e: logger.error(f"Root user migration failed: {e}") + # Run voice & documents migration (one-time, sets completion flag) + try: + from modules.migration.migrateVoiceAndDocuments import migrateVoiceAndDocuments + migrateVoiceAndDocuments(db) + except Exception as e: + logger.error(f"Voice & documents migration failed: {e}") + # After migration: root mandate is purely technical — no feature instances if not migrationDone and mandateId: initRootMandateFeatures(db, mandateId) diff --git a/modules/interfaces/interfaceVoiceObjects.py b/modules/interfaces/interfaceVoiceObjects.py index dc391bae..38807bac 100644 --- a/modules/interfaces/interfaceVoiceObjects.py +++ b/modules/interfaces/interfaceVoiceObjects.py @@ -11,9 +11,7 @@ import logging from typing import AsyncGenerator, Callable, Dict, Any, Optional, List from modules.connectors.connectorVoiceGoogle import ConnectorGoogleSpeech -from modules.datamodels.datamodelVoice import VoiceSettings from modules.datamodels.datamodelUam import User -from modules.shared.timeUtils import getUtcTimestamp logger = logging.getLogger(__name__) @@ -335,123 +333,6 @@ class VoiceObjects: "error": str(e) } - # Voice Settings Management - - def getVoiceSettings(self, userId: str) -> Optional[VoiceSettings]: - """ - Get voice settings for a user. - - Args: - userId: User ID to get settings for - - Returns: - VoiceSettings object or None if not found - """ - try: - # This would typically query the database - # For now, return None as this is handled by the database interface - logger.debug(f"Getting voice settings for user: {userId}") - return None - - except Exception as e: - logger.error(f"❌ Error getting voice settings: {e}") - return None - - def createVoiceSettings(self, settingsData: Dict[str, Any]) -> Optional[VoiceSettings]: - """ - Create new voice settings. - - Args: - settingsData: Dictionary containing voice settings data - - Returns: - Created VoiceSettings object or None if failed - """ - try: - logger.info(f"Creating voice settings: {settingsData}") - - # Ensure mandateId is set from context if not provided - if "mandateId" not in settingsData or not settingsData["mandateId"]: - if not self.mandateId: - raise ValueError("mandateId is required but not provided and context has no mandateId") - settingsData["mandateId"] = self.mandateId - - # Add timestamps - currentTime = getUtcTimestamp() - settingsData["creationDate"] = currentTime - settingsData["lastModified"] = currentTime - - # Create VoiceSettings object - voiceSettings = VoiceSettings(**settingsData) - - logger.info(f"✅ Voice settings created: {voiceSettings.id}") - return voiceSettings - - except Exception as e: - logger.error(f"❌ Error creating voice settings: {e}") - return None - - def updateVoiceSettings(self, userId: str, settingsData: Dict[str, Any]) -> Optional[VoiceSettings]: - """ - Update existing voice settings. - - Args: - userId: User ID to update settings for - settingsData: Dictionary containing updated voice settings data - - Returns: - Updated VoiceSettings object or None if failed - """ - try: - logger.info(f"Updating voice settings for user {userId}: {settingsData}") - - # Add last modified timestamp - settingsData["lastModified"] = getUtcTimestamp() - - # Create updated VoiceSettings object - voiceSettings = VoiceSettings(**settingsData) - - logger.info(f"✅ Voice settings updated: {voiceSettings.id}") - return voiceSettings - - except Exception as e: - logger.error(f"❌ Error updating voice settings: {e}") - return None - - def getOrCreateVoiceSettings(self, userId: str) -> Optional[VoiceSettings]: - """ - Get existing voice settings or create default ones. - - Args: - userId: User ID to get/create settings for - - Returns: - VoiceSettings object - """ - try: - # Try to get existing settings - existingSettings = self.getVoiceSettings(userId) - - if existingSettings: - return existingSettings - - # Create default settings if none exist - defaultSettings = { - "userId": userId, - "mandateId": self.mandateId, - "sttLanguage": "de-DE", - "ttsLanguage": "de-DE", - "ttsVoice": "de-DE-Wavenet-A", - "translationEnabled": True, - "targetLanguage": "en-US" - } - - return self.createVoiceSettings(defaultSettings) - - except Exception as e: - logger.error(f"❌ Error getting or creating voice settings: {e}") - return None - # Language and Voice Information async def getAvailableLanguages(self) -> Dict[str, Any]: diff --git a/modules/migration/migrateVoiceAndDocuments.py b/modules/migration/migrateVoiceAndDocuments.py new file mode 100644 index 00000000..0fc5ee02 --- /dev/null +++ b/modules/migration/migrateVoiceAndDocuments.py @@ -0,0 +1,316 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +""" +Migration: Voice settings consolidation and CoachingDocument scope-tagging. +Moves VoiceSettings (workspace DB) and CoachingUserProfile voice fields (commcoach DB) +into the unified UserVoicePreferences model, and tags CoachingDocument files with +featureInstance scope before deleting the legacy records. +Called once from bootstrap, sets a DB flag to prevent re-execution. +""" + +import logging +import uuid +from typing import Dict, List, Optional + +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +from modules.datamodels.datamodelUam import UserVoicePreferences + +logger = logging.getLogger(__name__) + +_MIGRATION_FLAG_KEY = "migration_voice_documents_completed" + + +def _isMigrationCompleted(db) -> bool: + """Check if migration has already been executed.""" + try: + from modules.datamodels.datamodelUam import Mandate + records = db.getRecordset(Mandate, recordFilter={"name": _MIGRATION_FLAG_KEY}) + return len(records) > 0 + except Exception: + return False + + +def _setMigrationCompleted(db) -> None: + """Set flag that migration is completed (uses a settings-like record).""" + if _isMigrationCompleted(db): + return + try: + from modules.datamodels.datamodelUam import Mandate + flag = Mandate(name=_MIGRATION_FLAG_KEY, label="Migration completed", enabled=False, isSystem=True) + db.recordCreate(Mandate, flag) + logger.info("Migration flag set: voice & documents migration completed") + except Exception as e: + logger.error(f"Failed to set migration flag: {e}") + + +def _getRawRows(connector: DatabaseConnector, tableName: str, columns: List[str]) -> List[Dict]: + """Read all rows from a table via raw SQL. Returns empty list if table doesn't exist.""" + try: + connector._ensure_connection() + colList = ", ".join(f'"{c}"' for c in columns) + with connector.connection.cursor() as cur: + cur.execute( + "SELECT COUNT(*) FROM information_schema.tables " + "WHERE LOWER(table_name) = LOWER(%s) AND table_schema = 'public'", + (tableName,), + ) + if cur.fetchone()["count"] == 0: + logger.info(f"Table '{tableName}' does not exist, skipping") + return [] + cur.execute(f'SELECT {colList} FROM "{tableName}"') + return [dict(row) for row in cur.fetchall()] + except Exception as e: + logger.warning(f"Raw query on '{tableName}' failed: {e}") + try: + connector.connection.rollback() + except Exception: + pass + return [] + + +def _deleteRawRow(connector: DatabaseConnector, tableName: str, rowId: str) -> bool: + """Delete a single row by id via raw SQL.""" + try: + connector._ensure_connection() + with connector.connection.cursor() as cur: + cur.execute(f'DELETE FROM "{tableName}" WHERE "id" = %s', (rowId,)) + connector.connection.commit() + return True + except Exception as e: + logger.warning(f"Failed to delete row {rowId} from '{tableName}': {e}") + try: + connector.connection.rollback() + except Exception: + pass + return False + + +def _createDbConnector(dbName: str) -> Optional[DatabaseConnector]: + """Create a DatabaseConnector for a named database, returns None on failure.""" + try: + dbHost = APP_CONFIG.get("DB_HOST") + dbUser = APP_CONFIG.get("DB_USER") + dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_PORT", 5432)) + return DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbName, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + ) + except Exception as e: + logger.warning(f"Could not connect to database '{dbName}': {e}") + return None + + +# ─── Part A ─────────────────────────────────────────────────────────────────── + +def _migrateVoiceSettings(db, wsDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Migrate VoiceSettings records from poweron_workspace into UserVoicePreferences.""" + rows = _getRawRows(wsDb, "VoiceSettings", [ + "id", "userId", "mandateId", "ttsVoiceMap", "sttLanguage", "ttsLanguage", "ttsVoice", + ]) + if not rows: + logger.info("Part A: No VoiceSettings records found, skipping") + return + + for row in rows: + userId = row.get("userId") + if not userId: + continue + + existing = db.getRecordset(UserVoicePreferences, recordFilter={"userId": userId}) + if existing: + stats["voiceSettingsSkipped"] += 1 + if not dryRun: + _deleteRawRow(wsDb, "VoiceSettings", row["id"]) + continue + + if dryRun: + logger.info(f"[DRY RUN] Would create UserVoicePreferences for user {userId} from VoiceSettings") + stats["voiceSettingsCreated"] += 1 + continue + + try: + import json + ttsVoiceMap = row.get("ttsVoiceMap") + if isinstance(ttsVoiceMap, str): + try: + ttsVoiceMap = json.loads(ttsVoiceMap) + except (json.JSONDecodeError, TypeError): + ttsVoiceMap = None + + prefs = UserVoicePreferences( + userId=userId, + mandateId=row.get("mandateId"), + ttsVoiceMap=ttsVoiceMap, + sttLanguage=row.get("sttLanguage", "de-DE"), + ttsLanguage=row.get("ttsLanguage", "de-DE"), + ttsVoice=row.get("ttsVoice"), + ) + db.recordCreate(UserVoicePreferences, prefs) + stats["voiceSettingsCreated"] += 1 + _deleteRawRow(wsDb, "VoiceSettings", row["id"]) + except Exception as e: + logger.error(f"Part A: Failed to migrate VoiceSettings {row['id']}: {e}") + stats["errors"] += 1 + + +# ─── Part B ─────────────────────────────────────────────────────────────────── + +def _migrateCoachingProfileVoice(db, ccDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Migrate preferredLanguage/preferredVoice from CoachingUserProfile into UserVoicePreferences.""" + rows = _getRawRows(ccDb, "CoachingUserProfile", [ + "id", "userId", "mandateId", "preferredLanguage", "preferredVoice", + ]) + if not rows: + logger.info("Part B: No CoachingUserProfile records with voice data found, skipping") + return + + for row in rows: + userId = row.get("userId") + prefLang = row.get("preferredLanguage") + prefVoice = row.get("preferredVoice") + if not userId or (not prefLang and not prefVoice): + continue + + existing = db.getRecordset(UserVoicePreferences, recordFilter={"userId": userId}) + if existing: + stats["coachingProfileSkipped"] += 1 + continue + + if dryRun: + logger.info(f"[DRY RUN] Would create UserVoicePreferences for user {userId} from CoachingUserProfile") + stats["coachingProfileCreated"] += 1 + continue + + try: + prefs = UserVoicePreferences( + userId=userId, + mandateId=row.get("mandateId"), + sttLanguage=prefLang or "de-DE", + ttsLanguage=prefLang or "de-DE", + ttsVoice=prefVoice, + ) + db.recordCreate(UserVoicePreferences, prefs) + stats["coachingProfileCreated"] += 1 + except Exception as e: + logger.error(f"Part B: Failed to migrate CoachingUserProfile {row['id']}: {e}") + stats["errors"] += 1 + + +# ─── Part C ─────────────────────────────────────────────────────────────────── + +def _migrateCoachingDocuments(ccDb: DatabaseConnector, dryRun: bool, stats: Dict) -> None: + """Tag FileItem/FileContentIndex with featureInstance scope for each CoachingDocument.""" + from modules.datamodels.datamodelFiles import FileItem + from modules.datamodels.datamodelKnowledge import FileContentIndex + + rows = _getRawRows(ccDb, "CoachingDocument", [ + "id", "fileRef", "instanceId", + ]) + if not rows: + logger.info("Part C: No CoachingDocument records found, skipping") + return + + mgmtDb = _createDbConnector("poweron_management") + knowledgeDb = _createDbConnector("poweron_knowledge") + if not mgmtDb: + logger.error("Part C: Cannot connect to poweron_management, aborting document migration") + return + + for row in rows: + fileRef = row.get("fileRef") + instanceId = row.get("instanceId") + docId = row.get("id") + if not fileRef: + if not dryRun: + _deleteRawRow(ccDb, "CoachingDocument", docId) + continue + + if dryRun: + logger.info(f"[DRY RUN] Would tag FileItem {fileRef} with featureInstanceId={instanceId}") + stats["documentsTagged"] += 1 + continue + + try: + fileRecords = mgmtDb.getRecordset(FileItem, recordFilter={"id": fileRef}) + if fileRecords: + updateData = {"scope": "featureInstance"} + if instanceId: + updateData["featureInstanceId"] = instanceId + mgmtDb.recordModify(FileItem, fileRef, updateData) + stats["documentsTagged"] += 1 + else: + logger.warning(f"Part C: FileItem {fileRef} not found in management DB") + + if knowledgeDb: + fciRecords = knowledgeDb.getRecordset(FileContentIndex, recordFilter={"id": fileRef}) + if fciRecords: + fciUpdate = {"scope": "featureInstance"} + if instanceId: + fciUpdate["featureInstanceId"] = instanceId + knowledgeDb.recordModify(FileContentIndex, fileRef, fciUpdate) + + _deleteRawRow(ccDb, "CoachingDocument", docId) + except Exception as e: + logger.error(f"Part C: Failed to migrate CoachingDocument {docId}: {e}") + stats["errors"] += 1 + + +# ─── Main entry ─────────────────────────────────────────────────────────────── + +def migrateVoiceAndDocuments(db, dryRun: bool = False) -> dict: + """ + Migrate VoiceSettings + CoachingUserProfile voice fields into UserVoicePreferences, + and tag CoachingDocument files with featureInstance scope. + + Args: + db: Root database connector (poweron_app) + dryRun: If True, log actions without making changes + + Returns: + Summary dict with migration statistics + """ + if _isMigrationCompleted(db): + logger.info("Voice & documents migration already completed, skipping") + return {"status": "already_completed"} + + stats = { + "voiceSettingsCreated": 0, + "voiceSettingsSkipped": 0, + "coachingProfileCreated": 0, + "coachingProfileSkipped": 0, + "documentsTagged": 0, + "errors": 0, + "dryRun": dryRun, + } + + wsDb = _createDbConnector("poweron_workspace") + ccDb = _createDbConnector("poweron_commcoach") + + # Part A + if wsDb: + _migrateVoiceSettings(db, wsDb, dryRun, stats) + else: + logger.warning("Skipping Part A: poweron_workspace DB unavailable") + + # Part B + if ccDb: + _migrateCoachingProfileVoice(db, ccDb, dryRun, stats) + else: + logger.warning("Skipping Part B: poweron_commcoach DB unavailable") + + # Part C + if ccDb: + _migrateCoachingDocuments(ccDb, dryRun, stats) + else: + logger.warning("Skipping Part C: poweron_commcoach DB unavailable") + + if not dryRun: + _setMigrationCompleted(db) + + logger.info(f"Voice & documents migration completed: {stats}") + return {"status": "completed", **stats} diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index c1afb8ff..8b1d9e8e 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -4,7 +4,7 @@ Routes for local security and authentication. """ -from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body +from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body, Query from fastapi.security import OAuth2PasswordRequestForm import logging from typing import Dict, Any @@ -816,22 +816,19 @@ def getVoicePreferences( currentUser: User = Depends(getCurrentUser), ) -> Dict[str, Any]: """Get user's voice/language preferences (optionally scoped to mandate via header).""" - try: - rootInterface = getRootInterface() - from modules.datamodels.datamodelUam import UserVoicePreferences + rootInterface = getRootInterface() + from modules.datamodels.datamodelUam import UserVoicePreferences - mandateId = request.headers.get("X-Mandate-Id") or None + mandateId = request.headers.get("X-Mandate-Id") or None + userId = str(currentUser.id) - prefs = rootInterface.db.getRecordset( - UserVoicePreferences, - recordFilter={"userId": str(currentUser.id), "mandateId": mandateId} - ) - if prefs: - return prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump() - return UserVoicePreferences(userId=str(currentUser.id), mandateId=mandateId).model_dump() - except Exception as e: - logger.error(f"Error getting voice preferences: {e}") - return {"sttLanguage": "de-DE", "ttsLanguage": "de-DE"} + prefs = rootInterface.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId, "mandateId": mandateId} + ) + if prefs: + return prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump() + return UserVoicePreferences(userId=userId, mandateId=mandateId).model_dump() @router.put("/voice-preferences") @@ -841,34 +838,87 @@ def updateVoicePreferences( preferences: Dict[str, Any] = Body(...), currentUser: User = Depends(getCurrentUser), ) -> Dict[str, Any]: - """Update user's voice/language preferences.""" - try: - rootInterface = getRootInterface() - from modules.datamodels.datamodelUam import UserVoicePreferences + """Update user's voice/language preferences (upsert).""" + rootInterface = getRootInterface() + from modules.datamodels.datamodelUam import UserVoicePreferences - mandateId = request.headers.get("X-Mandate-Id") or None - userId = str(currentUser.id) + mandateId = request.headers.get("X-Mandate-Id") or None + userId = str(currentUser.id) - existing = rootInterface.db.getRecordset( - UserVoicePreferences, - recordFilter={"userId": userId, "mandateId": mandateId} - ) + existing = rootInterface.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId, "mandateId": mandateId} + ) - allowedFields = { - "sttLanguage", "ttsLanguage", "ttsVoice", "ttsVoiceMap", - "translationSourceLanguage", "translationTargetLanguage", - } - updateData = {k: v for k, v in preferences.items() if k in allowedFields} + allowedFields = { + "sttLanguage", "ttsLanguage", "ttsVoice", "ttsVoiceMap", + "translationSourceLanguage", "translationTargetLanguage", + } + updateData = {k: v for k, v in preferences.items() if k in allowedFields} - if existing: - existingRecord = existing[0] - existingId = existingRecord.get("id") if isinstance(existingRecord, dict) else existingRecord.id - rootInterface.db.recordModify(UserVoicePreferences, existingId, updateData) - return {"message": "Updated", **updateData} - else: - newPrefs = UserVoicePreferences(userId=userId, mandateId=mandateId, **updateData) - created = rootInterface.db.recordCreate(UserVoicePreferences, newPrefs.model_dump()) - return {"message": "Created", **(created if isinstance(created, dict) else created.model_dump())} - except Exception as e: - logger.error(f"Error updating voice preferences: {e}") - raise HTTPException(status_code=500, detail=str(e)) + if existing: + existingRecord = existing[0] + existingId = existingRecord.get("id") if isinstance(existingRecord, dict) else existingRecord.id + rootInterface.db.recordModify(UserVoicePreferences, existingId, updateData) + updated = rootInterface.db.getRecordset(UserVoicePreferences, recordFilter={"id": existingId}) + return updated[0] if updated else {"message": "Updated", **updateData} + else: + newPrefs = UserVoicePreferences(userId=userId, mandateId=mandateId, **updateData) + created = rootInterface.db.recordCreate(UserVoicePreferences, newPrefs.model_dump()) + return created if isinstance(created, dict) else created.model_dump() + + +@router.get("/voice/languages") +@limiter.limit("120/minute") +async def getVoiceLanguages( + request: Request, + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Return available TTS languages (user-level, no instance context needed).""" + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + voiceInterface = getVoiceInterface(currentUser) + languagesResult = await voiceInterface.getAvailableLanguages() + languageList = languagesResult.get("languages", []) if isinstance(languagesResult, dict) else languagesResult + return {"languages": languageList} + + +@router.get("/voice/voices") +@limiter.limit("120/minute") +async def getVoiceVoices( + request: Request, + language: str = Query("de-DE"), + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Return available TTS voices for a given language.""" + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + voiceInterface = getVoiceInterface(currentUser) + voicesResult = await voiceInterface.getAvailableVoices(language) + voiceList = voicesResult.get("voices", []) if isinstance(voicesResult, dict) else voicesResult + return {"voices": voiceList} + + +@router.post("/voice/test") +@limiter.limit("30/minute") +async def testVoice( + request: Request, + body: Dict[str, Any] = Body(...), + currentUser: User = Depends(getCurrentUser), +) -> Dict[str, Any]: + """Test a specific voice with a sample text.""" + import base64 + from modules.interfaces.interfaceVoiceObjects import getVoiceInterface + + text = body.get("text", "Hallo, das ist ein Stimmtest.") + language = body.get("language", "de-DE") + voiceId = body.get("voiceId") + + voiceInterface = getVoiceInterface(currentUser) + result = await voiceInterface.textToSpeech(text=text, languageCode=language, voiceName=voiceId) + if result and isinstance(result, dict): + audioContent = result.get("audioContent") + if audioContent: + audioB64 = base64.b64encode( + audioContent if isinstance(audioContent, bytes) else audioContent.encode() + ).decode() + return {"success": True, "audio": audioB64, "format": "mp3", "text": text} + return {"success": False, "error": "TTS returned no audio"} diff --git a/modules/routes/routeVoiceGoogle.py b/modules/routes/routeVoiceGoogle.py index af4db355..dc0c7a85 100644 --- a/modules/routes/routeVoiceGoogle.py +++ b/modules/routes/routeVoiceGoogle.py @@ -442,113 +442,50 @@ async def health_check(currentUser: User = Depends(getCurrentUser)): @router.get("/settings") async def get_voice_settings(currentUser: User = Depends(getCurrentUser)): - """Get voice settings for the current user.""" - try: - logger.info(f"Getting voice settings for user: {currentUser.id}") - - # Get voice interface - voiceInterface = _getVoiceInterface(currentUser) - - # Get or create voice settings for the user - voice_settings = voiceInterface.getOrCreateVoiceSettings(currentUser.id) - - if voice_settings: - # Return user settings - return { - "success": True, - "data": { - "user_settings": voice_settings.model_dump(), - "default_settings": { - "sttLanguage": "de-DE", - "ttsLanguage": "de-DE", - "ttsVoice": "de-DE-Wavenet-A", - "translationEnabled": True, - "targetLanguage": "en-US" - } - } - } - else: - # Fallback to default settings if database fails - logger.warning("Failed to get voice settings from database, using defaults") - return { - "success": True, - "data": { - "user_settings": None, - "default_settings": { - "sttLanguage": "de-DE", - "ttsLanguage": "de-DE", - "ttsVoice": "de-DE-Wavenet-A", - "translationEnabled": True, - "targetLanguage": "en-US" - } - } - } - - except Exception as e: - logger.error(f"Error getting voice settings: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to get voice settings: {str(e)}" - ) + """Get voice settings for the current user (reads from UserVoicePreferences).""" + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.security.rootAccess import getRootInterface + rootInterface = getRootInterface() + userId = str(currentUser.id) + + prefs = rootInterface.db.getRecordset( + UserVoicePreferences, recordFilter={"userId": userId} + ) + if prefs: + data = prefs[0] if isinstance(prefs[0], dict) else prefs[0].model_dump() + return {"success": True, "data": {"user_settings": data}} + return {"success": True, "data": {"user_settings": UserVoicePreferences(userId=userId).model_dump()}} + @router.post("/settings") async def save_voice_settings( settings: Dict[str, Any] = Body(...), currentUser: User = Depends(getCurrentUser) ): - """Save voice settings for the current user.""" - try: - logger.info(f"Saving voice settings for user: {currentUser.id}") - logger.info(f"Settings: {settings}") - - # Validate required settings - requiredFields = ["sttLanguage", "ttsLanguage", "ttsVoice"] - for field in requiredFields: - if field not in settings: - raise HTTPException( - status_code=400, - detail=f"Missing required field: {field}" - ) - - # Set default values for optional fields if not provided - if "translationEnabled" not in settings: - settings["translationEnabled"] = True - if "targetLanguage" not in settings: - settings["targetLanguage"] = "en-US" - - # Get voice interface - voiceInterface = _getVoiceInterface(currentUser) - - # Check if settings already exist for this user - existing_settings = voiceInterface.getVoiceSettings(currentUser.id) - - if existing_settings: - # Update existing settings - logger.info(f"Updating existing voice settings for user {currentUser.id}") - updated_settings = voiceInterface.updateVoiceSettings(currentUser.id, settings) - logger.info(f"Voice settings updated for user {currentUser.id}: {updated_settings}") - else: - # Create new settings - logger.info(f"Creating new voice settings for user {currentUser.id}") - # Add userId to settings - settings["userId"] = currentUser.id - created_settings = voiceInterface.createVoiceSettings(settings) - logger.info(f"Voice settings created for user {currentUser.id}: {created_settings}") - - return { - "success": True, - "message": "Voice settings saved successfully", - "data": settings - } - - except HTTPException: - raise - except Exception as e: - logger.error(f"Error saving voice settings: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to save voice settings: {str(e)}" - ) + """Save voice settings for the current user (writes to UserVoicePreferences).""" + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.security.rootAccess import getRootInterface + rootInterface = getRootInterface() + userId = str(currentUser.id) + + allowedFields = { + "sttLanguage", "ttsLanguage", "ttsVoice", "ttsVoiceMap", + "translationSourceLanguage", "translationTargetLanguage", + } + updateData = {k: v for k, v in settings.items() if k in allowedFields} + + existing = rootInterface.db.getRecordset( + UserVoicePreferences, recordFilter={"userId": userId} + ) + if existing: + existingRecord = existing[0] + existingId = existingRecord.get("id") if isinstance(existingRecord, dict) else existingRecord.id + rootInterface.db.recordModify(UserVoicePreferences, existingId, updateData) + else: + newPrefs = UserVoicePreferences(userId=userId, **updateData) + rootInterface.db.recordCreate(UserVoicePreferences, newPrefs.model_dump()) + + return {"success": True, "message": "Voice settings saved successfully", "data": updateData} # ========================================================================= # STT Streaming WebSocket — generic, used by all features diff --git a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py index cbea5631..f9e72ea6 100644 --- a/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py +++ b/modules/serviceCenter/services/serviceAgent/mainServiceAgent.py @@ -2517,55 +2517,55 @@ def _registerCoreTools(registry: ToolRegistry, services): if not voiceName: try: - from modules.features.workspace import interfaceFeatureWorkspace - featureInstanceId = context.get("featureInstanceId", "") + from modules.datamodels.datamodelUam import UserVoicePreferences + from modules.security.rootAccess import getRootInterface userId = context.get("userId", "") if userId: - wsIf = interfaceFeatureWorkspace.getInterface( - services.user, - mandateId=mandateId or None, - featureInstanceId=featureInstanceId or None, + rootIf = getRootInterface() + prefRecords = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId, "mandateId": mandateId} ) - vs = wsIf.getVoiceSettings(userId) if wsIf else None - if vs: - voiceMap = {} - if hasattr(vs, "ttsVoiceMap") and vs.ttsVoiceMap: - voiceMap = vs.ttsVoiceMap if isinstance(vs.ttsVoiceMap, dict) else {} + if not prefRecords and mandateId: + prefRecords = rootIf.db.getRecordset( + UserVoicePreferences, + recordFilter={"userId": userId} + ) + if prefRecords: + vs = prefRecords[0] if isinstance(prefRecords[0], dict) else prefRecords[0].model_dump() if hasattr(prefRecords[0], "model_dump") else prefRecords[0] + voiceMap = vs.get("ttsVoiceMap", {}) or {} + if isinstance(voiceMap, dict) and voiceMap: + selectedKey = None + selectedVoiceEntry = None + baseLanguage = language.split("-")[0].lower() if isinstance(language, str) and language else "" - selectedKey = None - selectedVoiceEntry = None - baseLanguage = language.split("-")[0].lower() if isinstance(language, str) and language else "" + if isinstance(language, str) and language in voiceMap: + selectedKey = language + selectedVoiceEntry = voiceMap[language] - # 1) Exact match first (e.g. de-DE) - if isinstance(language, str) and language in voiceMap: - selectedKey = language - selectedVoiceEntry = voiceMap[language] + if selectedVoiceEntry is None and baseLanguage and baseLanguage in voiceMap: + selectedKey = baseLanguage + selectedVoiceEntry = voiceMap[baseLanguage] - # 2) Match short language key (e.g. de) - if selectedVoiceEntry is None and baseLanguage and baseLanguage in voiceMap: - selectedKey = baseLanguage - selectedVoiceEntry = voiceMap[baseLanguage] + if selectedVoiceEntry is None and baseLanguage: + for mapKey, mapValue in voiceMap.items(): + mapKeyNorm = str(mapKey).lower() + if mapKeyNorm == baseLanguage or mapKeyNorm.startswith(f"{baseLanguage}-"): + selectedKey = str(mapKey) + selectedVoiceEntry = mapValue + break - # 3) Match by same language family (e.g. de-CH -> de-DE mapping) - if selectedVoiceEntry is None and baseLanguage: - for mapKey, mapValue in voiceMap.items(): - mapKeyNorm = str(mapKey).lower() - if mapKeyNorm == baseLanguage or mapKeyNorm.startswith(f"{baseLanguage}-"): - selectedKey = str(mapKey) - selectedVoiceEntry = mapValue - break - - if selectedVoiceEntry is not None: - voiceName = ( - selectedVoiceEntry.get("voiceName") - if isinstance(selectedVoiceEntry, dict) - else selectedVoiceEntry - ) - logger.info( - f"textToSpeech: using configured voice '{voiceName}' for requested language '{language}' (matched key '{selectedKey}')" - ) - elif hasattr(vs, "ttsVoice") and vs.ttsVoice and hasattr(vs, "ttsLanguage") and vs.ttsLanguage == language: - voiceName = vs.ttsVoice + if selectedVoiceEntry is not None: + voiceName = ( + selectedVoiceEntry.get("voiceName") + if isinstance(selectedVoiceEntry, dict) + else selectedVoiceEntry + ) + logger.info( + f"textToSpeech: using configured voice '{voiceName}' for requested language '{language}' (matched key '{selectedKey}')" + ) + if not voiceName and vs.get("ttsVoice") and vs.get("ttsLanguage") == language: + voiceName = vs["ttsVoice"] except Exception as prefErr: logger.debug(f"textToSpeech: could not load voice preferences: {prefErr}") diff --git a/modules/serviceCenter/services/serviceAi/mainServiceAi.py b/modules/serviceCenter/services/serviceAi/mainServiceAi.py index 494389ff..37b8b0ba 100644 --- a/modules/serviceCenter/services/serviceAi/mainServiceAi.py +++ b/modules/serviceCenter/services/serviceAi/mainServiceAi.py @@ -557,23 +557,24 @@ detectedIntent-Werte: def _neutralizeRequest(self, request: AiCallRequest) -> Tuple[AiCallRequest, bool]: """Neutralize the prompt text in an AiCallRequest. - Returns (modifiedRequest, wasNeutralized).""" - try: - neutralSvc = self._get_service("neutralization") - if not neutralSvc or not hasattr(neutralSvc, 'processText'): - return request, False + Returns (modifiedRequest, wasNeutralized). + Raises RuntimeError if neutralization is required but fails (fail-safe).""" + neutralSvc = self._get_service("neutralization") + if not neutralSvc or not hasattr(neutralSvc, 'processText'): + raise RuntimeError("Neutralization required but neutralization service is unavailable") - if request.prompt: - result = neutralSvc.processText(request.prompt) - if result and result.get("neutralized_text"): - request.prompt = result["neutralized_text"] - logger.debug("Neutralized prompt in AiCallRequest") - return request, True + if request.prompt: + result = neutralSvc.processText(request.prompt) + if result and result.get("neutralized_text"): + request.prompt = result["neutralized_text"] + logger.debug("Neutralized prompt in AiCallRequest") + return request, True + raise RuntimeError( + "Neutralization required but processText returned no neutralized_text — " + "AI call blocked to protect sensitive data" + ) - return request, False - except Exception as e: - logger.warning(f"Request neutralization failed: {e}") - return request, False + return request, False def _rehydrateResponse(self, responseText: str) -> str: """Replace neutralization placeholders with original values in AI response."""