From 584c5125de0f6f6b33a609df8681d2edda9158c3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 16 Mar 2026 00:47:47 +0100 Subject: [PATCH] enhanced stt/tts functions --- src/pages/FeatureView.tsx | 7 +- src/pages/views/workspace/ChatStream.tsx | 124 +++++++- src/pages/views/workspace/WorkspaceInput.tsx | 209 ++++++++++--- src/pages/views/workspace/WorkspacePage.tsx | 1 - .../workspace/WorkspaceSettings.module.css | 173 +++++++++++ .../views/workspace/WorkspaceSettings.tsx | 275 ++++++++++++++++++ .../views/workspace/WorkspaceSettingsPage.tsx | 72 +++++ src/pages/views/workspace/useWorkspace.ts | 27 +- src/types/mandate.ts | 1 + 9 files changed, 835 insertions(+), 54 deletions(-) create mode 100644 src/pages/views/workspace/WorkspaceSettings.module.css create mode 100644 src/pages/views/workspace/WorkspaceSettings.tsx create mode 100644 src/pages/views/workspace/WorkspaceSettingsPage.tsx diff --git a/src/pages/FeatureView.tsx b/src/pages/FeatureView.tsx index c7629c2..80a10e1 100644 --- a/src/pages/FeatureView.tsx +++ b/src/pages/FeatureView.tsx @@ -38,6 +38,7 @@ import { CodeEditorPage, CodeEditorWorkflowsPage } from './views/codeeditor'; // Workspace Views import { WorkspacePage } from './views/workspace/WorkspacePage'; +import { WorkspaceSettingsPage } from './views/workspace/WorkspaceSettingsPage'; // Teamsbot Views import { TeamsbotDashboardView } from './views/teamsbot/TeamsbotDashboardView'; @@ -142,6 +143,7 @@ const VIEW_COMPONENTS: Record> = { }, workspace: { dashboard: WorkspacePage, + settings: WorkspaceSettingsPage, }, teamsbot: { dashboard: TeamsbotDashboardView, @@ -205,8 +207,9 @@ export const FeatureViewPage: React.FC = ({ view }) => { return ; } - // Workspace is rendered persistently by WorkspaceKeepAlive at MainLayout level - if (featureCode === 'workspace') { + // Workspace dashboard is rendered persistently by WorkspaceKeepAlive at MainLayout level; + // other workspace views (e.g. settings) use the standard FeatureViewPage rendering. + if (featureCode === 'workspace' && view !== 'settings') { return null; } diff --git a/src/pages/views/workspace/ChatStream.tsx b/src/pages/views/workspace/ChatStream.tsx index 8111b22..f944db4 100644 --- a/src/pages/views/workspace/ChatStream.tsx +++ b/src/pages/views/workspace/ChatStream.tsx @@ -5,7 +5,7 @@ * highlighting), agent progress indicators, and file edit proposals. */ -import React, { useRef, useEffect, useCallback } from 'react'; +import React, { useRef, useEffect, useCallback, useState } from 'react'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import api from '../../../api'; @@ -126,6 +126,13 @@ export const ChatStream: React.FC = ({ ))} )} + {(msg as any)._audioUrl && ( + <_AudioPlayer + url={(msg as any)._audioUrl} + language={(msg as any)._audioLang} + charCount={(msg as any)._audioCharCount} + /> + )} )} @@ -336,6 +343,121 @@ function _getFileIcon(ext: string): string { return map[ext] || '\uD83D\uDCC4'; } +function _AudioPlayer({ url, language, charCount }: { url: string; language?: string; charCount?: number }) { + const audioRef = useRef(null); + const [playing, setPlaying] = useState(false); + const [progress, setProgress] = useState(0); + const [duration, setDuration] = useState(0); + + useEffect(() => { + const audio = new Audio(url); + audioRef.current = audio; + + audio.addEventListener('loadedmetadata', () => setDuration(audio.duration)); + audio.addEventListener('timeupdate', () => { + if (audio.duration) setProgress(audio.currentTime / audio.duration); + }); + audio.addEventListener('ended', () => { setPlaying(false); setProgress(0); }); + audio.addEventListener('pause', () => setPlaying(false)); + audio.addEventListener('play', () => setPlaying(true)); + + audio.play().catch(() => {}); + + return () => { + audio.pause(); + audio.src = ''; + }; + }, [url]); + + const _togglePlay = useCallback(() => { + const audio = audioRef.current; + if (!audio) return; + if (playing) { audio.pause(); } else { audio.play().catch(() => {}); } + }, [playing]); + + const _stop = useCallback(() => { + const audio = audioRef.current; + if (!audio) return; + audio.pause(); + audio.currentTime = 0; + setPlaying(false); + setProgress(0); + }, []); + + const _formatTime = (s: number) => { + const m = Math.floor(s / 60); + const sec = Math.floor(s % 60); + return `${m}:${sec.toString().padStart(2, '0')}`; + }; + + return ( +
+ + +
+
+
+
+
+ + {duration > 0 ? _formatTime(progress * duration) : '0:00'} + + + {duration > 0 ? _formatTime(duration) : '--:--'} + +
+
+ + + + {language && ( + + {language} + + )} +
+ ); +} + function _CodeBlock({ className, children, diff --git a/src/pages/views/workspace/WorkspaceInput.tsx b/src/pages/views/workspace/WorkspaceInput.tsx index f91f450..5c736d1 100644 --- a/src/pages/views/workspace/WorkspaceInput.tsx +++ b/src/pages/views/workspace/WorkspaceInput.tsx @@ -1,12 +1,31 @@ /** * WorkspaceInput -- Prompt input with @file autocomplete, attachment bar, - * voice toggle, and data source selection. + * voice toggle (live transcript via SpeechRecognition), and data source selection. */ -import React, { useState, useCallback, useRef } from 'react'; +import React, { useState, useCallback, useRef, useEffect } from 'react'; import { ProviderMultiSelect } from '../../../components/ProviderSelector'; import type { WorkspaceFile, DataSource } from './useWorkspace'; +const _STT_LANGUAGES = [ + { code: 'de-DE', label: 'Deutsch' }, + { code: 'en-US', label: 'English (US)' }, + { code: 'en-GB', label: 'English (UK)' }, + { code: 'fr-FR', label: 'Francais' }, + { code: 'it-IT', label: 'Italiano' }, + { code: 'es-ES', label: 'Espanol' }, + { code: 'pt-BR', label: 'Portugues' }, + { code: 'nl-NL', label: 'Nederlands' }, + { code: 'pl-PL', label: 'Polski' }, + { code: 'ru-RU', label: 'Russkij' }, + { code: 'ja-JP', label: 'Japanese' }, + { code: 'zh-CN', label: 'Chinese' }, +]; + +function _getSpeechRecognitionApi(): (new () => SpeechRecognition) | null { + return (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition || null; +} + interface PendingFile { fileId: string; fileName: string; @@ -45,11 +64,20 @@ export const WorkspaceInput: React.FC = ({ const [showAutocomplete, setShowAutocomplete] = useState(false); const [autocompleteFilter, setAutocompleteFilter] = useState(''); const [voiceActive, setVoiceActive] = useState(false); + const [voiceLanguage, setVoiceLanguage] = useState(() => localStorage.getItem('workspace_stt_lang') || 'de-DE'); + const [liveTranscript, setLiveTranscript] = useState(''); + const [showLangPicker, setShowLangPicker] = useState(false); const [attachedFileIds, setAttachedFileIds] = useState([]); const [attachedDataSourceIds, setAttachedDataSourceIds] = useState([]); const textareaRef = useRef(null); - const mediaRecorderRef = useRef(null); - const chunksRef = useRef([]); + const recognitionRef = useRef(null); + const transcriptPartsRef = useRef([]); + const processedIndexRef = useRef(0); + const promptBeforeVoiceRef = useRef(''); + + useEffect(() => { + localStorage.setItem('workspace_stt_lang', voiceLanguage); + }, [voiceLanguage]); const _extractFileRefs = useCallback( (text: string): string[] => { @@ -133,42 +161,98 @@ export const WorkspaceInput: React.FC = ({ setAttachedDataSourceIds(prev => prev.filter(id => id !== dsId)); }, []); + const _stopRecognition = useCallback(() => { + if (recognitionRef.current) { + try { recognitionRef.current.stop(); } catch { /* ignore */ } + recognitionRef.current = null; + } + const finalText = transcriptPartsRef.current.join(' ').trim(); + if (finalText) { + setPrompt(prev => { + const base = promptBeforeVoiceRef.current; + return base ? `${base} ${finalText}` : finalText; + }); + } + setLiveTranscript(''); + transcriptPartsRef.current = []; + processedIndexRef.current = 0; + setVoiceActive(false); + }, []); + const _toggleVoice = useCallback(async () => { if (voiceActive) { - mediaRecorderRef.current?.stop(); - setVoiceActive(false); + _stopRecognition(); return; } + + const SpeechRecognitionApi = _getSpeechRecognitionApi(); + if (!SpeechRecognitionApi) { + console.error('SpeechRecognition not supported in this browser'); + return; + } + try { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - const recorder = new MediaRecorder(stream); - chunksRef.current = []; - recorder.ondataavailable = (e) => chunksRef.current.push(e.data); - recorder.onstop = async () => { - stream.getTracks().forEach(t => t.stop()); - const blob = new Blob(chunksRef.current, { type: 'audio/webm' }); - try { - const formData = new FormData(); - formData.append('audio', blob, 'voice.webm'); - const res = await fetch(`/api/workspace/${instanceId}/voice/transcribe`, { - method: 'POST', - body: formData, - }); - const data = await res.json(); - if (data.text) { - setPrompt(prev => prev + (prev ? ' ' : '') + data.text); - } - } catch (err) { - console.error('Voice transcription failed:', err); + await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + console.error('Microphone access denied'); + return; + } + + promptBeforeVoiceRef.current = prompt; + transcriptPartsRef.current = []; + processedIndexRef.current = 0; + setLiveTranscript(''); + + const recognition = new SpeechRecognitionApi(); + recognition.continuous = true; + recognition.interimResults = true; + recognition.lang = voiceLanguage; + + recognition.onresult = (event: SpeechRecognitionEvent) => { + const interimParts: string[] = []; + for (let i = processedIndexRef.current; i < event.results.length; i++) { + const r = event.results[i]; + if (r.isFinal) { + const text = r[0].transcript.trim(); + if (text) transcriptPartsRef.current.push(text); + processedIndexRef.current = i + 1; + } else { + const text = r[0].transcript.trim(); + if (text) interimParts.push(text); } - }; - recorder.start(); - mediaRecorderRef.current = recorder; + } + const finalSoFar = transcriptPartsRef.current.join(' '); + const interim = interimParts.join(' '); + const combined = [finalSoFar, interim].filter(Boolean).join(' '); + setLiveTranscript(combined); + + const base = promptBeforeVoiceRef.current; + const display = base ? `${base} ${combined}` : combined; + setPrompt(display); + }; + + recognition.onerror = (event: any) => { + if (event.error === 'no-speech' || event.error === 'aborted') return; + console.warn('SpeechRecognition error:', event.error); + }; + + recognition.onend = () => { + if (!recognitionRef.current) return; + processedIndexRef.current = 0; + setTimeout(() => { + if (!recognitionRef.current) return; + try { recognitionRef.current.start(); } catch { /* ignore */ } + }, 300); + }; + + try { + recognition.start(); + recognitionRef.current = recognition; setVoiceActive(true); } catch (err) { - console.error('Microphone access denied:', err); + console.error('SpeechRecognition start failed:', err); } - }, [voiceActive, instanceId]); + }, [voiceActive, voiceLanguage, prompt, _stopRecognition]); const filteredFiles = showAutocomplete ? files.filter(f => f.fileName.toLowerCase().includes(autocompleteFilter)) @@ -367,18 +451,57 @@ export const WorkspaceInput: React.FC = ({ /> )} - +
+ + + {showLangPicker && ( +
+ {_STT_LANGUAGES.map(lang => ( +
{ setVoiceLanguage(lang.code); setShowLangPicker(false); }} + style={{ + padding: '8px 12px', cursor: 'pointer', fontSize: 13, + background: lang.code === voiceLanguage ? 'var(--primary-color, #1976d2)' : 'transparent', + color: lang.code === voiceLanguage ? '#fff' : 'var(--text-primary, #333)', + }} + onMouseEnter={e => { if (lang.code !== voiceLanguage) e.currentTarget.style.background = '#f5f5f5'; }} + onMouseLeave={e => { if (lang.code !== voiceLanguage) e.currentTarget.style.background = ''; }} + > + {lang.label} ({lang.code}) +
+ ))} +
+ )} +
{isProcessing ? ( +
+ + + + + + + + ); +}; + +export default WorkspaceSettings; diff --git a/src/pages/views/workspace/WorkspaceSettingsPage.tsx b/src/pages/views/workspace/WorkspaceSettingsPage.tsx new file mode 100644 index 0000000..52ea9f7 --- /dev/null +++ b/src/pages/views/workspace/WorkspaceSettingsPage.tsx @@ -0,0 +1,72 @@ +/** + * WorkspaceSettingsPage -- Tabbed settings for the AI Workspace. + * + * First tab: Voice / Language (WorkspaceSettings). + * Additional tabs can be added here as needed. + */ + +import React, { useState } from 'react'; +import { useInstanceId } from '../../../hooks/useCurrentInstance'; +import { WorkspaceSettings } from './WorkspaceSettings'; + +type SettingsTab = 'voice'; + +const _TABS: { key: SettingsTab; label: string }[] = [ + { key: 'voice', label: 'Sprache & Stimme' }, +]; + +export const WorkspaceSettingsPage: React.FC = () => { + const instanceId = useInstanceId(); + const [activeTab, setActiveTab] = useState('voice'); + + if (!instanceId) { + return ( +
+ Keine Workspace-Instanz ausgewaehlt. +
+ ); + } + + return ( +
+ + +
+ {activeTab === 'voice' && ( + + )} +
+
+ ); +}; + +export default WorkspaceSettingsPage; diff --git a/src/pages/views/workspace/useWorkspace.ts b/src/pages/views/workspace/useWorkspace.ts index 6b3d77c..41f258f 100644 --- a/src/pages/views/workspace/useWorkspace.ts +++ b/src/pages/views/workspace/useWorkspace.ts @@ -273,7 +273,22 @@ export function useWorkspace(instanceId: string): UseWorkspaceReturn { }]); }, onVoiceResponse: (event) => { - _handleVoiceResponse(event); + const audioUrl = _buildAudioUrl(event); + if (audioUrl) { + const lang = event.item?.language || event.data?.language || ''; + const charCount = event.item?.charCount || event.data?.charCount || 0; + setMessages(prev => [ + ...prev, + { + id: `voice-${Date.now()}`, + role: 'assistant', + message: '', + _audioUrl: audioUrl, + _audioLang: lang, + _audioCharCount: charCount, + } as any, + ]); + } }, onComplete: (event) => { setIsProcessing(false); @@ -479,9 +494,9 @@ function _handleToolResult( }); } -function _handleVoiceResponse(event: SseEvent) { +function _buildAudioUrl(event: SseEvent): string | null { const audioData = event.item?.audio || event.data?.audio; - if (!audioData) return; + if (!audioData) return null; try { const byteChars = atob(audioData); @@ -490,11 +505,9 @@ function _handleVoiceResponse(event: SseEvent) { byteArray[i] = byteChars.charCodeAt(i); } const blob = new Blob([byteArray], { type: 'audio/mp3' }); - const url = URL.createObjectURL(blob); - const audio = new Audio(url); - audio.play().catch(err => console.error('TTS playback failed:', err)); - audio.onended = () => URL.revokeObjectURL(url); + return URL.createObjectURL(blob); } catch (err) { console.error('Failed to decode voice response:', err); + return null; } } diff --git a/src/types/mandate.ts b/src/types/mandate.ts index 73361e6..a9128cb 100644 --- a/src/types/mandate.ts +++ b/src/types/mandate.ts @@ -307,6 +307,7 @@ export const FEATURE_REGISTRY: Record = { icon: 'psychology', views: [ { code: 'dashboard', label: { de: 'Dashboard', en: 'Dashboard', fr: 'Tableau de bord' }, path: 'dashboard' }, + { code: 'settings', label: { de: 'Einstellungen', en: 'Settings', fr: 'Parametres' }, path: 'settings' }, ] }, };