From 777bc198a27383d16e2238e5ee01191ef8712519 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 18 Feb 2026 17:50:13 +0100 Subject: [PATCH] feat: fix chat in both-mode, add TeamsActionsService for AI commands Co-authored-by: Cursor --- src/bot/orchestrator.ts | 57 ++++++- src/bot/teamsActionsService.ts | 287 +++++++++++++++++++++++++++++++++ 2 files changed, 338 insertions(+), 6 deletions(-) create mode 100644 src/bot/teamsActionsService.ts diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index ac9d25a..6027084 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -14,6 +14,7 @@ import { AudioProcedure } from './audioProcedure'; import { AudioCaptureProcedure } from './audioCaptureProcedure'; import { ChatProcedure, ChatMessageEntry } from './chatProcedure'; import { AuthProcedure } from './authProcedure'; +import { TeamsActionsService } from './teamsActionsService'; import { isValidMeetingUrl } from './meetingUrlParser'; // Camera / fake video injection is disabled for now to focus on stability. @@ -64,6 +65,7 @@ export class BotOrchestrator { private _audioProcedure: AudioProcedure | null = null; private _audioCaptureProcedure: AudioCaptureProcedure | null = null; private _chatProcedure: ChatProcedure | null = null; + private _teamsActions: TeamsActionsService | null = null; private _state: BotState = 'idle'; private _isShuttingDown: boolean = false; @@ -617,7 +619,9 @@ export class BotOrchestrator { }); this._gatewayWs.on('message', (data) => { - this._handleGatewayMessage(data.toString()); + this._handleGatewayMessage(data.toString()).catch((err) => { + this._logger.error('Unhandled error in gateway message handler:', err); + }); }); this._gatewayWs.on('close', (code, reason) => { @@ -641,20 +645,27 @@ export class BotOrchestrator { /** * Handle incoming messages from the Gateway. + * Async operations are awaited to ensure proper error handling + * and serialized execution (e.g. audio + chat don't interfere). */ - private _handleGatewayMessage(data: string): void { + private async _handleGatewayMessage(data: string): Promise { try { const message = JSON.parse(data); switch (message.type) { case 'playAudio': const audioMsg = message as PlayAudioMessage; - this.playAudio(audioMsg.audio.data, audioMsg.audio.format); + await this.playAudio(audioMsg.audio.data, audioMsg.audio.format); break; case 'sendChatMessage': const chatMsg = message as SendChatMessage; - this.sendChatMessageToMeeting(chatMsg.text); + this._logger.info(`Gateway sendChatMessage received: ${chatMsg.text?.substring(0, 60)}...`); + try { + await this.sendChatMessageToMeeting(chatMsg.text); + } catch (chatErr) { + this._logger.error(`Failed to send chat message to meeting: ${chatErr}`); + } break; case 'stopAudio': @@ -664,15 +675,18 @@ export class BotOrchestrator { } break; + case 'botCommand': + await this._handleBotCommand(message.command, message.params || {}); + break; + case 'pong': - // Heartbeat response break; default: this._logger.debug('Unknown Gateway message type:', message.type); } } catch (error) { - this._logger.error('Error parsing Gateway message:', error); + this._logger.error('Error handling Gateway message:', error); } } @@ -925,6 +939,7 @@ export class BotOrchestrator { this._options.language ); this._audioProcedure = new AudioProcedure(this._page, this._logger); + this._teamsActions = new TeamsActionsService(this._page, this._logger); this._chatProcedure = new ChatProcedure( this._page, this._logger, @@ -1203,6 +1218,36 @@ export class BotOrchestrator { await this._chatProcedure.sendChatMessage(text); } + /** + * Handle structured commands from the Gateway (issued by AI). + */ + private async _handleBotCommand(command: string, params: Record): Promise { + if (!this._teamsActions || this._state !== 'in_meeting') { + this._logger.warn(`Cannot execute command '${command}' — not in meeting`); + return; + } + + this._logger.info(`Executing bot command: ${command} params=${JSON.stringify(params)}`); + + try { + switch (command) { + case 'toggleTranscript': + await this._teamsActions.toggleTranscript(params.enable !== false); + break; + case 'toggleMic': + await this._teamsActions.toggleMic(params.enable !== false); + break; + case 'toggleCamera': + await this._teamsActions.toggleCamera(params.enable !== false); + break; + default: + this._logger.warn(`Unknown bot command: ${command}`); + } + } catch (err) { + this._logger.error(`Bot command '${command}' failed: ${err}`); + } + } + /** * Update the bot state and notify callbacks + Gateway. */ diff --git a/src/bot/teamsActionsService.ts b/src/bot/teamsActionsService.ts new file mode 100644 index 0000000..94ed31f --- /dev/null +++ b/src/bot/teamsActionsService.ts @@ -0,0 +1,287 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; + +/** + * Service center for all Teams meeting UI actions. + * + * Each method is idempotent and state-aware: it checks the current state + * before acting (e.g. won't toggle transcript OFF if it's already ON). + * This makes calls safe to repeat and resilient to UI flicker. + */ +export class TeamsActionsService { + private _page: Page; + private _logger: Logger; + + constructor(page: Page, logger: Logger) { + this._page = page; + this._logger = logger; + } + + // ========================================================================= + // Transcript / Captions + // ========================================================================= + + async toggleTranscript(enable: boolean): Promise { + this._logger.info(`TeamsActions: toggleTranscript enable=${enable}`); + + try { + // Open "More" menu + if (!(await this._openMoreMenu())) return false; + + // Try "Language and speech" → submenu → "Show live captions" + const captionsBtn = await this._findCaptionsButton(); + if (captionsBtn) { + const ariaChecked = await captionsBtn.getAttribute('aria-checked'); + const text = await captionsBtn.evaluate((el: HTMLElement) => el.textContent?.trim() || ''); + const isCurrentlyOn = ariaChecked === 'true' || text.toLowerCase().includes('hide'); + + if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) { + this._logger.info(`TeamsActions: Transcript already ${enable ? 'ON' : 'OFF'} — no action needed`); + await this._page.keyboard.press('Escape'); + return true; + } + + await captionsBtn.click(); + this._logger.info(`TeamsActions: Transcript toggled ${enable ? 'ON' : 'OFF'}`); + await this._page.waitForTimeout(1000); + return true; + } + + await this._page.keyboard.press('Escape'); + this._logger.warn('TeamsActions: Could not find captions button'); + return false; + } catch (err) { + this._logger.warn(`TeamsActions: toggleTranscript error: ${err}`); + await this._page.keyboard.press('Escape').catch(() => {}); + return false; + } + } + + // ========================================================================= + // Chat + // ========================================================================= + + async sendChatMessage(text: string): Promise { + this._logger.info(`TeamsActions: sendChatMessage: ${text.substring(0, 60)}...`); + + try { + await this._ensureChatPanelOpen(); + + const inputSelectors = [ + '[data-tid="ckeditor-replyConversation"]', + 'div[role="textbox"][data-tid*="chat"]', + 'div[role="textbox"][aria-label*="message" i]', + 'div[role="textbox"][aria-label*="Nachricht" i]', + 'div[contenteditable="true"][data-tid*="message"]', + 'div[contenteditable="true"][aria-label*="message" i]', + ]; + + let input: any = null; + for (const selector of inputSelectors) { + input = await this._page.$(selector); + if (input) break; + } + + if (!input) { + this._logger.warn('TeamsActions: Could not find chat input field'); + return false; + } + + await input.click(); + await this._page.waitForTimeout(200); + await this._page.keyboard.type(text, { delay: 10 }); + await this._page.waitForTimeout(200); + await this._page.keyboard.press('Enter'); + + this._logger.info('TeamsActions: Chat message sent'); + return true; + } catch (err) { + this._logger.error(`TeamsActions: sendChatMessage error: ${err}`); + return false; + } + } + + // ========================================================================= + // Microphone + // ========================================================================= + + async toggleMic(enable: boolean): Promise { + this._logger.info(`TeamsActions: toggleMic enable=${enable}`); + + try { + const micBtn = await this._page.$('button#microphone-button'); + if (!micBtn) { + const fallbacks = [ + 'button[data-inp="microphone-button"]', + 'button[aria-label*="microphone" i]', + 'button[aria-label*="Mikrofon" i]', + ]; + for (const sel of fallbacks) { + const btn = await this._page.$(sel); + if (btn) return this._toggleMediaButton(btn, enable, 'mic'); + } + this._logger.warn('TeamsActions: Microphone button not found'); + return false; + } + + return this._toggleMediaButton(micBtn, enable, 'mic'); + } catch (err) { + this._logger.warn(`TeamsActions: toggleMic error: ${err}`); + return false; + } + } + + // ========================================================================= + // Camera + // ========================================================================= + + async toggleCamera(enable: boolean): Promise { + this._logger.info(`TeamsActions: toggleCamera enable=${enable}`); + + try { + const camBtn = await this._page.$('button#video-button'); + if (!camBtn) { + const fallbacks = [ + 'button[data-inp="video-button"]', + 'button[aria-label*="camera" i]', + 'button[aria-label*="Camera" i]', + 'button[aria-label*="Video" i]', + ]; + for (const sel of fallbacks) { + const btn = await this._page.$(sel); + if (btn) return this._toggleMediaButton(btn, enable, 'camera'); + } + this._logger.warn('TeamsActions: Camera button not found'); + return false; + } + + return this._toggleMediaButton(camBtn, enable, 'camera'); + } catch (err) { + this._logger.warn(`TeamsActions: toggleCamera error: ${err}`); + return false; + } + } + + // ========================================================================= + // Internal Helpers + // ========================================================================= + + private async _toggleMediaButton( + btn: any, + enable: boolean, + name: string, + ): Promise { + const state = await btn.evaluate((el: HTMLElement) => ({ + dataState: el.getAttribute('data-state') || '', + ariaLabel: el.getAttribute('aria-label') || '', + })); + + const isOff = + state.dataState.includes('off') || + state.ariaLabel.toLowerCase().includes('turn on') || + state.ariaLabel.toLowerCase().includes('einschalten'); + + const isCurrentlyOn = !isOff; + + if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) { + this._logger.info(`TeamsActions: ${name} already ${enable ? 'ON' : 'OFF'}`); + return true; + } + + await btn.click(); + this._logger.info(`TeamsActions: ${name} toggled ${enable ? 'ON' : 'OFF'}`); + await this._page.waitForTimeout(500); + return true; + } + + private async _openMoreMenu(): Promise { + const selectors = [ + 'button[id="callingButtons-showMoreBtn"]', + '[data-tid="callingButtons-showMoreBtn"]', + 'button[aria-label*="More actions"]', + 'button[aria-label*="More"]', + '[data-tid="more-button"]', + ]; + + for (const selector of selectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + await this._page.waitForTimeout(1000); + return true; + } + } catch { + // Continue + } + } + this._logger.warn('TeamsActions: Could not find More actions menu'); + return false; + } + + private async _findCaptionsButton(): Promise { + // Direct button + let btn = await this._page.$('#closed-captions-button'); + if (btn) return btn; + + // "Language and speech" → submenu + const langSpeechSelectors = [ + '#LanguageSpeechMenuControl-id', + '[data-tid="LanguageSpeechMenuControl-id"]', + 'div[role="menuitem"]:has-text("Language and speech")', + 'div[role="menuitem"]:has-text("Sprache und Spracheingabe")', + ]; + + for (const sel of langSpeechSelectors) { + try { + const item = await this._page.$(sel); + if (item) { + await item.click(); + await this._page.waitForTimeout(1500); + btn = await this._page.$('#closed-captions-button'); + if (btn) return btn; + break; + } + } catch { + // Continue + } + } + return null; + } + + private async _ensureChatPanelOpen(): Promise { + const alreadyOpen = await this._page.evaluate(() => { + const chatBtn = document.querySelector( + 'button[id="chat-button"], button[data-tid="chat-button"]', + ) as HTMLElement | null; + if (chatBtn && chatBtn.getAttribute('aria-pressed') === 'true') return true; + const chatInput = document.querySelector( + '[data-tid="ckeditor-replyConversation"], div[role="textbox"][data-tid*="chat"]', + ) as HTMLElement | null; + return chatInput ? chatInput.offsetHeight > 0 : false; + }); + + if (alreadyOpen) return; + + const chatButtonSelectors = [ + 'button[id="chat-button"]', + 'button[data-tid="chat-button"]', + 'button[aria-label*="Chat" i]', + '#chat-button', + ]; + + for (const selector of chatButtonSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info(`TeamsActions: Opened chat panel: ${selector}`); + await this._page.waitForTimeout(1000); + return; + } + } catch { + // Continue + } + } + } +}