From efa648d6fe621cb003df4e7d6bd10ee6b0264170 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Mon, 16 Feb 2026 00:07:40 +0100 Subject: [PATCH] feat: chat monitoring, auth join fix (Teams session), audio shutting-down guard, join mode selector, response channel, browser permission modals Co-authored-by: Cursor --- src/bot/chatProcedure.ts | 289 +++++++++++++++++++++++++++++++++++++++ src/bot/joinProcedure.ts | 34 ++++- src/bot/orchestrator.ts | 115 +++++++++++++++- src/types/index.ts | 20 ++- 4 files changed, 453 insertions(+), 5 deletions(-) create mode 100644 src/bot/chatProcedure.ts diff --git a/src/bot/chatProcedure.ts b/src/bot/chatProcedure.ts new file mode 100644 index 0000000..0de16c8 --- /dev/null +++ b/src/bot/chatProcedure.ts @@ -0,0 +1,289 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; + +export interface ChatMessageEntry { + speaker: string; + text: string; + timestamp: Date; + source: 'chat'; +} + +/** + * Handles reading and writing chat messages in a Teams meeting. + * + * Teams meeting chat is a separate panel that can be opened via the chat button. + * Messages are rendered as DOM elements that we observe via MutationObserver. + */ +export class ChatProcedure { + private _page: Page; + private _logger: Logger; + private _onChatMessage: (entry: ChatMessageEntry) => void; + private _isSubscribed: boolean = false; + private _lastMessageText: string = ''; + + constructor( + page: Page, + logger: Logger, + onChatMessage: (entry: ChatMessageEntry) => void + ) { + this._page = page; + this._logger = logger; + this._onChatMessage = onChatMessage; + } + + /** + * Open the chat panel and start monitoring messages. + */ + async enableChatMonitoring(): Promise { + this._logger.info('Enabling chat monitoring...'); + + // Open chat panel + await this._openChatPanel(); + + // Wait for chat to load + await this._page.waitForTimeout(2000); + + this._logger.info('Chat panel opened'); + } + + /** + * Open the chat panel by clicking the chat button. + */ + private async _openChatPanel(): Promise { + const chatButtonSelectors = [ + 'button[id="chat-button"]', + 'button[data-tid="chat-button"]', + 'button[aria-label*="Chat" i]', + 'button[aria-label*="chat" i]', + '#chat-button', + ]; + + for (const selector of chatButtonSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info(`Opened chat panel: ${selector}`); + return; + } + } catch { + // Continue + } + } + + this._logger.warn('Could not find chat button - chat monitoring will not work'); + } + + /** + * Subscribe to chat messages using MutationObserver. + */ + async subscribeToChatMessages(): Promise { + if (this._isSubscribed) { + this._logger.warn('Already subscribed to chat messages'); + return; + } + + this._isSubscribed = true; + this._logger.info('Subscribing to chat messages...'); + + // Expose callback from Node.js to browser + try { + await this._page.exposeFunction('__onChatMessageEvent', (msg: { + speaker: string; + text: string; + timestamp: string; + }) => { + this._handleChatMessage(msg); + }); + } catch { + // Function may already be exposed from a previous subscription + this._logger.debug('__onChatMessageEvent already exposed'); + } + + // Find chat container and set up observer + await this._page.evaluate(() => { + // Teams chat containers - try multiple selectors + const chatContainerSelectors = [ + '[data-tid="message-pane-list"]', + '[data-tid="chat-pane"]', + '[data-tid="chat-pane-list"]', + '.ts-message-list-container', + '[role="log"]', + ]; + + let chatContainer: Element | null = null; + for (const sel of chatContainerSelectors) { + chatContainer = document.querySelector(sel); + if (chatContainer) break; + } + + if (!chatContainer) { + // Fallback: find any scrollable container that looks like a chat + const candidates = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"]'); + if (candidates.length > 0) { + chatContainer = candidates[0]; + } + } + + if (!chatContainer) { + return; + } + + const observer = new MutationObserver((mutations) => { + for (const mutation of mutations) { + if (mutation.type === 'childList') { + mutation.addedNodes.forEach((node) => { + if (node.nodeType !== Node.ELEMENT_NODE) return; + const el = node as HTMLElement; + + // Look for message elements + const messageSelectors = [ + '[data-tid="chat-message"]', + '.fui-ChatMessage', + '[data-tid*="message-body"]', + ]; + + let messageEl: HTMLElement | null = null; + for (const sel of messageSelectors) { + messageEl = el.matches(sel) ? el : el.querySelector(sel); + if (messageEl) break; + } + + if (!messageEl) return; + + // Extract author + const authorSelectors = [ + '[data-tid="message-author"]', + '[data-tid="message-author-name"]', + '.fui-ChatMessage__author', + ]; + let author = 'Unknown'; + for (const sel of authorSelectors) { + const authorEl = messageEl.querySelector(sel) || el.querySelector(sel); + if (authorEl?.textContent) { + author = authorEl.textContent.trim(); + break; + } + } + + // Extract text + const bodySelectors = [ + '[data-tid="message-body"]', + '.fui-ChatMessage__body', + '[data-tid="chat-message-text"]', + ]; + let text = ''; + for (const sel of bodySelectors) { + const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel); + if (bodyEl) { + text = (bodyEl as HTMLElement).innerText?.trim() || ''; + break; + } + } + + if (text && text.length > 0) { + (window as any).__onChatMessageEvent({ + speaker: author, + text, + timestamp: new Date().toISOString(), + }); + } + }); + } + } + }); + + observer.observe(chatContainer, { childList: true, subtree: true }); + (window as any).__chatObserver = observer; + }); + + this._logger.info('Chat MutationObserver set up'); + } + + /** + * Handle a chat message event from the browser. + */ + private _handleChatMessage(msg: { speaker: string; text: string; timestamp: string }): void { + if (!this._isSubscribed || !msg.text) return; + + // Dedup + if (msg.text === this._lastMessageText) return; + this._lastMessageText = msg.text; + + this._logger.info(`Chat: [${msg.speaker}] ${msg.text}`); + + this._onChatMessage({ + speaker: msg.speaker, + text: msg.text, + timestamp: new Date(msg.timestamp), + source: 'chat', + }); + } + + /** + * Send a chat message in the meeting. + * Finds the chat input, types the message, and sends it. + */ + async sendChatMessage(text: string): Promise { + this._logger.info(`Sending chat message: ${text.substring(0, 60)}...`); + + try { + // Find chat input + const inputSelectors = [ + '[data-tid="ckeditor-replyConversation"]', + 'div[role="textbox"][data-tid*="chat"]', + 'div[role="textbox"][aria-label*="message" i]', + 'div[role="textbox"][aria-label*="Nachricht" i]', + 'div[contenteditable="true"][data-tid*="message"]', + 'div[contenteditable="true"][aria-label*="message" i]', + ]; + + let input: any = null; + for (const selector of inputSelectors) { + input = await this._page.$(selector); + if (input) break; + } + + if (!input) { + this._logger.warn('Could not find chat input field'); + return false; + } + + // Click to focus + await input.click(); + await this._page.waitForTimeout(200); + + // Type the message + await this._page.keyboard.type(text, { delay: 10 }); + await this._page.waitForTimeout(200); + + // Press Enter to send + await this._page.keyboard.press('Enter'); + this._logger.info('Chat message sent'); + return true; + + } catch (error) { + this._logger.error('Error sending chat message:', error); + return false; + } + } + + /** + * Stop monitoring chat messages. + */ + async unsubscribe(): Promise { + this._isSubscribed = false; + + try { + await this._page.evaluate(() => { + if ((window as any).__chatObserver) { + (window as any).__chatObserver.disconnect(); + } + }); + } catch { + // Page might be closed + } + + this._logger.info('Unsubscribed from chat messages'); + } +} diff --git a/src/bot/joinProcedure.ts b/src/bot/joinProcedure.ts index 9567d53..7a64fb6 100644 --- a/src/bot/joinProcedure.ts +++ b/src/bot/joinProcedure.ts @@ -40,7 +40,7 @@ export class JoinProcedure { this._logger.info(`Resolved launch URL: ${launchUrl} (authenticated: ${this._isAuthenticated})`); } catch (error) { this._logger.warn(`Could not resolve launch URL, using fallback: ${error}`); - launchUrl = getMeetingLaunchUrl(meetingUrl); + launchUrl = getMeetingLaunchUrl(meetingUrl, this._isAuthenticated); } this._logger.info(`Navigating to meeting: ${launchUrl}`); @@ -265,6 +265,38 @@ export class JoinProcedure { // No modal found - that's fine, it means devices were detected properly } + /** + * Dismiss the "Manage windows on all your displays" permission dialog. + * Teams v2 shows this after joining. Must click "Allow" to proceed. + * Note: This is a browser permission prompt handled by Playwright's context permissions, + * but Teams may also show its own UI overlay. + */ + async dismissBrowserPermissionModals(): Promise { + const permissionSelectors = [ + 'button:has-text("Allow")', + 'button:has-text("Erlauben")', + 'button:has-text("Zulassen")', + ]; + + for (const selector of permissionSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + // Only click if it looks like a permission dialog (not a meeting button) + const text = await button.evaluate(el => el.closest('[role="dialog"]')?.textContent || ''); + if (text.includes('manage') || text.includes('Manage') || text.includes('display') || text.includes('window')) { + await button.click(); + this._logger.info(`Dismissed browser permission modal: ${selector}`); + await this._page.waitForTimeout(1000); + return; + } + } + } catch { + // Continue + } + } + } + /** * Check if the bot is currently in the lobby (waiting to be admitted). * Primary check: text "Someone will let you in shortly" (confirmed by Recall.ai). diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index d34e261..1e20f90 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -7,10 +7,11 @@ import WebSocket from 'ws'; import { config } from '../config'; import { createSessionLogger } from '../utils/logger'; -import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage } from '../types'; +import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage } from '../types'; import { JoinProcedure } from './joinProcedure'; import { CaptionsProcedure } from './captionsProcedure'; import { AudioProcedure } from './audioProcedure'; +import { ChatProcedure, ChatMessageEntry } from './chatProcedure'; import { isValidMeetingUrl } from './meetingUrlParser'; export interface OrchestratorCallbacks { @@ -56,6 +57,7 @@ export class BotOrchestrator { private _joinProcedure: JoinProcedure | null = null; private _captionsProcedure: CaptionsProcedure | null = null; private _audioProcedure: AudioProcedure | null = null; + private _chatProcedure: ChatProcedure | null = null; private _state: BotState = 'idle'; private _isShuttingDown: boolean = false; @@ -142,6 +144,23 @@ export class BotOrchestrator { if (!authSuccess) { throw new Error('Microsoft authentication failed'); } + + // CRITICAL: After auth, navigate to Teams web app first to establish + // a Teams session. Without this, Teams redirects to anonymous mode + // when navigating directly to the meeting URL. + this._logger.info('Establishing Teams session after auth...'); + try { + await this._page!.goto('https://teams.microsoft.com', { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + // Wait for Teams v2 to load (look for any Teams UI element) + await this._page!.waitForTimeout(5000); + const teamsUrl = this._page!.url(); + this._logger.info(`Teams session established at: ${teamsUrl}`); + } catch (teamsNavError) { + this._logger.warn(`Teams session establishment failed (non-fatal): ${teamsNavError}`); + } } // Update JoinProcedure with correct auth state @@ -152,6 +171,24 @@ export class BotOrchestrator { // Navigate to meeting and handle launcher await this._joinProcedure.startMeetingLauncherFlow(this._meetingUrl); + // After navigation, check if Teams put us on the anonymous page despite auth + if (authenticate && this._page) { + const currentUrl = this._page.url(); + if (currentUrl.includes('anon=true') || currentUrl.includes('light-meetings/launch')) { + this._logger.warn(`Teams redirected to anonymous mode despite auth. URL: ${currentUrl}`); + // Strip anon params and re-navigate + const cleanUrl = currentUrl + .replace(/[&?]anon=true/gi, '') + .replace(/%26anon%3Dtrue/gi, ''); + this._logger.info(`Re-navigating without anon: ${cleanUrl}`); + await this._page.goto(cleanUrl, { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + await this._page.waitForTimeout(3000); + } + } + // Set virtual background if configured (must be done on pre-join screen, before "Join now") if (this._options.backgroundImageUrl && this._page && authenticate) { try { @@ -179,11 +216,17 @@ export class BotOrchestrator { this._setState('in_meeting'); this._logger.info(`Bot joined the meeting! (authenticated: ${authenticate})`); + // Dismiss any post-join permission modals (e.g. "Manage windows on all displays") + await this._joinProcedure!.dismissBrowserPermissionModals(); + // Initialize audio await this._audioProcedure!.initialize(); // Enable and subscribe to captions await this._enableCaptions(); + + // Enable chat monitoring + await this._enableChat(); } /** @@ -200,6 +243,7 @@ export class BotOrchestrator { this._joinProcedure = null; this._captionsProcedure = null; this._audioProcedure = null; + this._chatProcedure = null; } catch { // Ignore cleanup errors } @@ -276,6 +320,11 @@ export class BotOrchestrator { this.playAudio(audioMsg.audio.data, audioMsg.audio.format); break; + case 'sendChatMessage': + const chatMsg = message as SendChatMessage; + this.sendChatMessageToMeeting(chatMsg.text); + break; + case 'pong': // Heartbeat response break; @@ -384,10 +433,13 @@ export class BotOrchestrator { try { this._setState('leaving'); - // Unsubscribe from captions + // Unsubscribe from captions and chat if (this._captionsProcedure) { await this._captionsProcedure.unsubscribe(); } + if (this._chatProcedure) { + await this._chatProcedure.unsubscribe(); + } // Clean up audio if (this._audioProcedure) { @@ -419,6 +471,10 @@ export class BotOrchestrator { * Play audio in the meeting. */ async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise { + if (this._isShuttingDown) { + this._logger.debug('Ignoring playAudio - bot is shutting down'); + return; + } if (this._state !== 'in_meeting' || !this._audioProcedure) { this._logger.warn('Cannot play audio - not in meeting'); return; @@ -467,6 +523,20 @@ export class BotOrchestrator { this._options.language ); this._audioProcedure = new AudioProcedure(this._page, this._logger); + this._chatProcedure = new ChatProcedure( + this._page, + this._logger, + (entry: ChatMessageEntry) => { + // Send chat message to Gateway as a special transcript + this._sendChatMessage(entry.speaker, entry.text); + this._callbacks.onTranscript({ + speaker: entry.speaker, + text: entry.text, + timestamp: entry.timestamp, + isFinal: true, + }); + } + ); // Inject audio getUserMedia override BEFORE any navigation // This ensures Teams gets our controlled audio stream when it calls getUserMedia @@ -574,6 +644,47 @@ export class BotOrchestrator { } } + /** + * Enable chat monitoring. + */ + private async _enableChat(): Promise { + try { + await this._chatProcedure!.enableChatMonitoring(); + await this._chatProcedure!.subscribeToChatMessages(); + this._logger.info('Chat monitoring enabled and subscribed'); + } catch (error) { + this._logger.warn('Could not enable chat monitoring:', error); + // Continue without chat - not a fatal error + } + } + + /** + * Send a chat message event to the Gateway. + */ + private _sendChatMessage(speaker: string, text: string): void { + const message: ChatMessage = { + type: 'chatMessage', + sessionId: this._sessionId, + chat: { + speaker, + text, + timestamp: new Date().toISOString(), + }, + }; + this._sendToGateway(message); + } + + /** + * Send a text message to the meeting chat. + */ + async sendChatMessageToMeeting(text: string): Promise { + if (this._isShuttingDown || this._state !== 'in_meeting' || !this._chatProcedure) { + this._logger.warn('Cannot send chat message - not in meeting'); + return; + } + await this._chatProcedure.sendChatMessage(text); + } + /** * Update the bot state and notify callbacks + Gateway. */ diff --git a/src/types/index.ts b/src/types/index.ts index c449227..415e789 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -39,8 +39,24 @@ export interface LeaveMeetingMessage { sessionId: string; } -export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage; -export type BotToGateway = TranscriptMessage | StatusMessage; +export interface ChatMessage { + type: 'chatMessage'; + sessionId: string; + chat: { + speaker: string; + text: string; + timestamp: string; + }; +} + +export interface SendChatMessage { + type: 'sendChatMessage'; + sessionId: string; + text: string; +} + +export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage | SendChatMessage; +export type BotToGateway = TranscriptMessage | StatusMessage | ChatMessage; // Bot State export type BotState =