feat: chat monitoring, auth join fix (Teams session), audio shutting-down guard, join mode selector, response channel, browser permission modals

Co-authored-by: Cursor <cursoragent@cursor.com>

src/bot/chatProcedure.ts

@@ -0,0 +1,289 @@
+import { Page } from 'playwright';
+import { Logger } from 'winston';
+
+export interface ChatMessageEntry {
+  speaker: string;
+  text: string;
+  timestamp: Date;
+  source: 'chat';
+}
+
+/**
+ * Handles reading and writing chat messages in a Teams meeting.
+ * 
+ * Teams meeting chat is a separate panel that can be opened via the chat button.
+ * Messages are rendered as DOM elements that we observe via MutationObserver.
+ */
+export class ChatProcedure {
+  private _page: Page;
+  private _logger: Logger;
+  private _onChatMessage: (entry: ChatMessageEntry) => void;
+  private _isSubscribed: boolean = false;
+  private _lastMessageText: string = '';
+
+  constructor(
+    page: Page,
+    logger: Logger,
+    onChatMessage: (entry: ChatMessageEntry) => void
+  ) {
+    this._page = page;
+    this._logger = logger;
+    this._onChatMessage = onChatMessage;
+  }
+
+  /**
+   * Open the chat panel and start monitoring messages.
+   */
+  async enableChatMonitoring(): Promise<void> {
+    this._logger.info('Enabling chat monitoring...');
+
+    // Open chat panel
+    await this._openChatPanel();
+
+    // Wait for chat to load
+    await this._page.waitForTimeout(2000);
+
+    this._logger.info('Chat panel opened');
+  }
+
+  /**
+   * Open the chat panel by clicking the chat button.
+   */
+  private async _openChatPanel(): Promise<void> {
+    const chatButtonSelectors = [
+      'button[id="chat-button"]',
+      'button[data-tid="chat-button"]',
+      'button[aria-label*="Chat" i]',
+      'button[aria-label*="chat" i]',
+      '#chat-button',
+    ];
+
+    for (const selector of chatButtonSelectors) {
+      try {
+        const button = await this._page.$(selector);
+        if (button) {
+          await button.click();
+          this._logger.info(`Opened chat panel: ${selector}`);
+          return;
+        }
+      } catch {
+        // Continue
+      }
+    }
+
+    this._logger.warn('Could not find chat button - chat monitoring will not work');
+  }
+
+  /**
+   * Subscribe to chat messages using MutationObserver.
+   */
+  async subscribeToChatMessages(): Promise<void> {
+    if (this._isSubscribed) {
+      this._logger.warn('Already subscribed to chat messages');
+      return;
+    }
+
+    this._isSubscribed = true;
+    this._logger.info('Subscribing to chat messages...');
+
+    // Expose callback from Node.js to browser
+    try {
+      await this._page.exposeFunction('__onChatMessageEvent', (msg: {
+        speaker: string;
+        text: string;
+        timestamp: string;
+      }) => {
+        this._handleChatMessage(msg);
+      });
+    } catch {
+      // Function may already be exposed from a previous subscription
+      this._logger.debug('__onChatMessageEvent already exposed');
+    }
+
+    // Find chat container and set up observer
+    await this._page.evaluate(() => {
+      // Teams chat containers - try multiple selectors
+      const chatContainerSelectors = [
+        '[data-tid="message-pane-list"]',
+        '[data-tid="chat-pane"]',
+        '[data-tid="chat-pane-list"]',
+        '.ts-message-list-container',
+        '[role="log"]',
+      ];
+
+      let chatContainer: Element | null = null;
+      for (const sel of chatContainerSelectors) {
+        chatContainer = document.querySelector(sel);
+        if (chatContainer) break;
+      }
+
+      if (!chatContainer) {
+        // Fallback: find any scrollable container that looks like a chat
+        const candidates = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"]');
+        if (candidates.length > 0) {
+          chatContainer = candidates[0];
+        }
+      }
+
+      if (!chatContainer) {
+        return;
+      }
+
+      const observer = new MutationObserver((mutations) => {
+        for (const mutation of mutations) {
+          if (mutation.type === 'childList') {
+            mutation.addedNodes.forEach((node) => {
+              if (node.nodeType !== Node.ELEMENT_NODE) return;
+              const el = node as HTMLElement;
+
+              // Look for message elements
+              const messageSelectors = [
+                '[data-tid="chat-message"]',
+                '.fui-ChatMessage',
+                '[data-tid*="message-body"]',
+              ];
+
+              let messageEl: HTMLElement | null = null;
+              for (const sel of messageSelectors) {
+                messageEl = el.matches(sel) ? el : el.querySelector(sel);
+                if (messageEl) break;
+              }
+
+              if (!messageEl) return;
+
+              // Extract author
+              const authorSelectors = [
+                '[data-tid="message-author"]',
+                '[data-tid="message-author-name"]',
+                '.fui-ChatMessage__author',
+              ];
+              let author = 'Unknown';
+              for (const sel of authorSelectors) {
+                const authorEl = messageEl.querySelector(sel) || el.querySelector(sel);
+                if (authorEl?.textContent) {
+                  author = authorEl.textContent.trim();
+                  break;
+                }
+              }
+
+              // Extract text
+              const bodySelectors = [
+                '[data-tid="message-body"]',
+                '.fui-ChatMessage__body',
+                '[data-tid="chat-message-text"]',
+              ];
+              let text = '';
+              for (const sel of bodySelectors) {
+                const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel);
+                if (bodyEl) {
+                  text = (bodyEl as HTMLElement).innerText?.trim() || '';
+                  break;
+                }
+              }
+
+              if (text && text.length > 0) {
+                (window as any).__onChatMessageEvent({
+                  speaker: author,
+                  text,
+                  timestamp: new Date().toISOString(),
+                });
+              }
+            });
+          }
+        }
+      });
+
+      observer.observe(chatContainer, { childList: true, subtree: true });
+      (window as any).__chatObserver = observer;
+    });
+
+    this._logger.info('Chat MutationObserver set up');
+  }
+
+  /**
+   * Handle a chat message event from the browser.
+   */
+  private _handleChatMessage(msg: { speaker: string; text: string; timestamp: string }): void {
+    if (!this._isSubscribed || !msg.text) return;
+
+    // Dedup
+    if (msg.text === this._lastMessageText) return;
+    this._lastMessageText = msg.text;
+
+    this._logger.info(`Chat: [${msg.speaker}] ${msg.text}`);
+
+    this._onChatMessage({
+      speaker: msg.speaker,
+      text: msg.text,
+      timestamp: new Date(msg.timestamp),
+      source: 'chat',
+    });
+  }
+
+  /**
+   * Send a chat message in the meeting.
+   * Finds the chat input, types the message, and sends it.
+   */
+  async sendChatMessage(text: string): Promise<boolean> {
+    this._logger.info(`Sending chat message: ${text.substring(0, 60)}...`);
+
+    try {
+      // Find chat input
+      const inputSelectors = [
+        '[data-tid="ckeditor-replyConversation"]',
+        'div[role="textbox"][data-tid*="chat"]',
+        'div[role="textbox"][aria-label*="message" i]',
+        'div[role="textbox"][aria-label*="Nachricht" i]',
+        'div[contenteditable="true"][data-tid*="message"]',
+        'div[contenteditable="true"][aria-label*="message" i]',
+      ];
+
+      let input: any = null;
+      for (const selector of inputSelectors) {
+        input = await this._page.$(selector);
+        if (input) break;
+      }
+
+      if (!input) {
+        this._logger.warn('Could not find chat input field');
+        return false;
+      }
+
+      // Click to focus
+      await input.click();
+      await this._page.waitForTimeout(200);
+
+      // Type the message
+      await this._page.keyboard.type(text, { delay: 10 });
+      await this._page.waitForTimeout(200);
+
+      // Press Enter to send
+      await this._page.keyboard.press('Enter');
+      this._logger.info('Chat message sent');
+      return true;
+
+    } catch (error) {
+      this._logger.error('Error sending chat message:', error);
+      return false;
+    }
+  }
+
+  /**
+   * Stop monitoring chat messages.
+   */
+  async unsubscribe(): Promise<void> {
+    this._isSubscribed = false;
+
+    try {
+      await this._page.evaluate(() => {
+        if ((window as any).__chatObserver) {
+          (window as any).__chatObserver.disconnect();
+        }
+      });
+    } catch {
+      // Page might be closed
+    }
+
+    this._logger.info('Unsubscribed from chat messages');
+  }
+}

src/bot/joinProcedure.ts

@@ -40,7 +40,7 @@ export class JoinProcedure {
       this._logger.info(`Resolved launch URL: ${launchUrl} (authenticated: ${this._isAuthenticated})`);
     } catch (error) {
       this._logger.warn(`Could not resolve launch URL, using fallback: ${error}`);
-      launchUrl = getMeetingLaunchUrl(meetingUrl);
+      launchUrl = getMeetingLaunchUrl(meetingUrl, this._isAuthenticated);
     }
 
     this._logger.info(`Navigating to meeting: ${launchUrl}`);
@@ -265,6 +265,38 @@ export class JoinProcedure {
     // No modal found - that's fine, it means devices were detected properly
   }
 
+  /**
+   * Dismiss the "Manage windows on all your displays" permission dialog.
+   * Teams v2 shows this after joining. Must click "Allow" to proceed.
+   * Note: This is a browser permission prompt handled by Playwright's context permissions,
+   * but Teams may also show its own UI overlay.
+   */
+  async dismissBrowserPermissionModals(): Promise<void> {
+    const permissionSelectors = [
+      'button:has-text("Allow")',
+      'button:has-text("Erlauben")',
+      'button:has-text("Zulassen")',
+    ];
+
+    for (const selector of permissionSelectors) {
+      try {
+        const button = await this._page.$(selector);
+        if (button) {
+          // Only click if it looks like a permission dialog (not a meeting button)
+          const text = await button.evaluate(el => el.closest('[role="dialog"]')?.textContent || '');
+          if (text.includes('manage') || text.includes('Manage') || text.includes('display') || text.includes('window')) {
+            await button.click();
+            this._logger.info(`Dismissed browser permission modal: ${selector}`);
+            await this._page.waitForTimeout(1000);
+            return;
+          }
+        }
+      } catch {
+        // Continue
+      }
+    }
+  }
+
   /**
    * Check if the bot is currently in the lobby (waiting to be admitted).
    * Primary check: text "Someone will let you in shortly" (confirmed by Recall.ai).

src/bot/orchestrator.ts

@@ -7,10 +7,11 @@ import WebSocket from 'ws';
 
 import { config } from '../config';
 import { createSessionLogger } from '../utils/logger';
-import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage } from '../types';
+import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage } from '../types';
 import { JoinProcedure } from './joinProcedure';
 import { CaptionsProcedure } from './captionsProcedure';
 import { AudioProcedure } from './audioProcedure';
+import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
 import { isValidMeetingUrl } from './meetingUrlParser';
 
 export interface OrchestratorCallbacks {
@@ -56,6 +57,7 @@ export class BotOrchestrator {
   private _joinProcedure: JoinProcedure | null = null;
   private _captionsProcedure: CaptionsProcedure | null = null;
   private _audioProcedure: AudioProcedure | null = null;
+  private _chatProcedure: ChatProcedure | null = null;
 
   private _state: BotState = 'idle';
   private _isShuttingDown: boolean = false;
@@ -142,6 +144,23 @@ export class BotOrchestrator {
       if (!authSuccess) {
         throw new Error('Microsoft authentication failed');
       }
+
+      // CRITICAL: After auth, navigate to Teams web app first to establish
+      // a Teams session. Without this, Teams redirects to anonymous mode
+      // when navigating directly to the meeting URL.
+      this._logger.info('Establishing Teams session after auth...');
+      try {
+        await this._page!.goto('https://teams.microsoft.com', {
+          waitUntil: 'domcontentloaded',
+          timeout: 30000,
+        });
+        // Wait for Teams v2 to load (look for any Teams UI element)
+        await this._page!.waitForTimeout(5000);
+        const teamsUrl = this._page!.url();
+        this._logger.info(`Teams session established at: ${teamsUrl}`);
+      } catch (teamsNavError) {
+        this._logger.warn(`Teams session establishment failed (non-fatal): ${teamsNavError}`);
+      }
     }
 
     // Update JoinProcedure with correct auth state
@@ -152,6 +171,24 @@ export class BotOrchestrator {
     // Navigate to meeting and handle launcher
     await this._joinProcedure.startMeetingLauncherFlow(this._meetingUrl);
 
+    // After navigation, check if Teams put us on the anonymous page despite auth
+    if (authenticate && this._page) {
+      const currentUrl = this._page.url();
+      if (currentUrl.includes('anon=true') || currentUrl.includes('light-meetings/launch')) {
+        this._logger.warn(`Teams redirected to anonymous mode despite auth. URL: ${currentUrl}`);
+        // Strip anon params and re-navigate
+        const cleanUrl = currentUrl
+          .replace(/[&?]anon=true/gi, '')
+          .replace(/%26anon%3Dtrue/gi, '');
+        this._logger.info(`Re-navigating without anon: ${cleanUrl}`);
+        await this._page.goto(cleanUrl, {
+          waitUntil: 'domcontentloaded',
+          timeout: 30000,
+        });
+        await this._page.waitForTimeout(3000);
+      }
+    }
+
     // Set virtual background if configured (must be done on pre-join screen, before "Join now")
     if (this._options.backgroundImageUrl && this._page && authenticate) {
       try {
@@ -179,11 +216,17 @@ export class BotOrchestrator {
     this._setState('in_meeting');
     this._logger.info(`Bot joined the meeting! (authenticated: ${authenticate})`);
 
+    // Dismiss any post-join permission modals (e.g. "Manage windows on all displays")
+    await this._joinProcedure!.dismissBrowserPermissionModals();
+
     // Initialize audio
     await this._audioProcedure!.initialize();
 
     // Enable and subscribe to captions
     await this._enableCaptions();
+
+    // Enable chat monitoring
+    await this._enableChat();
   }
 
   /**
@@ -200,6 +243,7 @@ export class BotOrchestrator {
       this._joinProcedure = null;
       this._captionsProcedure = null;
       this._audioProcedure = null;
+      this._chatProcedure = null;
     } catch {
       // Ignore cleanup errors
     }
@@ -276,6 +320,11 @@ export class BotOrchestrator {
           this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
           break;
 
+        case 'sendChatMessage':
+          const chatMsg = message as SendChatMessage;
+          this.sendChatMessageToMeeting(chatMsg.text);
+          break;
+
         case 'pong':
           // Heartbeat response
           break;
@@ -384,10 +433,13 @@ export class BotOrchestrator {
     try {
       this._setState('leaving');
 
-      // Unsubscribe from captions
+      // Unsubscribe from captions and chat
       if (this._captionsProcedure) {
         await this._captionsProcedure.unsubscribe();
       }
+      if (this._chatProcedure) {
+        await this._chatProcedure.unsubscribe();
+      }
 
       // Clean up audio
       if (this._audioProcedure) {
@@ -419,6 +471,10 @@ export class BotOrchestrator {
    * Play audio in the meeting.
    */
   async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise<void> {
+    if (this._isShuttingDown) {
+      this._logger.debug('Ignoring playAudio - bot is shutting down');
+      return;
+    }
     if (this._state !== 'in_meeting' || !this._audioProcedure) {
       this._logger.warn('Cannot play audio - not in meeting');
       return;
@@ -467,6 +523,20 @@ export class BotOrchestrator {
       this._options.language
     );
     this._audioProcedure = new AudioProcedure(this._page, this._logger);
+    this._chatProcedure = new ChatProcedure(
+      this._page,
+      this._logger,
+      (entry: ChatMessageEntry) => {
+        // Send chat message to Gateway as a special transcript
+        this._sendChatMessage(entry.speaker, entry.text);
+        this._callbacks.onTranscript({
+          speaker: entry.speaker,
+          text: entry.text,
+          timestamp: entry.timestamp,
+          isFinal: true,
+        });
+      }
+    );
 
     // Inject audio getUserMedia override BEFORE any navigation
     // This ensures Teams gets our controlled audio stream when it calls getUserMedia
@@ -574,6 +644,47 @@ export class BotOrchestrator {
     }
   }
 
+  /**
+   * Enable chat monitoring.
+   */
+  private async _enableChat(): Promise<void> {
+    try {
+      await this._chatProcedure!.enableChatMonitoring();
+      await this._chatProcedure!.subscribeToChatMessages();
+      this._logger.info('Chat monitoring enabled and subscribed');
+    } catch (error) {
+      this._logger.warn('Could not enable chat monitoring:', error);
+      // Continue without chat - not a fatal error
+    }
+  }
+
+  /**
+   * Send a chat message event to the Gateway.
+   */
+  private _sendChatMessage(speaker: string, text: string): void {
+    const message: ChatMessage = {
+      type: 'chatMessage',
+      sessionId: this._sessionId,
+      chat: {
+        speaker,
+        text,
+        timestamp: new Date().toISOString(),
+      },
+    };
+    this._sendToGateway(message);
+  }
+
+  /**
+   * Send a text message to the meeting chat.
+   */
+  async sendChatMessageToMeeting(text: string): Promise<void> {
+    if (this._isShuttingDown || this._state !== 'in_meeting' || !this._chatProcedure) {
+      this._logger.warn('Cannot send chat message - not in meeting');
+      return;
+    }
+    await this._chatProcedure.sendChatMessage(text);
+  }
+
   /**
    * Update the bot state and notify callbacks + Gateway.
    */

src/types/index.ts

@@ -39,8 +39,24 @@ export interface LeaveMeetingMessage {
   sessionId: string;
 }
 
-export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage;
+export interface ChatMessage {
-export type BotToGateway = TranscriptMessage | StatusMessage;
+  type: 'chatMessage';
+  sessionId: string;
+  chat: {
+    speaker: string;
+    text: string;
+    timestamp: string;
+  };
+}
+
+export interface SendChatMessage {
+  type: 'sendChatMessage';
+  sessionId: string;
+  text: string;
+}
+
+export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage | SendChatMessage;
+export type BotToGateway = TranscriptMessage | StatusMessage | ChatMessage;
 
 // Bot State
 export type BotState =