feat: fix chat in both-mode, add TeamsActionsService for AI commands

Co-authored-by: Cursor <cursoragent@cursor.com>

src/bot/orchestrator.ts

@@ -14,6 +14,7 @@ import { AudioProcedure } from './audioProcedure';
 import { AudioCaptureProcedure } from './audioCaptureProcedure';
 import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
 import { AuthProcedure } from './authProcedure';
+import { TeamsActionsService } from './teamsActionsService';
 import { isValidMeetingUrl } from './meetingUrlParser';
 
 // Camera / fake video injection is disabled for now to focus on stability.
@@ -64,6 +65,7 @@ export class BotOrchestrator {
   private _audioProcedure: AudioProcedure | null = null;
   private _audioCaptureProcedure: AudioCaptureProcedure | null = null;
   private _chatProcedure: ChatProcedure | null = null;
+  private _teamsActions: TeamsActionsService | null = null;
 
   private _state: BotState = 'idle';
   private _isShuttingDown: boolean = false;
@@ -617,7 +619,9 @@ export class BotOrchestrator {
       });
 
       this._gatewayWs.on('message', (data) => {
-        this._handleGatewayMessage(data.toString());
+        this._handleGatewayMessage(data.toString()).catch((err) => {
+          this._logger.error('Unhandled error in gateway message handler:', err);
+        });
       });
 
       this._gatewayWs.on('close', (code, reason) => {
@@ -641,20 +645,27 @@ export class BotOrchestrator {
 
   /**
    * Handle incoming messages from the Gateway.
+   * Async operations are awaited to ensure proper error handling
+   * and serialized execution (e.g. audio + chat don't interfere).
    */
-  private _handleGatewayMessage(data: string): void {
+  private async _handleGatewayMessage(data: string): Promise<void> {
     try {
       const message = JSON.parse(data);
 
       switch (message.type) {
         case 'playAudio':
           const audioMsg = message as PlayAudioMessage;
-          this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
+          await this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
           break;
 
         case 'sendChatMessage':
           const chatMsg = message as SendChatMessage;
-          this.sendChatMessageToMeeting(chatMsg.text);
+          this._logger.info(`Gateway sendChatMessage received: ${chatMsg.text?.substring(0, 60)}...`);
+          try {
+            await this.sendChatMessageToMeeting(chatMsg.text);
+          } catch (chatErr) {
+            this._logger.error(`Failed to send chat message to meeting: ${chatErr}`);
+          }
           break;
 
         case 'stopAudio':
@@ -664,15 +675,18 @@ export class BotOrchestrator {
           }
           break;
 
+        case 'botCommand':
+          await this._handleBotCommand(message.command, message.params || {});
+          break;
+
         case 'pong':
-          // Heartbeat response
           break;
 
         default:
           this._logger.debug('Unknown Gateway message type:', message.type);
       }
     } catch (error) {
-      this._logger.error('Error parsing Gateway message:', error);
+      this._logger.error('Error handling Gateway message:', error);
     }
   }
 
@@ -925,6 +939,7 @@ export class BotOrchestrator {
       this._options.language
     );
     this._audioProcedure = new AudioProcedure(this._page, this._logger);
+    this._teamsActions = new TeamsActionsService(this._page, this._logger);
     this._chatProcedure = new ChatProcedure(
       this._page,
       this._logger,
@@ -1203,6 +1218,36 @@ export class BotOrchestrator {
     await this._chatProcedure.sendChatMessage(text);
   }
 
+  /**
+   * Handle structured commands from the Gateway (issued by AI).
+   */
+  private async _handleBotCommand(command: string, params: Record<string, any>): Promise<void> {
+    if (!this._teamsActions || this._state !== 'in_meeting') {
+      this._logger.warn(`Cannot execute command '${command}' — not in meeting`);
+      return;
+    }
+
+    this._logger.info(`Executing bot command: ${command} params=${JSON.stringify(params)}`);
+
+    try {
+      switch (command) {
+        case 'toggleTranscript':
+          await this._teamsActions.toggleTranscript(params.enable !== false);
+          break;
+        case 'toggleMic':
+          await this._teamsActions.toggleMic(params.enable !== false);
+          break;
+        case 'toggleCamera':
+          await this._teamsActions.toggleCamera(params.enable !== false);
+          break;
+        default:
+          this._logger.warn(`Unknown bot command: ${command}`);
+      }
+    } catch (err) {
+      this._logger.error(`Bot command '${command}' failed: ${err}`);
+    }
+  }
+
   /**
    * Update the bot state and notify callbacks + Gateway.
    */

src/bot/teamsActionsService.ts

@@ -0,0 +1,287 @@
+import { Page } from 'playwright';
+import { Logger } from 'winston';
+
+/**
+ * Service center for all Teams meeting UI actions.
+ *
+ * Each method is idempotent and state-aware: it checks the current state
+ * before acting (e.g. won't toggle transcript OFF if it's already ON).
+ * This makes calls safe to repeat and resilient to UI flicker.
+ */
+export class TeamsActionsService {
+  private _page: Page;
+  private _logger: Logger;
+
+  constructor(page: Page, logger: Logger) {
+    this._page = page;
+    this._logger = logger;
+  }
+
+  // =========================================================================
+  // Transcript / Captions
+  // =========================================================================
+
+  async toggleTranscript(enable: boolean): Promise<boolean> {
+    this._logger.info(`TeamsActions: toggleTranscript enable=${enable}`);
+
+    try {
+      // Open "More" menu
+      if (!(await this._openMoreMenu())) return false;
+
+      // Try "Language and speech" → submenu → "Show live captions"
+      const captionsBtn = await this._findCaptionsButton();
+      if (captionsBtn) {
+        const ariaChecked = await captionsBtn.getAttribute('aria-checked');
+        const text = await captionsBtn.evaluate((el: HTMLElement) => el.textContent?.trim() || '');
+        const isCurrentlyOn = ariaChecked === 'true' || text.toLowerCase().includes('hide');
+
+        if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) {
+          this._logger.info(`TeamsActions: Transcript already ${enable ? 'ON' : 'OFF'} — no action needed`);
+          await this._page.keyboard.press('Escape');
+          return true;
+        }
+
+        await captionsBtn.click();
+        this._logger.info(`TeamsActions: Transcript toggled ${enable ? 'ON' : 'OFF'}`);
+        await this._page.waitForTimeout(1000);
+        return true;
+      }
+
+      await this._page.keyboard.press('Escape');
+      this._logger.warn('TeamsActions: Could not find captions button');
+      return false;
+    } catch (err) {
+      this._logger.warn(`TeamsActions: toggleTranscript error: ${err}`);
+      await this._page.keyboard.press('Escape').catch(() => {});
+      return false;
+    }
+  }
+
+  // =========================================================================
+  // Chat
+  // =========================================================================
+
+  async sendChatMessage(text: string): Promise<boolean> {
+    this._logger.info(`TeamsActions: sendChatMessage: ${text.substring(0, 60)}...`);
+
+    try {
+      await this._ensureChatPanelOpen();
+
+      const inputSelectors = [
+        '[data-tid="ckeditor-replyConversation"]',
+        'div[role="textbox"][data-tid*="chat"]',
+        'div[role="textbox"][aria-label*="message" i]',
+        'div[role="textbox"][aria-label*="Nachricht" i]',
+        'div[contenteditable="true"][data-tid*="message"]',
+        'div[contenteditable="true"][aria-label*="message" i]',
+      ];
+
+      let input: any = null;
+      for (const selector of inputSelectors) {
+        input = await this._page.$(selector);
+        if (input) break;
+      }
+
+      if (!input) {
+        this._logger.warn('TeamsActions: Could not find chat input field');
+        return false;
+      }
+
+      await input.click();
+      await this._page.waitForTimeout(200);
+      await this._page.keyboard.type(text, { delay: 10 });
+      await this._page.waitForTimeout(200);
+      await this._page.keyboard.press('Enter');
+
+      this._logger.info('TeamsActions: Chat message sent');
+      return true;
+    } catch (err) {
+      this._logger.error(`TeamsActions: sendChatMessage error: ${err}`);
+      return false;
+    }
+  }
+
+  // =========================================================================
+  // Microphone
+  // =========================================================================
+
+  async toggleMic(enable: boolean): Promise<boolean> {
+    this._logger.info(`TeamsActions: toggleMic enable=${enable}`);
+
+    try {
+      const micBtn = await this._page.$('button#microphone-button');
+      if (!micBtn) {
+        const fallbacks = [
+          'button[data-inp="microphone-button"]',
+          'button[aria-label*="microphone" i]',
+          'button[aria-label*="Mikrofon" i]',
+        ];
+        for (const sel of fallbacks) {
+          const btn = await this._page.$(sel);
+          if (btn) return this._toggleMediaButton(btn, enable, 'mic');
+        }
+        this._logger.warn('TeamsActions: Microphone button not found');
+        return false;
+      }
+
+      return this._toggleMediaButton(micBtn, enable, 'mic');
+    } catch (err) {
+      this._logger.warn(`TeamsActions: toggleMic error: ${err}`);
+      return false;
+    }
+  }
+
+  // =========================================================================
+  // Camera
+  // =========================================================================
+
+  async toggleCamera(enable: boolean): Promise<boolean> {
+    this._logger.info(`TeamsActions: toggleCamera enable=${enable}`);
+
+    try {
+      const camBtn = await this._page.$('button#video-button');
+      if (!camBtn) {
+        const fallbacks = [
+          'button[data-inp="video-button"]',
+          'button[aria-label*="camera" i]',
+          'button[aria-label*="Camera" i]',
+          'button[aria-label*="Video" i]',
+        ];
+        for (const sel of fallbacks) {
+          const btn = await this._page.$(sel);
+          if (btn) return this._toggleMediaButton(btn, enable, 'camera');
+        }
+        this._logger.warn('TeamsActions: Camera button not found');
+        return false;
+      }
+
+      return this._toggleMediaButton(camBtn, enable, 'camera');
+    } catch (err) {
+      this._logger.warn(`TeamsActions: toggleCamera error: ${err}`);
+      return false;
+    }
+  }
+
+  // =========================================================================
+  // Internal Helpers
+  // =========================================================================
+
+  private async _toggleMediaButton(
+    btn: any,
+    enable: boolean,
+    name: string,
+  ): Promise<boolean> {
+    const state = await btn.evaluate((el: HTMLElement) => ({
+      dataState: el.getAttribute('data-state') || '',
+      ariaLabel: el.getAttribute('aria-label') || '',
+    }));
+
+    const isOff =
+      state.dataState.includes('off') ||
+      state.ariaLabel.toLowerCase().includes('turn on') ||
+      state.ariaLabel.toLowerCase().includes('einschalten');
+
+    const isCurrentlyOn = !isOff;
+
+    if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) {
+      this._logger.info(`TeamsActions: ${name} already ${enable ? 'ON' : 'OFF'}`);
+      return true;
+    }
+
+    await btn.click();
+    this._logger.info(`TeamsActions: ${name} toggled ${enable ? 'ON' : 'OFF'}`);
+    await this._page.waitForTimeout(500);
+    return true;
+  }
+
+  private async _openMoreMenu(): Promise<boolean> {
+    const selectors = [
+      'button[id="callingButtons-showMoreBtn"]',
+      '[data-tid="callingButtons-showMoreBtn"]',
+      'button[aria-label*="More actions"]',
+      'button[aria-label*="More"]',
+      '[data-tid="more-button"]',
+    ];
+
+    for (const selector of selectors) {
+      try {
+        const button = await this._page.$(selector);
+        if (button) {
+          await button.click();
+          await this._page.waitForTimeout(1000);
+          return true;
+        }
+      } catch {
+        // Continue
+      }
+    }
+    this._logger.warn('TeamsActions: Could not find More actions menu');
+    return false;
+  }
+
+  private async _findCaptionsButton(): Promise<any> {
+    // Direct button
+    let btn = await this._page.$('#closed-captions-button');
+    if (btn) return btn;
+
+    // "Language and speech" → submenu
+    const langSpeechSelectors = [
+      '#LanguageSpeechMenuControl-id',
+      '[data-tid="LanguageSpeechMenuControl-id"]',
+      'div[role="menuitem"]:has-text("Language and speech")',
+      'div[role="menuitem"]:has-text("Sprache und Spracheingabe")',
+    ];
+
+    for (const sel of langSpeechSelectors) {
+      try {
+        const item = await this._page.$(sel);
+        if (item) {
+          await item.click();
+          await this._page.waitForTimeout(1500);
+          btn = await this._page.$('#closed-captions-button');
+          if (btn) return btn;
+          break;
+        }
+      } catch {
+        // Continue
+      }
+    }
+    return null;
+  }
+
+  private async _ensureChatPanelOpen(): Promise<void> {
+    const alreadyOpen = await this._page.evaluate(() => {
+      const chatBtn = document.querySelector(
+        'button[id="chat-button"], button[data-tid="chat-button"]',
+      ) as HTMLElement | null;
+      if (chatBtn && chatBtn.getAttribute('aria-pressed') === 'true') return true;
+      const chatInput = document.querySelector(
+        '[data-tid="ckeditor-replyConversation"], div[role="textbox"][data-tid*="chat"]',
+      ) as HTMLElement | null;
+      return chatInput ? chatInput.offsetHeight > 0 : false;
+    });
+
+    if (alreadyOpen) return;
+
+    const chatButtonSelectors = [
+      'button[id="chat-button"]',
+      'button[data-tid="chat-button"]',
+      'button[aria-label*="Chat" i]',
+      '#chat-button',
+    ];
+
+    for (const selector of chatButtonSelectors) {
+      try {
+        const button = await this._page.$(selector);
+        if (button) {
+          await button.click();
+          this._logger.info(`TeamsActions: Opened chat panel: ${selector}`);
+          await this._page.waitForTimeout(1000);
+          return;
+        }
+      } catch {
+        // Continue
+      }
+    }
+  }
+}