teamsbot auth fixes

src/bot/captionsProcedure.ts

@@ -65,15 +65,17 @@ export class CaptionsProcedure {
    * Works for both anonymous (light-meetings) and authenticated (full Teams) UI.
    */
   private async _openMoreMenu(): Promise<void> {
-    const allSelectors = [
+    // Specific call-controls selectors first, broad ones last.
+    // In the full Teams web app, broad selectors like `aria-label*="More"`
+    // might match sidebar navigation buttons — restrict them to the
+    // call-controls area to avoid navigating away from the meeting.
+    const specificSelectors = [
       'button[id="callingButtons-showMoreBtn"]',
       '[data-tid="callingButtons-showMoreBtn"]',
-      'button[aria-label*="More actions"]',
-      'button[aria-label*="More"]',
       '[data-tid="more-button"]',
     ];
 
-    for (const selector of allSelectors) {
+    for (const selector of specificSelectors) {
       try {
         const button = await this._page.$(selector);
         if (button) {
@@ -87,10 +89,37 @@ export class CaptionsProcedure {
       }
     }
 
+    // Broader selectors scoped to the calling/meeting toolbar only
+    const found = await this._page.evaluate(() => {
+      const btns = Array.from(document.querySelectorAll('button')) as HTMLElement[];
+      for (const btn of btns) {
+        const aria = (btn.getAttribute('aria-label') || '').toLowerCase();
+        if (!aria.includes('more')) continue;
+        if (btn.offsetHeight === 0 || btn.offsetWidth === 0) continue;
+        // Exclude sidebar navigation buttons
+        if (btn.closest('[data-tid="app-bar"], nav[role="navigation"]')) continue;
+        // Prefer buttons near calling controls
+        const nearCalling = btn.closest(
+          '[data-tid="calling-unified-bar"], [data-tid="call-controls"], '
+          + '[data-tid="callingButtons-area"], [id="callingButtons-area"]',
+        );
+        if (nearCalling || aria.includes('more actions') || aria.includes('weitere aktionen')) {
+          btn.click();
+          return aria;
+        }
+      }
+      return null;
+    });
+    if (found) {
+      this._logger.info(`Clicked "More" button via scoped search: "${found}"`);
+      await this._page.waitForTimeout(1000);
+      return;
+    }
+
     // Last resort: wait for the primary selector with a short timeout
     try {
-      await this._page.waitForSelector(allSelectors[0], { timeout: 10000 });
+      await this._page.waitForSelector(specificSelectors[0], { timeout: 10000 });
-      await this._page.click(allSelectors[0]);
+      await this._page.click(specificSelectors[0]);
       this._logger.info('Found "More" button (after wait)');
       await this._page.waitForTimeout(1000);
       return;

src/bot/chatProcedure.ts

@@ -26,15 +26,18 @@ export class ChatProcedure {
   private _scanIntervalId: ReturnType<typeof setInterval> | null = null;
   private _consecutiveOpenFailures: number = 0;
   private static readonly _MAX_OPEN_FAILURES = 5;
+  private _isAuthMode: boolean;
 
   constructor(
     page: Page,
     logger: Logger,
-    onChatMessage: (entry: ChatMessageEntry) => void
+    onChatMessage: (entry: ChatMessageEntry) => void,
+    isAuthMode: boolean = false,
   ) {
     this._page = page;
     this._logger = logger;
     this._onChatMessage = onChatMessage;
+    this._isAuthMode = isAuthMode;
   }
 
   /**
@@ -55,7 +58,7 @@ export class ChatProcedure {
    * authenticated Teams meeting layout.
    */
   async enableChatMonitoring(): Promise<boolean> {
-    this._logger.info('Enabling chat monitoring...');
+    this._logger.info(`Enabling chat monitoring (authMode=${this._isAuthMode})...`);
     await this._dumpChatButtonDiagnostics();
 
     await this._openChatPanel();
@@ -64,11 +67,6 @@ export class ChatProcedure {
     const isOpen = await this._isChatPanelOpen();
     if (isOpen) {
       this._logger.info('Chat panel opened successfully');
-      // Light-meetings ships a "simplified compose" with a collapsed
-      // placeholder + dedicated expand button. The real ckeditor textbox
-      // is rendered but Playwright considers it invisible until expanded.
-      // Expand once now so the periodic scan and send path see the
-      // canonical ckeditor surface.
       await this._ensureComposeExpanded();
     } else {
       this._logger.warn('Chat panel could not be opened - chat send/receive will not work');
@@ -178,14 +176,15 @@ export class ChatProcedure {
    */
   private async _isChatPanelOpen(): Promise<boolean> {
     return this._page.evaluate(() => {
+      // Strategy 1: light-meetings / standard meeting UI — side panel container
       const sidePanel = document.querySelector(
         '[data-tid="calling-right-side-panel"]',
       ) as HTMLElement | null;
-      if (!sidePanel) return false;
+      if (sidePanel) {
         const isVisible = sidePanel.offsetWidth > 0
           && sidePanel.offsetHeight > 0
           && sidePanel.offsetParent !== null;
-      if (!isVisible) return false;
+        if (isVisible) {
           const chatHallmark = sidePanel.querySelector(
             '[data-tid="message-pane-layout"], '
             + '[data-tid="message-pane-body"], '
@@ -194,7 +193,27 @@ export class ChatProcedure {
             + '#chat-pane-list, '
             + '[data-app-name="chats"]',
           );
-      return chatHallmark !== null;
+          if (chatHallmark) return true;
+        }
+      }
+
+      // Strategy 2: full Teams web app (auth mode) — the chat toggle button
+      // has aria-pressed="true" when the in-meeting chat panel is open.
+      // The correct toggle is the one with a keyboard shortcut in the label
+      // (e.g. "Chat (Ctrl+Shift+2)").
+      const allBtns = Array.from(document.querySelectorAll(
+        'button[aria-pressed], [role="button"][aria-pressed]',
+      ));
+      for (const btn of allBtns) {
+        const aria = (btn.getAttribute('aria-label') || '').toLowerCase();
+        const hasChatHint = aria.includes('chat') || aria.includes('unterhalt');
+        const hasShortcutHint = aria.includes('ctrl') || aria.includes('strg');
+        if (hasChatHint && hasShortcutHint && btn.getAttribute('aria-pressed') === 'true') {
+          return true;
+        }
+      }
+
+      return false;
     });
   }
 
@@ -250,6 +269,26 @@ export class ChatProcedure {
             || v.includes('conversation'),
           );
         };
+        const isDangerousNavButton = (el: Element): boolean => {
+          const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+          const tid = (el.getAttribute('data-tid') || '').toLowerCase();
+          const id = (el.id || '').toLowerCase();
+          // Teams sidebar "Chats" navigation — clicking navigates away from meeting
+          if (aria === 'chats' || aria === 'unterhaltungen') return true;
+          // Tab items inside the chat panel (Files, Recap, Whiteboard, etc.)
+          if (tid.startsWith('tab-item-')) return true;
+          // "More chat options" submenu triggers — open dropdowns, not chat panel
+          if (aria.includes('more chat options') || aria.includes('weitere chatoptionen')) return true;
+          // Participant count link
+          if (tid === 'chat-header-participant-count') return true;
+          // Chat join button (call join, not chat toggle)
+          if (tid === 'chat-join-button') return true;
+          // Chat app tab IDs (com.microsoft.chattabs.*)
+          if (id.startsWith('com.microsoft.chattabs.')) return true;
+          // Buttons inside the Teams left app-bar / sidebar navigation
+          if (el.closest('[data-tid="app-bar"], [data-tid="app-bar-search"], nav[role="navigation"]')) return true;
+          return false;
+        };
         const isVisible = (el: HTMLElement): boolean =>
           el.offsetHeight > 0 && el.offsetWidth > 0 && el.offsetParent !== null;
         const keyOf = (el: Element): string =>
@@ -263,7 +302,7 @@ export class ChatProcedure {
           document.querySelectorAll('button, [role="button"], [role="menuitem"]'),
         ) as HTMLElement[];
         const candidates = all
-          .filter((el) => matchesChatHint(el) && isVisible(el))
+          .filter((el) => matchesChatHint(el) && isVisible(el) && !isDangerousNavButton(el))
           .filter((el) => !alreadyTried.includes(keyOf(el)));
 
         if (candidates.length === 0) return { picked: null as null | { key: string; id: string; tid: string; aria: string; toggle: boolean } };
@@ -308,6 +347,31 @@ export class ChatProcedure {
         this._logger.info('Chat panel opened successfully');
         return true;
       }
+
+      // Fallback for auth mode: if we clicked a toggle button and its
+      // aria-pressed is now "true", the panel IS open even if the DOM
+      // detection via calling-right-side-panel fails (different layout
+      // in the full Teams web app).
+      if (click.picked.toggle) {
+        const pressedNow = await this._page.evaluate((btnKey: string) => {
+          const all = Array.from(
+            document.querySelectorAll('button, [role="button"], [role="menuitem"]'),
+          );
+          for (const el of all) {
+            const k = `${el.id || ''}|${el.getAttribute('data-tid') || ''}|${el.getAttribute('aria-label') || ''}`;
+            if (k === btnKey) return el.getAttribute('aria-pressed');
+          }
+          return null;
+        }, click.picked.key);
+
+        if (pressedNow === 'true') {
+          this._logger.info(
+            'Chat panel DOM not detected, but toggle aria-pressed="true" — treating as open (auth mode fallback)',
+          );
+          return true;
+        }
+      }
+
       this._logger.info(
         'Chat button clicked but panel not detected — will try a different candidate next round',
       );
@@ -662,6 +726,11 @@ export class ChatProcedure {
       if (!this._isSubscribed || !this._page) return;
       try {
         if (!(await this._isChatPanelOpen())) {
+          // In auth mode, never try to reopen the chat panel — the toggle
+          // navigates to the Chat section and minimizes the meeting to PiP.
+          if (this._isAuthMode) {
+            this._logger.info('Chat panel closed (auth mode) — skipping reopen to avoid PiP');
+          } else {
             if (this._consecutiveOpenFailures >= ChatProcedure._MAX_OPEN_FAILURES) {
               this._logger.warn(
                 `Chat panel failed to open ${this._consecutiveOpenFailures} consecutive times - ` +
@@ -686,6 +755,7 @@ export class ChatProcedure {
               return;
             }
             await this._page.waitForTimeout(1000);
+          }
         } else {
           this._consecutiveOpenFailures = 0;
         }
@@ -958,8 +1028,6 @@ export class ChatProcedure {
     // overlay used in anonymous / pre-join layouts has a generic
     // [data-tid="ckeditor"] [role="textbox"] in a floating layer that
     // looks like a chat input but does NOT post into the meeting chat).
-    // Surfacing the failure fast lets the periodic scan re-toggle the
-    // panel and the Gateway resend the message.
     const panelOpen = await this._isChatPanelOpen();
     if (!panelOpen) {
       this._logger.warn('Chat panel not open — aborting send so the periodic scan can re-toggle it');
@@ -1168,8 +1236,13 @@ export class ChatProcedure {
           }
 
           await this._page.waitForTimeout(200);
+
+          // Try clicking the send button first; fall back to Enter key.
+          const sent = await this._clickSendButton();
+          if (!sent) {
             await this._page.keyboard.press('Enter');
-          this._logger.info(`Chat message sent (stage=${stageUsed})`);
+          }
+          this._logger.info(`Chat message sent (stage=${stageUsed}, via=${sent ? 'sendBtn' : 'enter'})`);
           return true;
         }
 
@@ -1192,6 +1265,42 @@ export class ChatProcedure {
     return false;
   }
 
+  /**
+   * Click the chat send button if visible. Teams renders a send button
+   * (arrow icon) once text is present in the compose box. Clicking it is
+   * more reliable than pressing Enter, which CKEditor sometimes swallows.
+   */
+  private async _clickSendButton(): Promise<boolean> {
+    try {
+      const sendSelectors = [
+        'button[data-tid="newMessageCommands-send"]',
+        'button[data-tid="send-message-button"]',
+        'button[aria-label="Send" i]',
+        'button[aria-label="Senden" i]',
+        'button[aria-label*="Send message" i]',
+        'button[aria-label*="Nachricht senden" i]',
+        '[data-tid="chat-pane-compose-message-footer"] button[type="submit"]',
+        '[data-tid="message-pane-footer"] button[type="submit"]',
+      ];
+      for (const sel of sendSelectors) {
+        const btn = await this._page.$(sel);
+        if (btn) {
+          const visible = await btn.isVisible().catch(() => false);
+          if (visible) {
+            await btn.click();
+            this._logger.info(`Clicked send button: ${sel}`);
+            await btn.dispose();
+            return true;
+          }
+          await btn.dispose();
+        }
+      }
+    } catch (err) {
+      this._logger.warn(`Send button click failed: ${err}`);
+    }
+    return false;
+  }
+
   /**
    * Dump rich diagnostics about contenteditable / textbox candidates in the
    * page so we can adapt selectors when Teams ships a UI change.

src/bot/orchestrator.ts

@@ -15,7 +15,7 @@ import { AudioCaptureProcedure } from './audioCaptureProcedure';
 import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
 import { AuthProcedure, MfaChallenge } from './authProcedure';
 import { TeamsActionsService } from './teamsActionsService';
-import { BackgroundProcedure } from './backgroundProcedure';
+
 import { isValidMeetingUrl, getMeetingLaunchUrl, resolveLaunchUrl } from './meetingUrlParser';
 
 // Optional: canvas "avatar" video (config.botUseCanvasVideo) replaces the Chromium
@@ -85,6 +85,8 @@ export class BotOrchestrator {
   private _frameNavMediaRebindTimer: ReturnType<typeof setTimeout> | null = null;
   /** Re-apply gUM + video senders for a few seconds after join */
   private _canvasRebindTimer: ReturnType<typeof setInterval> | null = null;
+  /** Whether the bot is running in authenticated mode (full Teams web app) */
+  private _isAuthMode: boolean = false;
 
   constructor(
     sessionId: string,
@@ -218,8 +220,6 @@ export class BotOrchestrator {
     await this._ensureMicOn();
     if (config.botUseCanvasVideo) {
       await this._ensureCameraOn();
-      const bg = new BackgroundProcedure(this._page!, this._logger);
-      void bg.trySelectNoVirtualBackground();
     }
 
     // STEP 2: Enter bot name and click "Join now"
@@ -435,8 +435,6 @@ export class BotOrchestrator {
     await this._ensureMicOn();
     if (config.botUseCanvasVideo) {
       await this._ensureCameraOn();
-      const bg = new BackgroundProcedure(this._page!, this._logger);
-      void bg.trySelectNoVirtualBackground();
     }
 
     // STEP 5: Poll for "Join now" on the pre-join screen
@@ -466,10 +464,22 @@ export class BotOrchestrator {
       await this._ensureCameraOnInMeeting();
       this._startCanvasRebindAfterJoin();
     }
+
+    // Auth mode: wait for meeting UI to stabilize, then captions first,
+    // then chat. The chat toggle in the full Teams web app navigates to
+    // the Chat section and minimizes the meeting to PiP if triggered too
+    // early or while the More menu is still open from captions setup.
+    if (this._isAuthMode) {
+      await this._page!.waitForTimeout(3000);
+      await this._enableTranscriptCapture();
+      await this._page!.waitForTimeout(2000);
+      await this._enableChat();
+    } else {
       await Promise.all([
         this._enableTranscriptCapture(),
         this._enableChat(),
       ]);
+    }
     await this._sendJoinGreeting();
   }
 
@@ -1048,6 +1058,7 @@ export class BotOrchestrator {
    * (`rejectMediaDescriptionsUpdateAsync`). Keep these flag sets separate.
    */
   private async _launchBrowser(authMode: boolean = false): Promise<void> {
+    this._isAuthMode = authMode;
     this._logger.info(`Launching browser (authMode=${authMode})...`);
 
     // When BOT_ANON_USE_AUTH_BROWSER_SETUP is on, the anon path uses the
@@ -1188,6 +1199,7 @@ export class BotOrchestrator {
           isFinal: true,
         });
       },
+      this._isAuthMode,
     );
 
     // DEBUG TOGGLE: skip both wrappers when isolating the Teams anonymous