From 414e2a5e40684d816b9ee6649bf8af2488671450 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 12 May 2026 21:31:25 +0200 Subject: [PATCH] teamsbot auth fixes --- src/bot/captionsProcedure.ts | 41 ++++++- src/bot/chatProcedure.ts | 205 +++++++++++++++++++++++++++-------- src/bot/orchestrator.ts | 30 +++-- 3 files changed, 213 insertions(+), 63 deletions(-) diff --git a/src/bot/captionsProcedure.ts b/src/bot/captionsProcedure.ts index 50fb237..c749254 100644 --- a/src/bot/captionsProcedure.ts +++ b/src/bot/captionsProcedure.ts @@ -65,15 +65,17 @@ export class CaptionsProcedure { * Works for both anonymous (light-meetings) and authenticated (full Teams) UI. */ private async _openMoreMenu(): Promise { - const allSelectors = [ + // Specific call-controls selectors first, broad ones last. + // In the full Teams web app, broad selectors like `aria-label*="More"` + // might match sidebar navigation buttons — restrict them to the + // call-controls area to avoid navigating away from the meeting. + const specificSelectors = [ 'button[id="callingButtons-showMoreBtn"]', '[data-tid="callingButtons-showMoreBtn"]', - 'button[aria-label*="More actions"]', - 'button[aria-label*="More"]', '[data-tid="more-button"]', ]; - for (const selector of allSelectors) { + for (const selector of specificSelectors) { try { const button = await this._page.$(selector); if (button) { @@ -87,10 +89,37 @@ export class CaptionsProcedure { } } + // Broader selectors scoped to the calling/meeting toolbar only + const found = await this._page.evaluate(() => { + const btns = Array.from(document.querySelectorAll('button')) as HTMLElement[]; + for (const btn of btns) { + const aria = (btn.getAttribute('aria-label') || '').toLowerCase(); + if (!aria.includes('more')) continue; + if (btn.offsetHeight === 0 || btn.offsetWidth === 0) continue; + // Exclude sidebar navigation buttons + if (btn.closest('[data-tid="app-bar"], nav[role="navigation"]')) continue; + // Prefer buttons near calling controls + const nearCalling = btn.closest( + '[data-tid="calling-unified-bar"], [data-tid="call-controls"], ' + + '[data-tid="callingButtons-area"], [id="callingButtons-area"]', + ); + if (nearCalling || aria.includes('more actions') || aria.includes('weitere aktionen')) { + btn.click(); + return aria; + } + } + return null; + }); + if (found) { + this._logger.info(`Clicked "More" button via scoped search: "${found}"`); + await this._page.waitForTimeout(1000); + return; + } + // Last resort: wait for the primary selector with a short timeout try { - await this._page.waitForSelector(allSelectors[0], { timeout: 10000 }); - await this._page.click(allSelectors[0]); + await this._page.waitForSelector(specificSelectors[0], { timeout: 10000 }); + await this._page.click(specificSelectors[0]); this._logger.info('Found "More" button (after wait)'); await this._page.waitForTimeout(1000); return; diff --git a/src/bot/chatProcedure.ts b/src/bot/chatProcedure.ts index 6ab15f7..8fa1cd6 100644 --- a/src/bot/chatProcedure.ts +++ b/src/bot/chatProcedure.ts @@ -26,15 +26,18 @@ export class ChatProcedure { private _scanIntervalId: ReturnType | null = null; private _consecutiveOpenFailures: number = 0; private static readonly _MAX_OPEN_FAILURES = 5; + private _isAuthMode: boolean; constructor( page: Page, logger: Logger, - onChatMessage: (entry: ChatMessageEntry) => void + onChatMessage: (entry: ChatMessageEntry) => void, + isAuthMode: boolean = false, ) { this._page = page; this._logger = logger; this._onChatMessage = onChatMessage; + this._isAuthMode = isAuthMode; } /** @@ -55,7 +58,7 @@ export class ChatProcedure { * authenticated Teams meeting layout. */ async enableChatMonitoring(): Promise { - this._logger.info('Enabling chat monitoring...'); + this._logger.info(`Enabling chat monitoring (authMode=${this._isAuthMode})...`); await this._dumpChatButtonDiagnostics(); await this._openChatPanel(); @@ -64,11 +67,6 @@ export class ChatProcedure { const isOpen = await this._isChatPanelOpen(); if (isOpen) { this._logger.info('Chat panel opened successfully'); - // Light-meetings ships a "simplified compose" with a collapsed - // placeholder + dedicated expand button. The real ckeditor textbox - // is rendered but Playwright considers it invisible until expanded. - // Expand once now so the periodic scan and send path see the - // canonical ckeditor surface. await this._ensureComposeExpanded(); } else { this._logger.warn('Chat panel could not be opened - chat send/receive will not work'); @@ -178,23 +176,44 @@ export class ChatProcedure { */ private async _isChatPanelOpen(): Promise { return this._page.evaluate(() => { + // Strategy 1: light-meetings / standard meeting UI — side panel container const sidePanel = document.querySelector( '[data-tid="calling-right-side-panel"]', ) as HTMLElement | null; - if (!sidePanel) return false; - const isVisible = sidePanel.offsetWidth > 0 - && sidePanel.offsetHeight > 0 - && sidePanel.offsetParent !== null; - if (!isVisible) return false; - const chatHallmark = sidePanel.querySelector( - '[data-tid="message-pane-layout"], ' - + '[data-tid="message-pane-body"], ' - + '[data-tid="chat-pane-compose-message-footer"], ' - + '[data-tid="message-pane-footer"], ' - + '#chat-pane-list, ' - + '[data-app-name="chats"]', - ); - return chatHallmark !== null; + if (sidePanel) { + const isVisible = sidePanel.offsetWidth > 0 + && sidePanel.offsetHeight > 0 + && sidePanel.offsetParent !== null; + if (isVisible) { + const chatHallmark = sidePanel.querySelector( + '[data-tid="message-pane-layout"], ' + + '[data-tid="message-pane-body"], ' + + '[data-tid="chat-pane-compose-message-footer"], ' + + '[data-tid="message-pane-footer"], ' + + '#chat-pane-list, ' + + '[data-app-name="chats"]', + ); + if (chatHallmark) return true; + } + } + + // Strategy 2: full Teams web app (auth mode) — the chat toggle button + // has aria-pressed="true" when the in-meeting chat panel is open. + // The correct toggle is the one with a keyboard shortcut in the label + // (e.g. "Chat (Ctrl+Shift+2)"). + const allBtns = Array.from(document.querySelectorAll( + 'button[aria-pressed], [role="button"][aria-pressed]', + )); + for (const btn of allBtns) { + const aria = (btn.getAttribute('aria-label') || '').toLowerCase(); + const hasChatHint = aria.includes('chat') || aria.includes('unterhalt'); + const hasShortcutHint = aria.includes('ctrl') || aria.includes('strg'); + if (hasChatHint && hasShortcutHint && btn.getAttribute('aria-pressed') === 'true') { + return true; + } + } + + return false; }); } @@ -250,6 +269,26 @@ export class ChatProcedure { || v.includes('conversation'), ); }; + const isDangerousNavButton = (el: Element): boolean => { + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const tid = (el.getAttribute('data-tid') || '').toLowerCase(); + const id = (el.id || '').toLowerCase(); + // Teams sidebar "Chats" navigation — clicking navigates away from meeting + if (aria === 'chats' || aria === 'unterhaltungen') return true; + // Tab items inside the chat panel (Files, Recap, Whiteboard, etc.) + if (tid.startsWith('tab-item-')) return true; + // "More chat options" submenu triggers — open dropdowns, not chat panel + if (aria.includes('more chat options') || aria.includes('weitere chatoptionen')) return true; + // Participant count link + if (tid === 'chat-header-participant-count') return true; + // Chat join button (call join, not chat toggle) + if (tid === 'chat-join-button') return true; + // Chat app tab IDs (com.microsoft.chattabs.*) + if (id.startsWith('com.microsoft.chattabs.')) return true; + // Buttons inside the Teams left app-bar / sidebar navigation + if (el.closest('[data-tid="app-bar"], [data-tid="app-bar-search"], nav[role="navigation"]')) return true; + return false; + }; const isVisible = (el: HTMLElement): boolean => el.offsetHeight > 0 && el.offsetWidth > 0 && el.offsetParent !== null; const keyOf = (el: Element): string => @@ -263,7 +302,7 @@ export class ChatProcedure { document.querySelectorAll('button, [role="button"], [role="menuitem"]'), ) as HTMLElement[]; const candidates = all - .filter((el) => matchesChatHint(el) && isVisible(el)) + .filter((el) => matchesChatHint(el) && isVisible(el) && !isDangerousNavButton(el)) .filter((el) => !alreadyTried.includes(keyOf(el))); if (candidates.length === 0) return { picked: null as null | { key: string; id: string; tid: string; aria: string; toggle: boolean } }; @@ -308,6 +347,31 @@ export class ChatProcedure { this._logger.info('Chat panel opened successfully'); return true; } + + // Fallback for auth mode: if we clicked a toggle button and its + // aria-pressed is now "true", the panel IS open even if the DOM + // detection via calling-right-side-panel fails (different layout + // in the full Teams web app). + if (click.picked.toggle) { + const pressedNow = await this._page.evaluate((btnKey: string) => { + const all = Array.from( + document.querySelectorAll('button, [role="button"], [role="menuitem"]'), + ); + for (const el of all) { + const k = `${el.id || ''}|${el.getAttribute('data-tid') || ''}|${el.getAttribute('aria-label') || ''}`; + if (k === btnKey) return el.getAttribute('aria-pressed'); + } + return null; + }, click.picked.key); + + if (pressedNow === 'true') { + this._logger.info( + 'Chat panel DOM not detected, but toggle aria-pressed="true" — treating as open (auth mode fallback)', + ); + return true; + } + } + this._logger.info( 'Chat button clicked but panel not detected — will try a different candidate next round', ); @@ -662,30 +726,36 @@ export class ChatProcedure { if (!this._isSubscribed || !this._page) return; try { if (!(await this._isChatPanelOpen())) { - if (this._consecutiveOpenFailures >= ChatProcedure._MAX_OPEN_FAILURES) { - this._logger.warn( - `Chat panel failed to open ${this._consecutiveOpenFailures} consecutive times - ` + - 'stopping periodic chat scan to avoid log noise', - ); - if (this._scanIntervalId) { - clearInterval(this._scanIntervalId); - this._scanIntervalId = null; - } - return; - } - this._logger.info('Chat panel closed, reopening'); - const opened = await this._openChatPanel(); - if (opened) { - this._consecutiveOpenFailures = 0; - await this._ensureComposeExpanded(); + // In auth mode, never try to reopen the chat panel — the toggle + // navigates to the Chat section and minimizes the meeting to PiP. + if (this._isAuthMode) { + this._logger.info('Chat panel closed (auth mode) — skipping reopen to avoid PiP'); } else { - this._consecutiveOpenFailures++; - this._logger.info( - `Chat reopen failed (${this._consecutiveOpenFailures}/${ChatProcedure._MAX_OPEN_FAILURES} before giving up)`, - ); - return; + if (this._consecutiveOpenFailures >= ChatProcedure._MAX_OPEN_FAILURES) { + this._logger.warn( + `Chat panel failed to open ${this._consecutiveOpenFailures} consecutive times - ` + + 'stopping periodic chat scan to avoid log noise', + ); + if (this._scanIntervalId) { + clearInterval(this._scanIntervalId); + this._scanIntervalId = null; + } + return; + } + this._logger.info('Chat panel closed, reopening'); + const opened = await this._openChatPanel(); + if (opened) { + this._consecutiveOpenFailures = 0; + await this._ensureComposeExpanded(); + } else { + this._consecutiveOpenFailures++; + this._logger.info( + `Chat reopen failed (${this._consecutiveOpenFailures}/${ChatProcedure._MAX_OPEN_FAILURES} before giving up)`, + ); + return; + } + await this._page.waitForTimeout(1000); } - await this._page.waitForTimeout(1000); } else { this._consecutiveOpenFailures = 0; } @@ -958,8 +1028,6 @@ export class ChatProcedure { // overlay used in anonymous / pre-join layouts has a generic // [data-tid="ckeditor"] [role="textbox"] in a floating layer that // looks like a chat input but does NOT post into the meeting chat). - // Surfacing the failure fast lets the periodic scan re-toggle the - // panel and the Gateway resend the message. const panelOpen = await this._isChatPanelOpen(); if (!panelOpen) { this._logger.warn('Chat panel not open — aborting send so the periodic scan can re-toggle it'); @@ -1168,8 +1236,13 @@ export class ChatProcedure { } await this._page.waitForTimeout(200); - await this._page.keyboard.press('Enter'); - this._logger.info(`Chat message sent (stage=${stageUsed})`); + + // Try clicking the send button first; fall back to Enter key. + const sent = await this._clickSendButton(); + if (!sent) { + await this._page.keyboard.press('Enter'); + } + this._logger.info(`Chat message sent (stage=${stageUsed}, via=${sent ? 'sendBtn' : 'enter'})`); return true; } @@ -1192,6 +1265,42 @@ export class ChatProcedure { return false; } + /** + * Click the chat send button if visible. Teams renders a send button + * (arrow icon) once text is present in the compose box. Clicking it is + * more reliable than pressing Enter, which CKEditor sometimes swallows. + */ + private async _clickSendButton(): Promise { + try { + const sendSelectors = [ + 'button[data-tid="newMessageCommands-send"]', + 'button[data-tid="send-message-button"]', + 'button[aria-label="Send" i]', + 'button[aria-label="Senden" i]', + 'button[aria-label*="Send message" i]', + 'button[aria-label*="Nachricht senden" i]', + '[data-tid="chat-pane-compose-message-footer"] button[type="submit"]', + '[data-tid="message-pane-footer"] button[type="submit"]', + ]; + for (const sel of sendSelectors) { + const btn = await this._page.$(sel); + if (btn) { + const visible = await btn.isVisible().catch(() => false); + if (visible) { + await btn.click(); + this._logger.info(`Clicked send button: ${sel}`); + await btn.dispose(); + return true; + } + await btn.dispose(); + } + } + } catch (err) { + this._logger.warn(`Send button click failed: ${err}`); + } + return false; + } + /** * Dump rich diagnostics about contenteditable / textbox candidates in the * page so we can adapt selectors when Teams ships a UI change. diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index 9f0eff9..0c9490e 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -15,7 +15,7 @@ import { AudioCaptureProcedure } from './audioCaptureProcedure'; import { ChatProcedure, ChatMessageEntry } from './chatProcedure'; import { AuthProcedure, MfaChallenge } from './authProcedure'; import { TeamsActionsService } from './teamsActionsService'; -import { BackgroundProcedure } from './backgroundProcedure'; + import { isValidMeetingUrl, getMeetingLaunchUrl, resolveLaunchUrl } from './meetingUrlParser'; // Optional: canvas "avatar" video (config.botUseCanvasVideo) replaces the Chromium @@ -85,6 +85,8 @@ export class BotOrchestrator { private _frameNavMediaRebindTimer: ReturnType | null = null; /** Re-apply gUM + video senders for a few seconds after join */ private _canvasRebindTimer: ReturnType | null = null; + /** Whether the bot is running in authenticated mode (full Teams web app) */ + private _isAuthMode: boolean = false; constructor( sessionId: string, @@ -218,8 +220,6 @@ export class BotOrchestrator { await this._ensureMicOn(); if (config.botUseCanvasVideo) { await this._ensureCameraOn(); - const bg = new BackgroundProcedure(this._page!, this._logger); - void bg.trySelectNoVirtualBackground(); } // STEP 2: Enter bot name and click "Join now" @@ -435,8 +435,6 @@ export class BotOrchestrator { await this._ensureMicOn(); if (config.botUseCanvasVideo) { await this._ensureCameraOn(); - const bg = new BackgroundProcedure(this._page!, this._logger); - void bg.trySelectNoVirtualBackground(); } // STEP 5: Poll for "Join now" on the pre-join screen @@ -466,10 +464,22 @@ export class BotOrchestrator { await this._ensureCameraOnInMeeting(); this._startCanvasRebindAfterJoin(); } - await Promise.all([ - this._enableTranscriptCapture(), - this._enableChat(), - ]); + + // Auth mode: wait for meeting UI to stabilize, then captions first, + // then chat. The chat toggle in the full Teams web app navigates to + // the Chat section and minimizes the meeting to PiP if triggered too + // early or while the More menu is still open from captions setup. + if (this._isAuthMode) { + await this._page!.waitForTimeout(3000); + await this._enableTranscriptCapture(); + await this._page!.waitForTimeout(2000); + await this._enableChat(); + } else { + await Promise.all([ + this._enableTranscriptCapture(), + this._enableChat(), + ]); + } await this._sendJoinGreeting(); } @@ -1048,6 +1058,7 @@ export class BotOrchestrator { * (`rejectMediaDescriptionsUpdateAsync`). Keep these flag sets separate. */ private async _launchBrowser(authMode: boolean = false): Promise { + this._isAuthMode = authMode; this._logger.info(`Launching browser (authMode=${authMode})...`); // When BOT_ANON_USE_AUTH_BROWSER_SETUP is on, the anon path uses the @@ -1188,6 +1199,7 @@ export class BotOrchestrator { isFinal: true, }); }, + this._isAuthMode, ); // DEBUG TOGGLE: skip both wrappers when isolating the Teams anonymous