From 8f0b7393088c600a6cb2a1c8641505d4c0007e04 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 17 Feb 2026 23:15:23 +0100 Subject: [PATCH] fix: Language and speech opens SUBMENU with closed-captions-button menuitemcheckbox, not a panel with switches Co-authored-by: Cursor --- src/bot/captionsProcedure.ts | 260 +++++++++++++++-------------------- 1 file changed, 111 insertions(+), 149 deletions(-) diff --git a/src/bot/captionsProcedure.ts b/src/bot/captionsProcedure.ts index b98d34a..ba24430 100644 --- a/src/bot/captionsProcedure.ts +++ b/src/bot/captionsProcedure.ts @@ -106,44 +106,134 @@ export class CaptionsProcedure { * Enable captions or transcription from the "More" menu. * * Strategies in priority order: - * 1. Direct captions button (anonymous / light-meetings UI) - * 2. "Record and transcribe" → "Start transcription" (authenticated Teams 2025+) - * → triggers spoken-language-selection-dialog handled by _handleLanguageDialog() - * → then "Show transcript" to open scraping panel - * 3. "Captions & transcripts" submenu (older authenticated Teams) - * 4. "Language and speech" panel toggle (fallback) - * 5. Generic text / DOM scan fallback + * 1. Direct captions button (anonymous / light-meetings UI — closed-captions-button in main menu) + * 2. "Language and speech" → SUBMENU → "Show live captions" (authenticated Teams 2025+) + * The submenu item is a menuitemcheckbox with id="closed-captions-button" + * aria-checked="false" + "Show live captions" → click to enable + * aria-checked="true" + "Hide live captions" → already on, don't click + * 3. "Record and transcribe" → "Start transcription" (fallback with transcript panel) + * 4. Generic text / DOM scan fallback */ private async _clickEnableCaptions(): Promise { await this._logVisibleMenuItems(); // ── Strategy 1: Direct captions button (anonymous / light-meetings UI) ── - const directSelectors = [ - 'div[id="closed-captions-button"]', - '[data-tid="closed-captions-button"]', - '[data-tid="captions-toggle"]', + // In anonymous mode, closed-captions-button appears directly in the More menu + const directBtn = await this._page.$('#closed-captions-button'); + if (directBtn) { + const ariaChecked = await directBtn.getAttribute('aria-checked'); + const text = await directBtn.evaluate(el => el.textContent?.trim() || ''); + this._logger.info(`Direct captions button found: aria-checked="${ariaChecked}", text="${text}"`); + + if (ariaChecked === 'true' || text.toLowerCase().includes('hide')) { + this._logger.info('Live captions already ON (found "Hide live captions")'); + await this._page.keyboard.press('Escape'); + return; + } + + await directBtn.click(); + this._logger.info('Clicked "Show live captions" (direct button)'); + await this._page.waitForTimeout(1000); + return; + } + + // ── Strategy 2: "Language and speech" → submenu → "Show live captions" ── + // Authenticated Teams 2025+: "Language and speech" has aria-haspopup="menu" + // Opening it reveals a submenu with closed-captions-button as menuitemcheckbox + const langSpeechSelectors = [ + '#LanguageSpeechMenuControl-id', + '[data-tid="LanguageSpeechMenuControl-id"]', + 'div[role="menuitem"]:has-text("Language and speech")', + 'div[role="menuitem"]:has-text("Sprache und Spracheingabe")', ]; - for (const selector of directSelectors) { + for (const selector of langSpeechSelectors) { try { - const button = await this._page.$(selector); - if (button) { - await button.click(); - this._logger.info(`Clicked direct captions button: ${selector}`); - await this._page.waitForTimeout(1000); - return; + const item = await this._page.$(selector); + if (item) { + await item.click(); + this._logger.info(`Clicked "Language and speech": ${selector}`); + await this._page.waitForTimeout(1500); + + // Now look for closed-captions-button in the submenu + const captionsBtn = await this._page.$('#closed-captions-button'); + if (captionsBtn) { + const ariaChecked = await captionsBtn.getAttribute('aria-checked'); + const btnText = await captionsBtn.evaluate(el => el.textContent?.trim() || ''); + this._logger.info(`Submenu captions button: aria-checked="${ariaChecked}", text="${btnText}"`); + + if (ariaChecked === 'true' || btnText.toLowerCase().includes('hide')) { + this._logger.info('Live captions already ON (found "Hide live captions")'); + await this._page.keyboard.press('Escape'); + return; + } + + await captionsBtn.click(); + this._logger.info('Clicked "Show live captions" in Language and speech submenu'); + await this._page.waitForTimeout(1000); + return; + } + + // Fallback: try menuitemcheckbox with captions-related text + const fallbackSelectors = [ + '[role="menuitemcheckbox"]:has-text("captions")', + '[role="menuitemcheckbox"]:has-text("Untertitel")', + '[role="menuitemcheckbox"]:has-text("caption")', + ]; + + for (const fbSel of fallbackSelectors) { + try { + const fbBtn = await this._page.$(fbSel); + if (fbBtn) { + const ariaChecked = await fbBtn.getAttribute('aria-checked'); + const fbText = await fbBtn.evaluate(el => el.textContent?.trim() || ''); + this._logger.info(`Fallback captions button: aria-checked="${ariaChecked}", text="${fbText}"`); + + if (ariaChecked === 'true' || fbText.toLowerCase().includes('hide')) { + this._logger.info('Live captions already ON'); + await this._page.keyboard.press('Escape'); + return; + } + + await fbBtn.click(); + this._logger.info(`Clicked captions button: ${fbSel}`); + await this._page.waitForTimeout(1000); + return; + } + } catch { + // Continue + } + } + + this._logger.warn('Language and speech submenu opened but no captions button found'); + await this._page.keyboard.press('Escape'); + break; } } catch { // Continue } } - // ── Strategy 2: "Record and transcribe" → "Start transcription" + "Show transcript" ── + // ── Strategy 3 (fallback): "Record and transcribe" → "Start transcription" + panel ── + this._logger.info('Live captions not found, trying transcription fallback...'); + + // Clean menu state first + await this._page.keyboard.press('Escape'); + await this._page.waitForTimeout(500); + await this._page.keyboard.press('Escape'); + await this._page.waitForTimeout(500); + + try { + await this._openMoreMenu(); + } catch { + this._logger.warn('Could not re-open More menu for transcription fallback'); + } + const recordMenuSelectors = [ '[data-tid="RecordingMenuControl-id"]', + '#RecordingMenuControl-id', 'div[role="menuitem"]:has-text("Record and transcribe")', 'div[role="menuitem"]:has-text("Aufzeichnen und transkribieren")', - 'div[role="menuitem"]:has-text("Aufnehmen und transkribieren")', ]; for (const selector of recordMenuSelectors) { @@ -179,13 +269,10 @@ export class CaptionsProcedure { } if (!alreadyRunning) { - // Click "Start transcription" (only explicit "Start" selectors) const startSelectors = [ '[data-tid="call-transcript-button"]:has-text("Start")', '[role="menuitem"]:has-text("Start transcription")', '[role="menuitem"]:has-text("Transkription starten")', - 'button:has-text("Start transcription")', - 'button:has-text("Transkription starten")', ]; let started = false; @@ -230,7 +317,6 @@ export class CaptionsProcedure { } } - // Close any remaining menu overlay await this._page.keyboard.press('Escape'); await this._page.waitForTimeout(500); return; @@ -240,124 +326,7 @@ export class CaptionsProcedure { } } - // ── Strategy 3: "Captions & transcripts" submenu (older Teams) ── - const submenuSelectors = [ - '[data-tid="captions-and-transcripts-button"]', - '[role="menuitem"]:has-text("Captions & transcripts")', - '[role="menuitem"]:has-text("Captions and transcripts")', - '[role="menuitem"]:has-text("Untertitel und Transkripte")', - '[role="menuitem"]:has-text("Untertitel")', - ]; - - for (const selector of submenuSelectors) { - try { - const item = await this._page.$(selector); - if (item) { - await item.click(); - this._logger.info(`Clicked captions submenu: ${selector}`); - await this._page.waitForTimeout(1500); - - const enableSelectors = [ - 'button:has-text("Turn on live captions")', - 'button:has-text("Live captions")', - 'button:has-text("Live-Untertitel aktivieren")', - '[role="menuitem"]:has-text("Turn on live captions")', - '[role="menuitem"]:has-text("Live captions")', - '[role="menuitemcheckbox"]:has-text("captions")', - '[data-tid="toggle-captions"]', - ]; - - for (const enableSel of enableSelectors) { - try { - const enableBtn = await this._page.$(enableSel); - if (enableBtn) { - await enableBtn.click(); - this._logger.info(`Clicked enable captions: ${enableSel}`); - await this._page.waitForTimeout(1000); - return; - } - } catch { - // Continue - } - } - - this._logger.info('Opened captions submenu but could not find enable button'); - break; - } - } catch { - // Continue - } - } - - // ── Strategy 4 (fallback): "Language and speech" panel toggle ── - this._logger.info('Trying "Language and speech" as fallback...'); - - // Ensure clean menu state: close any open panels/menus first - await this._page.keyboard.press('Escape'); - await this._page.waitForTimeout(500); - await this._page.keyboard.press('Escape'); - await this._page.waitForTimeout(500); - - try { - await this._openMoreMenu(); - } catch { - this._logger.warn('Could not re-open More menu for Language and speech fallback'); - } - - const langSpeechSelectors = [ - '[data-tid="LanguageSpeechMenuControl-id"]', - 'div[role="menuitem"]:has-text("Language and speech")', - 'div[role="menuitem"]:has-text("Sprache und Spracheingabe")', - ]; - - for (const selector of langSpeechSelectors) { - try { - const item = await this._page.$(selector); - if (item) { - await item.click(); - this._logger.info(`Clicked "Language and speech": ${selector}`); - await this._page.waitForTimeout(2000); - - const toggleResult = await this._page.evaluate(() => { - const switches = document.querySelectorAll( - 'input[role="switch"], [role="switch"], input[type="checkbox"]' - ); - for (const sw of Array.from(switches)) { - const label = (sw.getAttribute('aria-label') || '').toLowerCase(); - const tid = (sw.getAttribute('data-tid') || '').toLowerCase(); - const parentEl = sw.closest('div, label, span') as HTMLElement; - const nearText = (parentEl?.textContent || '').toLowerCase(); - const isCaptions = - label.includes('caption') || label.includes('untertitel') || - tid.includes('caption') || tid.includes('subtitle') || - nearText.includes('live caption') || nearText.includes('liveuntertitel'); - if (isCaptions) { - if (!(sw as HTMLInputElement).checked) { - (sw as HTMLElement).click(); - return { found: true, clicked: true, info: label || tid || nearText.substring(0, 60) }; - } - return { found: true, clicked: false, info: `already on: ${label || tid}` }; - } - } - return { found: false, clicked: false, info: '' }; - }); - - this._logger.info(`Captions toggle result: ${JSON.stringify(toggleResult)}`); - if (toggleResult.found && toggleResult.clicked) { - await this._page.waitForTimeout(1500); - } - await this._page.keyboard.press('Escape'); - if (toggleResult.found) return; - - this._logger.warn('Language panel opened but no captions toggle found'); - break; - } - } catch { - // Continue - } - } - - // ── Strategy 5: DOM scan for anything containing "caption" / "transcri" ── + // ── Strategy 4: DOM scan for anything containing "caption" / "transcri" ── const found = await this._page.evaluate(() => { const keywords = ['caption', 'captions', 'untertitel', 'live caption', 'transcri', 'transkri']; const candidates = document.querySelectorAll( @@ -379,13 +348,6 @@ export class CaptionsProcedure { if (found.clicked) { this._logger.info(`Clicked via DOM scan: "${found.clicked}"`); await this._page.waitForTimeout(1500); - - const turnOnBtn = await this._page.$('button:has-text("Turn on"), [role="menuitem"]:has-text("Turn on")'); - if (turnOnBtn) { - await turnOnBtn.click(); - this._logger.info('Clicked "Turn on" in submenu'); - await this._page.waitForTimeout(1000); - } return; }