From 4c0afa3a12bbd3ac61b8b5f0d33f549baf994b01 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 17 Feb 2026 22:38:57 +0100 Subject: [PATCH] fix: handle Stop transcription (already running), click Show transcript to open panel, fix wildcard container matching to exclude buttons Co-authored-by: Cursor --- src/bot/captionsProcedure.ts | 202 ++++++++++++++++++++++++++++------- 1 file changed, 163 insertions(+), 39 deletions(-) diff --git a/src/bot/captionsProcedure.ts b/src/bot/captionsProcedure.ts index 5fb6404..edb0913 100644 --- a/src/bot/captionsProcedure.ts +++ b/src/bot/captionsProcedure.ts @@ -155,37 +155,84 @@ export class CaptionsProcedure { this._logger.info(`Clicked "Record and transcribe": ${selector}`); await this._page.waitForTimeout(1500); - // Log the submenu items await this._logVisibleMenuItems(); - // Click "Start transcription" - const transcriptionSelectors = [ - '[role="menuitem"]:has-text("Start transcription")', - '[role="menuitem"]:has-text("Transkription starten")', - '[role="menuitem"]:has-text("transcription")', - '[role="menuitem"]:has-text("Transkription")', - 'button:has-text("Start transcription")', - 'button:has-text("Transkription starten")', - 'div:has-text("Start transcription")[role="menuitem"]', + // Check if transcription is ALREADY running ("Stop transcription" visible) + const stopSelectors = [ + '[data-tid="call-transcript-button"]:has-text("Stop")', + '[role="menuitem"]:has-text("Stop transcription")', + '[role="menuitem"]:has-text("Transkription beenden")', + '[role="menuitem"]:has-text("Transkription stoppen")', ]; - for (const transSel of transcriptionSelectors) { + let alreadyRunning = false; + for (const stopSel of stopSelectors) { try { - const transBtn = await this._page.$(transSel); - if (transBtn) { - await transBtn.click(); - this._logger.info(`Clicked "Start transcription": ${transSel}`); - await this._page.waitForTimeout(2000); - return; // language dialog handled by _handleLanguageDialog() + const stopBtn = await this._page.$(stopSel); + if (stopBtn) { + this._logger.info('Transcription already running (found "Stop transcription") — not clicking'); + alreadyRunning = true; + break; } } catch { // Continue } } - this._logger.warn('"Record and transcribe" opened but "Start transcription" not found'); - await this._page.keyboard.press('Escape'); - break; + if (!alreadyRunning) { + // Click "Start transcription" (only explicit "Start" selectors) + const startSelectors = [ + '[data-tid="call-transcript-button"]:has-text("Start")', + '[role="menuitem"]:has-text("Start transcription")', + '[role="menuitem"]:has-text("Transkription starten")', + 'button:has-text("Start transcription")', + 'button:has-text("Transkription starten")', + ]; + + let started = false; + for (const startSel of startSelectors) { + try { + const startBtn = await this._page.$(startSel); + if (startBtn) { + await startBtn.click(); + this._logger.info(`Clicked "Start transcription": ${startSel}`); + await this._page.waitForTimeout(2000); + started = true; + break; + } + } catch { + // Continue + } + } + + if (!started) { + this._logger.warn('"Record and transcribe" opened but "Start transcription" not found'); + } + } + + // Click "Show transcript" to open the transcript panel for scraping + const showTranscriptSelectors = [ + '[data-tid="transcript-panel-button"]', + '[role="menuitem"]:has-text("Show transcript")', + '[role="menuitem"]:has-text("Transkript anzeigen")', + '[role="menuitem"]:has-text("Transkript")', + ]; + + for (const showSel of showTranscriptSelectors) { + try { + const showBtn = await this._page.$(showSel); + if (showBtn) { + await showBtn.click(); + this._logger.info(`Clicked "Show transcript": ${showSel}`); + await this._page.waitForTimeout(2000); + break; + } + } catch { + // Continue + } + } + + return; } } catch { // Continue @@ -493,15 +540,14 @@ export class CaptionsProcedure { 'div[data-tid="closed-caption-renderer-wrapper"]', 'div[data-tid="live-captions-renderer"]', '[data-tid="caption-area"]', - // Transcript panel (authenticated Teams "Record and transcribe" flow) '[data-tid="transcript-pane"]', '[data-tid="transcript-view"]', - '[data-tid*="transcript"]', + '[data-tid="transcript-content"]', ]; for (const selector of containerSelectors) { try { - await this._page.waitForSelector(selector, { timeout: 10000 }); + await this._page.waitForSelector(selector, { timeout: 8000 }); this._logger.info(`Found captions/transcript container: ${selector}`); return; } catch { @@ -509,17 +555,22 @@ export class CaptionsProcedure { } } - // Log visible data-tid elements for debugging + // Log ALL transcript/caption related data-tid elements for debugging const tids = await this._page.evaluate(() => { const els = document.querySelectorAll('[data-tid]'); return Array.from(els) - .map(e => e.getAttribute('data-tid') || '') - .filter(t => t.includes('caption') || t.includes('transcript') || t.includes('subtitle')) - .slice(0, 10); + .map(e => ({ + tid: e.getAttribute('data-tid') || '', + tag: e.tagName, + h: (e as HTMLElement).offsetHeight, + w: (e as HTMLElement).offsetWidth, + })) + .filter(t => + t.tid.includes('caption') || t.tid.includes('transcript') || t.tid.includes('subtitle'), + ) + .slice(0, 15); }); - if (tids.length > 0) { - this._logger.info(`Related data-tid elements found: ${JSON.stringify(tids)}`); - } + this._logger.info(`Transcript/caption data-tid elements: ${JSON.stringify(tids)}`); this._logger.warn('Could not find captions/transcript container with known selectors'); } @@ -940,12 +991,14 @@ export class CaptionsProcedure { 'div[data-tid="closed-caption-renderer-wrapper"]', 'div[data-tid="live-captions-renderer"]', '[data-tid="caption-area"]', - '[data-tid*="transcript"]', + '[data-tid="transcript-pane"]', + '[data-tid="transcript-view"]', + '[data-tid="transcript-content"]', ]; let containerFound = false; for (const sel of waitSelectors) { try { - await this._page.waitForSelector(sel, { timeout: 10000 }); + await this._page.waitForSelector(sel, { timeout: 8000 }); containerFound = true; this._logger.info(`Captions/transcript container found: ${sel}`); break; @@ -953,7 +1006,27 @@ export class CaptionsProcedure { // Try next } } + if (!containerFound) { + // Log all transcript/caption related elements for debugging + const transcriptTids = await this._page.evaluate(() => { + const els = document.querySelectorAll('[data-tid]'); + return Array.from(els) + .map(e => ({ + tid: e.getAttribute('data-tid') || '', + tag: e.tagName, + h: (e as HTMLElement).offsetHeight, + w: (e as HTMLElement).offsetWidth, + children: e.children?.length || 0, + })) + .filter(t => + t.tid.includes('caption') || t.tid.includes('transcript') || t.tid.includes('subtitle'), + ) + .slice(0, 20); + }); + this._logger.info( + `No exact container match. Transcript/caption elements: ${JSON.stringify(transcriptTids)}`, + ); this._logger.warn('Captions/transcript container not found, subscribing with body fallback'); } @@ -1087,12 +1160,22 @@ export class CaptionsProcedure { } } - // Also try wildcard match (transcript container) + // Also try wildcard match (transcript container — exclude buttons/controls) if (!targetNode) { - const transcriptEl = document.querySelector('[data-tid*="transcript"]'); - if (transcriptEl) { - targetNode = transcriptEl; - targetSelector = `[data-tid="${transcriptEl.getAttribute('data-tid')}"]`; + const candidates = document.querySelectorAll('[data-tid*="transcript"]'); + for (const c of Array.from(candidates)) { + const tid = c.getAttribute('data-tid') || ''; + const tag = c.tagName; + const height = (c as HTMLElement).offsetHeight || 0; + // Skip buttons, small elements, and control-related elements + if ( + tag === 'BUTTON' || tag === 'SPAN' || tag === 'SVG' || + tid.includes('button') || tid.includes('cancel') || tid.includes('stop') || + height < 100 + ) continue; + targetNode = c; + targetSelector = `[data-tid="${tid}"]`; + break; } } @@ -1111,7 +1194,15 @@ export class CaptionsProcedure { } // ── Fallback: observe document.body ── - const allSelectors = [...containerSelectors, '[data-tid*="transcript"]']; + const allSelectors = [...containerSelectors]; + function _isTranscriptContainer(el: Element): boolean { + const tid = el.getAttribute('data-tid') || ''; + if (!tid.includes('transcript')) return false; + if (el.tagName === 'BUTTON' || el.tagName === 'SPAN' || el.tagName === 'SVG') return false; + if (tid.includes('button') || tid.includes('cancel') || tid.includes('stop')) return false; + if ((el as HTMLElement).offsetHeight < 100) return false; + return true; + } const bodyObserver = new MutationObserver((mutationsList) => { for (const mutation of mutationsList) { if (mutation.type !== 'childList') continue; @@ -1119,7 +1210,7 @@ export class CaptionsProcedure { if (node.nodeType !== Node.ELEMENT_NODE) return; const el = node as HTMLElement; - // Check if a container just appeared + // Check if a known container just appeared for (const sel of allSelectors) { const container = el.matches?.(sel) ? el : el.querySelector?.(sel); if (container) { @@ -1138,6 +1229,39 @@ export class CaptionsProcedure { } } + // Check if a transcript container appeared dynamically + if (_isTranscriptContainer(el)) { + bodyObserver.disconnect(); + const tid = el.getAttribute('data-tid') || ''; + const targeted = new MutationObserver((muts) => { + for (const m of muts) { + if (m.type === 'childList') { + m.addedNodes.forEach(_handleAddedNode); + } + } + }); + targeted.observe(el, { childList: true, subtree: true }); + (window as any).__captionsObserver = targeted; + return; + } + + // Also check inside the added node for transcript containers + const transcriptChild = el.querySelector?.('[data-tid*="transcript"]'); + if (transcriptChild && _isTranscriptContainer(transcriptChild)) { + bodyObserver.disconnect(); + const tid = transcriptChild.getAttribute('data-tid') || ''; + const targeted = new MutationObserver((muts) => { + for (const m of muts) { + if (m.type === 'childList') { + m.addedNodes.forEach(_handleAddedNode); + } + } + }); + targeted.observe(transcriptChild, { childList: true, subtree: true }); + (window as any).__captionsObserver = targeted; + return; + } + _handleAddedNode(node); }); }