fix: handle Stop transcription (already running), click Show transcript to open panel, fix wildcard container matching to exclude buttons
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG
parent
c36c954ac8
commit
4c0afa3a12
1 changed files with 163 additions and 39 deletions
|
|
@ -155,37 +155,84 @@ export class CaptionsProcedure {
|
|||
this._logger.info(`Clicked "Record and transcribe": ${selector}`);
|
||||
await this._page.waitForTimeout(1500);
|
||||
|
||||
// Log the submenu items
|
||||
await this._logVisibleMenuItems();
|
||||
|
||||
// Click "Start transcription"
|
||||
const transcriptionSelectors = [
|
||||
'[role="menuitem"]:has-text("Start transcription")',
|
||||
'[role="menuitem"]:has-text("Transkription starten")',
|
||||
'[role="menuitem"]:has-text("transcription")',
|
||||
'[role="menuitem"]:has-text("Transkription")',
|
||||
'button:has-text("Start transcription")',
|
||||
'button:has-text("Transkription starten")',
|
||||
'div:has-text("Start transcription")[role="menuitem"]',
|
||||
// Check if transcription is ALREADY running ("Stop transcription" visible)
|
||||
const stopSelectors = [
|
||||
'[data-tid="call-transcript-button"]:has-text("Stop")',
|
||||
'[role="menuitem"]:has-text("Stop transcription")',
|
||||
'[role="menuitem"]:has-text("Transkription beenden")',
|
||||
'[role="menuitem"]:has-text("Transkription stoppen")',
|
||||
];
|
||||
|
||||
for (const transSel of transcriptionSelectors) {
|
||||
let alreadyRunning = false;
|
||||
for (const stopSel of stopSelectors) {
|
||||
try {
|
||||
const transBtn = await this._page.$(transSel);
|
||||
if (transBtn) {
|
||||
await transBtn.click();
|
||||
this._logger.info(`Clicked "Start transcription": ${transSel}`);
|
||||
await this._page.waitForTimeout(2000);
|
||||
return; // language dialog handled by _handleLanguageDialog()
|
||||
const stopBtn = await this._page.$(stopSel);
|
||||
if (stopBtn) {
|
||||
this._logger.info('Transcription already running (found "Stop transcription") — not clicking');
|
||||
alreadyRunning = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
|
||||
this._logger.warn('"Record and transcribe" opened but "Start transcription" not found');
|
||||
await this._page.keyboard.press('Escape');
|
||||
break;
|
||||
if (!alreadyRunning) {
|
||||
// Click "Start transcription" (only explicit "Start" selectors)
|
||||
const startSelectors = [
|
||||
'[data-tid="call-transcript-button"]:has-text("Start")',
|
||||
'[role="menuitem"]:has-text("Start transcription")',
|
||||
'[role="menuitem"]:has-text("Transkription starten")',
|
||||
'button:has-text("Start transcription")',
|
||||
'button:has-text("Transkription starten")',
|
||||
];
|
||||
|
||||
let started = false;
|
||||
for (const startSel of startSelectors) {
|
||||
try {
|
||||
const startBtn = await this._page.$(startSel);
|
||||
if (startBtn) {
|
||||
await startBtn.click();
|
||||
this._logger.info(`Clicked "Start transcription": ${startSel}`);
|
||||
await this._page.waitForTimeout(2000);
|
||||
started = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
|
||||
if (!started) {
|
||||
this._logger.warn('"Record and transcribe" opened but "Start transcription" not found');
|
||||
}
|
||||
}
|
||||
|
||||
// Click "Show transcript" to open the transcript panel for scraping
|
||||
const showTranscriptSelectors = [
|
||||
'[data-tid="transcript-panel-button"]',
|
||||
'[role="menuitem"]:has-text("Show transcript")',
|
||||
'[role="menuitem"]:has-text("Transkript anzeigen")',
|
||||
'[role="menuitem"]:has-text("Transkript")',
|
||||
];
|
||||
|
||||
for (const showSel of showTranscriptSelectors) {
|
||||
try {
|
||||
const showBtn = await this._page.$(showSel);
|
||||
if (showBtn) {
|
||||
await showBtn.click();
|
||||
this._logger.info(`Clicked "Show transcript": ${showSel}`);
|
||||
await this._page.waitForTimeout(2000);
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
|
|
@ -493,15 +540,14 @@ export class CaptionsProcedure {
|
|||
'div[data-tid="closed-caption-renderer-wrapper"]',
|
||||
'div[data-tid="live-captions-renderer"]',
|
||||
'[data-tid="caption-area"]',
|
||||
// Transcript panel (authenticated Teams "Record and transcribe" flow)
|
||||
'[data-tid="transcript-pane"]',
|
||||
'[data-tid="transcript-view"]',
|
||||
'[data-tid*="transcript"]',
|
||||
'[data-tid="transcript-content"]',
|
||||
];
|
||||
|
||||
for (const selector of containerSelectors) {
|
||||
try {
|
||||
await this._page.waitForSelector(selector, { timeout: 10000 });
|
||||
await this._page.waitForSelector(selector, { timeout: 8000 });
|
||||
this._logger.info(`Found captions/transcript container: ${selector}`);
|
||||
return;
|
||||
} catch {
|
||||
|
|
@ -509,17 +555,22 @@ export class CaptionsProcedure {
|
|||
}
|
||||
}
|
||||
|
||||
// Log visible data-tid elements for debugging
|
||||
// Log ALL transcript/caption related data-tid elements for debugging
|
||||
const tids = await this._page.evaluate(() => {
|
||||
const els = document.querySelectorAll('[data-tid]');
|
||||
return Array.from(els)
|
||||
.map(e => e.getAttribute('data-tid') || '')
|
||||
.filter(t => t.includes('caption') || t.includes('transcript') || t.includes('subtitle'))
|
||||
.slice(0, 10);
|
||||
.map(e => ({
|
||||
tid: e.getAttribute('data-tid') || '',
|
||||
tag: e.tagName,
|
||||
h: (e as HTMLElement).offsetHeight,
|
||||
w: (e as HTMLElement).offsetWidth,
|
||||
}))
|
||||
.filter(t =>
|
||||
t.tid.includes('caption') || t.tid.includes('transcript') || t.tid.includes('subtitle'),
|
||||
)
|
||||
.slice(0, 15);
|
||||
});
|
||||
if (tids.length > 0) {
|
||||
this._logger.info(`Related data-tid elements found: ${JSON.stringify(tids)}`);
|
||||
}
|
||||
this._logger.info(`Transcript/caption data-tid elements: ${JSON.stringify(tids)}`);
|
||||
|
||||
this._logger.warn('Could not find captions/transcript container with known selectors');
|
||||
}
|
||||
|
|
@ -940,12 +991,14 @@ export class CaptionsProcedure {
|
|||
'div[data-tid="closed-caption-renderer-wrapper"]',
|
||||
'div[data-tid="live-captions-renderer"]',
|
||||
'[data-tid="caption-area"]',
|
||||
'[data-tid*="transcript"]',
|
||||
'[data-tid="transcript-pane"]',
|
||||
'[data-tid="transcript-view"]',
|
||||
'[data-tid="transcript-content"]',
|
||||
];
|
||||
let containerFound = false;
|
||||
for (const sel of waitSelectors) {
|
||||
try {
|
||||
await this._page.waitForSelector(sel, { timeout: 10000 });
|
||||
await this._page.waitForSelector(sel, { timeout: 8000 });
|
||||
containerFound = true;
|
||||
this._logger.info(`Captions/transcript container found: ${sel}`);
|
||||
break;
|
||||
|
|
@ -953,7 +1006,27 @@ export class CaptionsProcedure {
|
|||
// Try next
|
||||
}
|
||||
}
|
||||
|
||||
if (!containerFound) {
|
||||
// Log all transcript/caption related elements for debugging
|
||||
const transcriptTids = await this._page.evaluate(() => {
|
||||
const els = document.querySelectorAll('[data-tid]');
|
||||
return Array.from(els)
|
||||
.map(e => ({
|
||||
tid: e.getAttribute('data-tid') || '',
|
||||
tag: e.tagName,
|
||||
h: (e as HTMLElement).offsetHeight,
|
||||
w: (e as HTMLElement).offsetWidth,
|
||||
children: e.children?.length || 0,
|
||||
}))
|
||||
.filter(t =>
|
||||
t.tid.includes('caption') || t.tid.includes('transcript') || t.tid.includes('subtitle'),
|
||||
)
|
||||
.slice(0, 20);
|
||||
});
|
||||
this._logger.info(
|
||||
`No exact container match. Transcript/caption elements: ${JSON.stringify(transcriptTids)}`,
|
||||
);
|
||||
this._logger.warn('Captions/transcript container not found, subscribing with body fallback');
|
||||
}
|
||||
|
||||
|
|
@ -1087,12 +1160,22 @@ export class CaptionsProcedure {
|
|||
}
|
||||
}
|
||||
|
||||
// Also try wildcard match (transcript container)
|
||||
// Also try wildcard match (transcript container — exclude buttons/controls)
|
||||
if (!targetNode) {
|
||||
const transcriptEl = document.querySelector('[data-tid*="transcript"]');
|
||||
if (transcriptEl) {
|
||||
targetNode = transcriptEl;
|
||||
targetSelector = `[data-tid="${transcriptEl.getAttribute('data-tid')}"]`;
|
||||
const candidates = document.querySelectorAll('[data-tid*="transcript"]');
|
||||
for (const c of Array.from(candidates)) {
|
||||
const tid = c.getAttribute('data-tid') || '';
|
||||
const tag = c.tagName;
|
||||
const height = (c as HTMLElement).offsetHeight || 0;
|
||||
// Skip buttons, small elements, and control-related elements
|
||||
if (
|
||||
tag === 'BUTTON' || tag === 'SPAN' || tag === 'SVG' ||
|
||||
tid.includes('button') || tid.includes('cancel') || tid.includes('stop') ||
|
||||
height < 100
|
||||
) continue;
|
||||
targetNode = c;
|
||||
targetSelector = `[data-tid="${tid}"]`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1111,7 +1194,15 @@ export class CaptionsProcedure {
|
|||
}
|
||||
|
||||
// ── Fallback: observe document.body ──
|
||||
const allSelectors = [...containerSelectors, '[data-tid*="transcript"]'];
|
||||
const allSelectors = [...containerSelectors];
|
||||
function _isTranscriptContainer(el: Element): boolean {
|
||||
const tid = el.getAttribute('data-tid') || '';
|
||||
if (!tid.includes('transcript')) return false;
|
||||
if (el.tagName === 'BUTTON' || el.tagName === 'SPAN' || el.tagName === 'SVG') return false;
|
||||
if (tid.includes('button') || tid.includes('cancel') || tid.includes('stop')) return false;
|
||||
if ((el as HTMLElement).offsetHeight < 100) return false;
|
||||
return true;
|
||||
}
|
||||
const bodyObserver = new MutationObserver((mutationsList) => {
|
||||
for (const mutation of mutationsList) {
|
||||
if (mutation.type !== 'childList') continue;
|
||||
|
|
@ -1119,7 +1210,7 @@ export class CaptionsProcedure {
|
|||
if (node.nodeType !== Node.ELEMENT_NODE) return;
|
||||
const el = node as HTMLElement;
|
||||
|
||||
// Check if a container just appeared
|
||||
// Check if a known container just appeared
|
||||
for (const sel of allSelectors) {
|
||||
const container = el.matches?.(sel) ? el : el.querySelector?.(sel);
|
||||
if (container) {
|
||||
|
|
@ -1138,6 +1229,39 @@ export class CaptionsProcedure {
|
|||
}
|
||||
}
|
||||
|
||||
// Check if a transcript container appeared dynamically
|
||||
if (_isTranscriptContainer(el)) {
|
||||
bodyObserver.disconnect();
|
||||
const tid = el.getAttribute('data-tid') || '';
|
||||
const targeted = new MutationObserver((muts) => {
|
||||
for (const m of muts) {
|
||||
if (m.type === 'childList') {
|
||||
m.addedNodes.forEach(_handleAddedNode);
|
||||
}
|
||||
}
|
||||
});
|
||||
targeted.observe(el, { childList: true, subtree: true });
|
||||
(window as any).__captionsObserver = targeted;
|
||||
return;
|
||||
}
|
||||
|
||||
// Also check inside the added node for transcript containers
|
||||
const transcriptChild = el.querySelector?.('[data-tid*="transcript"]');
|
||||
if (transcriptChild && _isTranscriptContainer(transcriptChild)) {
|
||||
bodyObserver.disconnect();
|
||||
const tid = transcriptChild.getAttribute('data-tid') || '';
|
||||
const targeted = new MutationObserver((muts) => {
|
||||
for (const m of muts) {
|
||||
if (m.type === 'childList') {
|
||||
m.addedNodes.forEach(_handleAddedNode);
|
||||
}
|
||||
}
|
||||
});
|
||||
targeted.observe(transcriptChild, { childList: true, subtree: true });
|
||||
(window as any).__captionsObserver = targeted;
|
||||
return;
|
||||
}
|
||||
|
||||
_handleAddedNode(node);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue