PowerOn/service-teams-browser-bot
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: add transcript panel detection with wildcard selector, debug logging, and structural extraction

Browse source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG 2026-02-17 22:22:30 +01:00
parent aef99057b2
commit c36c954ac8
1 changed files with 135 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

170
src/bot/captionsProcedure.ts
View file

@ -906,7 +906,7 @@ export class CaptionsProcedure {
this._isSubscribed = true;
this._logger.info('Subscribing to captions...');
// Expose a callback function from Node.js to the browser context
// Expose callback functions from Node.js to the browser context
await this._page.exposeFunction('__onCaptionEvent', (caption: {
speaker: string;
text: string;
@ -915,35 +915,53 @@ export class CaptionsProcedure {
this._handleCaptionEvent(caption);
});
// Verify captions container is present
const containerSelectors = [
// Debug callback: logs transcript DOM structure to help identify selectors
try {
await this._page.exposeFunction('__onCaptionDebug', (info: {
tag: string;
tid: string;
classes: string;
text: string;
children: number;
html: string;
}) => {
this._logger.info(
`TranscriptDOM: <${info.tag} data-tid="${info.tid}"> ` +
`children=${info.children}, text="${info.text}"`,
);
this._logger.debug(`TranscriptDOM html: ${info.html}`);
});
} catch {
// May already be exposed
}
// Wait for a known container
const waitSelectors = [
'div[data-tid="closed-caption-renderer-wrapper"]',
'div[data-tid="live-captions-renderer"]',
'[data-tid="caption-area"]',
'[data-tid*="transcript"]',
];
let containerFound = false;
for (const sel of containerSelectors) {
for (const sel of waitSelectors) {
try {
await this._page.waitForSelector(sel, { timeout: 10000 });
containerFound = true;
this._logger.info(`Captions container found: ${sel}`);
this._logger.info(`Captions/transcript container found: ${sel}`);
break;
} catch {
// Try next
}
}
if (!containerFound) {
this._logger.warn('Captions container not found with known selectors, subscribing anyway');
this._logger.warn('Captions/transcript container not found, subscribing with body fallback');
}
this._logger.info('Setting up MutationObserver for captions...');
this._logger.info('Setting up MutationObserver for captions/transcription...');
// Set up MutationObserver in the browser (Recall.ai approach)
// Falls back to document.body when the specific container is not found
// (authenticated Teams may use different container selectors)
const observerTarget = await this._page.evaluate(() => {
// Helper: extract caption data from an element tree
function _extractCaption(element: HTMLElement): void {
// ── Helper: extract caption data (anonymous/light-meetings captions) ──
function _extractCaption(element: HTMLElement): boolean {
const captionMessage = element.querySelector('.fui-ChatMessageCompact')
|| (element.classList?.contains('fui-ChatMessageCompact') ? element : null);
@ -952,7 +970,6 @@ export class CaptionsProcedure {
const contentElement = captionMessage.querySelector('span[data-tid="closed-caption-text"]');
if (authorElement && contentElement) {
// Watch for real-time updates in the caption text
const textObserver = new MutationObserver(() => {
const speaker = authorElement.textContent?.trim() ?? 'Unknown';
const text = (contentElement as any).innerText?.trim() ?? '';
@ -962,17 +979,96 @@ export class CaptionsProcedure {
timestamp: new Date().toISOString(),
});
});
textObserver.observe(contentElement, {
childList: true,
subtree: true,
characterData: true,
});
return true;
}
}
return false;
}
// ── Helper: extract transcript entry (authenticated Teams transcript panel) ──
function _extractTranscript(element: HTMLElement): boolean {
const text = element.innerText?.trim();
if (!text || text.length < 2) return false;
// Strategy A: data-tid based speaker/text elements
const speakerEl = element.querySelector(
'[data-tid*="speaker"], [data-tid*="author"], [data-tid*="name"], ' +
'[data-tid*="participant"]'
);
const textEl = element.querySelector(
'[data-tid*="text"], [data-tid*="content"], [data-tid*="body"], ' +
'[data-tid*="message"]'
);
if (speakerEl && textEl) {
const speaker = speakerEl.textContent?.trim() || 'Unknown';
const content = (textEl as HTMLElement).innerText?.trim() || '';
if (content) {
(window as any).__onCaptionEvent({
speaker,
text: content,
timestamp: new Date().toISOString(),
});
return true;
}
}
// Try specific caption/transcript container selectors first
// Strategy B: structural — first short text child = speaker, rest = text
const directChildren = Array.from(element.children) as HTMLElement[];
if (directChildren.length >= 2) {
const first = directChildren[0].innerText?.trim() || '';
const rest = directChildren
.slice(1)
.map(c => c.innerText?.trim())
.filter(Boolean)
.join(' ')
.trim();
if (first && first.length < 60 && rest && rest.length > 2) {
(window as any).__onCaptionEvent({
speaker: first,
text: rest,
timestamp: new Date().toISOString(),
});
return true;
}
}
return false;
}
// ── Combined handler for mutation observer ──
function _handleAddedNode(node: Node): void {
if (node.nodeType !== Node.ELEMENT_NODE) return;
const el = node as HTMLElement;
// Skip tiny/empty elements
const text = el.innerText?.trim();
if (!text || text.length < 2) return;
// Try caption extraction first (anonymous UI)
if (_extractCaption(el)) return;
// Try transcript extraction (authenticated UI)
if (_extractTranscript(el)) return;
// Not recognized — log for debugging (only elements with meaningful text)
if (text.length > 3) {
(window as any).__onCaptionDebug?.({
tag: el.tagName,
tid: el.getAttribute('data-tid') || '',
classes: (el.className || '').substring(0, 100),
text: text.substring(0, 200),
children: el.children?.length || 0,
html: el.innerHTML?.substring(0, 500) || '',
});
}
}
// ── Find container ──
const containerSelectors = [
'div[data-tid="closed-caption-renderer-wrapper"]',
'div[data-tid="live-captions-renderer"]',
@ -982,31 +1078,40 @@ export class CaptionsProcedure {
];
let targetNode: Element | null = null;
let targetSelector = '';
for (const sel of containerSelectors) {
targetNode = document.querySelector(sel);
if (targetNode) break;
if (targetNode) {
targetSelector = sel;
break;
}
}
// Also try wildcard match (transcript container)
if (!targetNode) {
const transcriptEl = document.querySelector('[data-tid*="transcript"]');
if (transcriptEl) {
targetNode = transcriptEl;
targetSelector = `[data-tid="${transcriptEl.getAttribute('data-tid')}"]`;
}
}
if (targetNode) {
// Targeted observer on the specific caption container
const tid = targetNode.getAttribute('data-tid') || '';
const observer = new MutationObserver((mutationsList) => {
for (const mutation of mutationsList) {
if (mutation.type === 'childList') {
mutation.addedNodes.forEach((node) => {
if (node.nodeType === Node.ELEMENT_NODE) {
_extractCaption(node as HTMLElement);
}
});
mutation.addedNodes.forEach(_handleAddedNode);
}
}
});
observer.observe(targetNode, { childList: true, subtree: true });
(window as any).__captionsObserver = observer;
return 'container';
return `container:${tid}`;
}
// Fallback: observe document.body and look for caption elements anywhere
// Also watches for the caption container to appear later (e.g. after enabling)
// ── Fallback: observe document.body ──
const allSelectors = [...containerSelectors, '[data-tid*="transcript"]'];
const bodyObserver = new MutationObserver((mutationsList) => {
for (const mutation of mutationsList) {
if (mutation.type !== 'childList') continue;
@ -1014,20 +1119,16 @@ export class CaptionsProcedure {
if (node.nodeType !== Node.ELEMENT_NODE) return;
const el = node as HTMLElement;
// Check if a specific caption container just appeared
for (const sel of containerSelectors) {
// Check if a container just appeared
for (const sel of allSelectors) {
const container = el.matches?.(sel) ? el : el.querySelector?.(sel);
if (container) {
// Switch to targeted observation
bodyObserver.disconnect();
const tid = container.getAttribute('data-tid') || '';
const targeted = new MutationObserver((muts) => {
for (const m of muts) {
if (m.type === 'childList') {
m.addedNodes.forEach((n) => {
if (n.nodeType === Node.ELEMENT_NODE) {
_extractCaption(n as HTMLElement);
}
});
m.addedNodes.forEach(_handleAddedNode);
}
}
});
@ -1037,8 +1138,7 @@ export class CaptionsProcedure {
}
}
// Direct caption element detection (body-level)
_extractCaption(el);
_handleAddedNode(node);
});
}
});