import { Page } from 'playwright'; import { Logger } from 'winston'; export interface ChatMessageEntry { speaker: string; text: string; timestamp: Date; source: 'chat'; isHistory: boolean; } /** * Handles reading and writing chat messages in a Teams meeting. * * Teams meeting chat is a separate panel that can be opened via the chat button. * Messages are rendered as DOM elements that we observe via MutationObserver. */ export class ChatProcedure { private _page: Page; private _logger: Logger; private _onChatMessage: (entry: ChatMessageEntry) => void; private _isSubscribed: boolean = false; private _lastMessageText: string = ''; private _botJoinedAtMs: number = 0; private _recentMessageKeyTimestamps: Map = new Map(); private _scanIntervalId: ReturnType | null = null; constructor( page: Page, logger: Logger, onChatMessage: (entry: ChatMessageEntry) => void ) { this._page = page; this._logger = logger; this._onChatMessage = onChatMessage; } /** * Set the bot's join timestamp so we can classify messages as history vs live. */ setBotJoinedAt(ms: number): void { this._botJoinedAtMs = ms; this._logger.info(`Chat history cutoff set: messages before ${new Date(ms).toISOString()} are history`); } /** * Open the chat panel and start monitoring messages. */ async enableChatMonitoring(): Promise { this._logger.info('Enabling chat monitoring...'); // Open chat panel await this._openChatPanel(); // Wait for chat to load await this._page.waitForTimeout(2000); this._logger.info('Chat panel opened'); } /** * Check if the chat panel is currently visible by probing for known * UI elements (chat input, message list, or aria-pressed toggle). */ private async _isChatPanelOpen(): Promise { return this._page.evaluate(() => { // 1. Chat button toggle state const chatBtn = document.querySelector('button[id="chat-button"], button[data-tid="chat-button"]') as HTMLElement | null; if (chatBtn?.getAttribute('aria-pressed') === 'true') return true; // 2. Chat input visible (multiple Teams UI variants) const inputSelectors = [ '[data-tid="ckeditor-replyConversation"]', 'div[role="textbox"][data-tid*="chat"]', 'div[role="textbox"][aria-label*="message" i]', 'div[role="textbox"][aria-label*="Nachricht" i]', '[contenteditable="true"][aria-label*="message" i]', '[contenteditable="true"][aria-label*="Nachricht" i]', '[placeholder*="message" i]', '[placeholder*="Nachricht" i]', 'div[aria-label="Type a message"]', ]; for (const sel of inputSelectors) { const el = document.querySelector(sel) as HTMLElement | null; if (el && el.offsetHeight > 0) return true; } // 3. Message list container visible const listSelectors = [ '[data-tid="message-pane-list"]', '[data-tid="chat-pane-list"]', '[data-tid="chat-pane"]', '.ts-message-list-container', '[role="log"]', ]; for (const sel of listSelectors) { const el = document.querySelector(sel) as HTMLElement | null; if (el && el.offsetHeight > 50) return true; } // 4. "Meeting chat" heading visible const headings = document.querySelectorAll('h2, h3, [role="heading"]'); for (const h of Array.from(headings)) { const txt = (h as HTMLElement).innerText?.toLowerCase() || ''; if (txt.includes('meeting chat') || txt.includes('besprechungschat')) return true; } return false; }); } /** * Open the chat panel by clicking the chat button. * In authenticated Teams, the chat panel may already be open (meeting loads * from a chat thread). Clicking again would TOGGLE it closed. */ private async _openChatPanel(): Promise { if (await this._isChatPanelOpen()) { this._logger.info('Chat panel already open - skipping toggle'); return; } const chatButtonSelectors = [ 'button[id="chat-button"]', 'button[data-tid="chat-button"]', 'button[aria-label*="Chat" i]', 'button[aria-label*="chat" i]', '#chat-button', ]; for (const selector of chatButtonSelectors) { try { const button = await this._page.$(selector); if (button) { await button.click(); this._logger.info(`Opened chat panel: ${selector}`); return; } } catch { // Continue } } this._logger.warn('Could not find chat button - chat monitoring will not work'); } /** * Subscribe to chat messages using MutationObserver. */ async subscribeToChatMessages(): Promise { if (this._isSubscribed) { this._logger.warn('Already subscribed to chat messages'); return; } this._isSubscribed = true; this._logger.info('Subscribing to chat messages...'); // Expose callbacks from Node.js to browser try { await this._page.exposeFunction('__onChatMessageEvent', (msg: { speaker: string; text: string; timestamp: string; teamsTimestamp?: string; messageKey?: string; }) => { this._handleChatMessage(msg); }); } catch { // Function may already be exposed from a previous subscription } try { await this._page.exposeFunction('__onChatDebug', (info: { tag: string; tid: string; text: string; children: number; html: string; }) => { this._logger.debug(`ChatDOM: <${info.tag} data-tid="${info.tid}"> children=${info.children}, text="${info.text.substring(0, 120)}"`); }); } catch { // Already exposed } // Find chat container and set up observer const chatObserverTarget = await this._page.evaluate(() => { // Noise patterns: system messages, not actual chat const noisePatterns = [ 'meeting ended', 'meeting started', 'was invited', 'left the chat', 'joined the meeting', 'left the meeting', 'doesn\'t have a teams account', 'verify their identity', 'new notification', 'last read', ]; function _isNoise(text: string): boolean { const lower = text.toLowerCase(); return noisePatterns.some(p => lower.includes(p)); } function _extractTeamsTimestamp(el: HTMLElement): string | undefined { const timeSelectors = [ 'time[datetime]', '[data-tid="message-timestamp"] time', '[data-tid*="timestamp"] time', '[data-tid="message-timestamp"]', ]; for (const sel of timeSelectors) { const timeEl = el.querySelector(sel) || el.closest?.('[data-tid*="message"]')?.querySelector(sel); if (timeEl) { const dt = timeEl.getAttribute('datetime'); if (dt) return dt; const title = timeEl.getAttribute('title'); if (title) return title; } } return undefined; } function _extractChatMessage(el: HTMLElement): boolean { const messageSelectors = [ '[data-tid="chat-message"]', '.fui-ChatMessage', '[data-tid*="message-body"]', '[data-tid*="chat-pane-message"]', ]; let messageEl: HTMLElement | null = null; for (const sel of messageSelectors) { messageEl = el.matches?.(sel) ? el : el.querySelector(sel); if (messageEl) break; } if (messageEl) { const authorSelectors = [ '[data-tid="message-author"]', '[data-tid="message-author-name"]', '.fui-ChatMessage__author', '[data-tid*="author"]', ]; let author = 'Unknown'; for (const sel of authorSelectors) { const authorEl = messageEl.querySelector(sel) || el.querySelector(sel); if (authorEl?.textContent) { author = authorEl.textContent.trim(); break; } } const bodySelectors = [ '[data-tid="message-body"]', '.fui-ChatMessage__body', '[data-tid="chat-message-text"]', '[data-tid*="message-body"]', ]; let text = ''; for (const sel of bodySelectors) { const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel); if (bodyEl) { text = (bodyEl as HTMLElement).innerText?.trim() || ''; break; } } if (text && text.length > 0) { const teamsTs = _extractTeamsTimestamp(messageEl) || _extractTeamsTimestamp(el); (window as any).__onChatMessageEvent({ speaker: author, text, timestamp: teamsTs || new Date().toISOString(), teamsTimestamp: teamsTs, messageKey: `${author}::${text}`, }); return true; } } // Strategy 2: Structural fallback for authenticated Teams chat // Chat messages typically have: author element + body element as children const fullText = el.innerText?.trim() || ''; if (!fullText || fullText.length < 2 || _isNoise(fullText)) return false; // Skip typing indicators, system messages const tid = el.getAttribute('data-tid') || ''; if (tid === 'typing-indicator') return false; // Look for elements that look like user messages (have author-like + body-like children) const children = Array.from(el.children) as HTMLElement[]; if (children.length >= 2) { // Find an element that looks like a name (short text, no data-tid with "body") for (let i = 0; i < children.length - 1; i++) { const candidateName = children[i].innerText?.trim() || ''; const candidateBody = children.slice(i + 1).map(c => c.innerText?.trim()).filter(Boolean).join(' ').trim(); if ( candidateName.length > 1 && candidateName.length < 60 && candidateBody.length > 1 && !_isNoise(candidateBody) && !candidateName.includes('meeting') && !candidateName.includes('Meeting') ) { // Check if this looks like a time-stamped message (not just any two children) const hasTid = children[i].getAttribute('data-tid') || ''; if (hasTid.includes('author') || hasTid.includes('name') || hasTid.includes('sender')) { const teamsTs = _extractTeamsTimestamp(el); (window as any).__onChatMessageEvent({ speaker: candidateName, text: candidateBody, timestamp: teamsTs || new Date().toISOString(), teamsTimestamp: teamsTs, messageKey: `${candidateName}::${candidateBody}`, }); return true; } } } } return false; } // Teams chat containers - try multiple selectors const chatContainerSelectors = [ '[data-tid="message-pane-list"]', '[data-tid="chat-pane"]', '[data-tid="chat-pane-list"]', '.ts-message-list-container', '[role="log"]', ]; let chatContainer: Element | null = null; let matchedSelector = ''; for (const sel of chatContainerSelectors) { chatContainer = document.querySelector(sel); if (chatContainer) { matchedSelector = sel; break; } } if (!chatContainer) { const candidates = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"]'); for (const c of Array.from(candidates)) { const cTid = c.getAttribute('data-tid') || ''; // Prefer larger containers, not buttons or small elements if ((c as HTMLElement).offsetHeight > 50 && c.tagName !== 'BUTTON') { chatContainer = c; matchedSelector = `[data-tid="${cTid}"]`; break; } } } // Use found container or fall back to document.body const target = chatContainer || document.body; const observer = new MutationObserver((mutations) => { for (const mutation of mutations) { if (mutation.type === 'childList') { mutation.addedNodes.forEach((node) => { if (node.nodeType !== Node.ELEMENT_NODE) return; const el = node as HTMLElement; const text = el.innerText?.trim() || ''; if (!text || text.length < 2) return; if (!_extractChatMessage(el)) { // Log unrecognized elements for debugging (skip noise) if (!_isNoise(text) && text.length > 3) { const tid = el.getAttribute('data-tid') || ''; if (tid !== 'typing-indicator') { (window as any).__onChatDebug?.({ tag: el.tagName, tid, text: text.substring(0, 200), children: el.children?.length || 0, html: el.innerHTML?.substring(0, 500) || '', }); } } } }); } } }); observer.observe(target, { childList: true, subtree: true }); (window as any).__chatObserver = observer; return chatContainer ? `container:${matchedSelector}` : 'body-fallback'; }); this._logger.info(`Chat MutationObserver set up (target: ${chatObserverTarget})`); this._startPeriodicChatScan(); } /** * Periodic scan as fallback: participant messages may load async and miss * the MutationObserver (addedNode with empty placeholder, then content update). * Also reopens the chat panel if it was closed (e.g. by user or Teams). */ private _startPeriodicChatScan(): void { if (this._scanIntervalId) return; const intervalMs = 5000; this._scanIntervalId = setInterval(async () => { if (!this._isSubscribed || !this._page) return; try { if (!(await this._isChatPanelOpen())) { this._logger.info('Chat panel closed, reopening'); await this._openChatPanel(); await this._page.waitForTimeout(1000); } const knownKeys = Array.from(this._recentMessageKeyTimestamps.keys()); const scanResult = await this._page.evaluate((knownKeysArr: string[]) => { const known = new Set(knownKeysArr); const noisePatterns = [ 'meeting ended', 'meeting started', 'was invited', 'left the chat', 'joined the meeting', 'left the meeting', 'doesn\'t have a teams account', 'verify their identity', ]; function isNoise(t: string) { const l = t.toLowerCase(); return noisePatterns.some(p => l.includes(p)); } const results: Array<{ speaker: string; text: string; timestamp: string; teamsTimestamp?: string; messageKey: string }> = []; const seenThisScan = new Set(); // Strategy 1: known selectors const containerSelectors = [ '[data-tid="message-pane-list"]', '[data-tid="chat-pane-list"]', '[data-tid="chat-pane"]', '[role="log"]', '.ts-message-list-container', ]; let container: Element | null = null; for (const sel of containerSelectors) { container = document.querySelector(sel); if (container) break; } const messageSelectors = [ '[data-tid="chat-message"]', '.fui-ChatMessage', '[data-tid*="chat-pane-message"]', '[data-tid*="message-body"]', ]; const target = container || document.body; const candidates = target.querySelectorAll(messageSelectors.join(', ')); for (const el of Array.from(candidates) as HTMLElement[]) { const messageEl = el.closest?.('[data-tid="chat-message"], .fui-ChatMessage') || el; let author = 'Unknown'; const authorSels = [ '[data-tid="message-author"]', '[data-tid="message-author-name"]', '.fui-ChatMessage__author', '[data-tid*="author"]', ]; for (const sel of authorSels) { const authorEl = messageEl.querySelector(sel) || el.querySelector(sel); if (authorEl?.textContent) { author = authorEl.textContent.trim(); break; } } const bodySels = [ '[data-tid="message-body"]', '.fui-ChatMessage__body', '[data-tid="chat-message-text"]', '[data-tid*="message-body"]', ]; let text = ''; for (const sel of bodySels) { const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel); if (bodyEl) { text = (bodyEl as HTMLElement).innerText?.trim() || ''; break; } } if (!text || text.length < 2 || isNoise(text)) continue; const key = `${author}::${text}`; if (known.has(key) || seenThisScan.has(key)) continue; seenThisScan.add(key); const timeEl = messageEl.querySelector('time[datetime], [data-tid*="timestamp"] time'); const ts = timeEl?.getAttribute?.('datetime') || new Date().toISOString(); results.push({ speaker: author, text, timestamp: ts, teamsTimestamp: ts, messageKey: key }); } // Diagnostics (once per 20s, controlled by caller) let diag: string | undefined; if ((window as any).__chatScanDiagCounter === undefined) (window as any).__chatScanDiagCounter = 0; (window as any).__chatScanDiagCounter++; if ((window as any).__chatScanDiagCounter % 4 === 1) { const info: string[] = []; info.push(`container=${container?.tagName || 'body'}[${container?.getAttribute?.('data-tid') || ''}]`); info.push(`candidates=${candidates.length}`); // Dump all chat-ish elements for debugging const allChat = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"], [role="log"], .fui-Chat'); const tags: string[] = []; for (const c of Array.from(allChat).slice(0, 15)) { const tid = c.getAttribute('data-tid') || ''; const cls = c.className?.toString?.()?.substring(0, 40) || ''; const h = (c as HTMLElement).offsetHeight || 0; tags.push(`<${c.tagName} tid="${tid}" cls="${cls}" h=${h}>`); } info.push(`domElements=[${tags.join(', ')}]`); diag = info.join(' | '); } return { messages: results, diag }; }, knownKeys); if (scanResult.diag) { this._logger.info(`[ChatScan] ${scanResult.diag}`); } for (const msg of scanResult.messages) { this._handleChatMessage(msg); } } catch { // Page may be closed } }, intervalMs); this._logger.info(`Chat periodic scan started (interval: ${intervalMs}ms)`); } /** * Handle a chat message event from the browser. */ private _handleChatMessage(msg: { speaker: string; text: string; timestamp: string; teamsTimestamp?: string; messageKey?: string; }): void { if (!this._isSubscribed || !msg.text) return; const nowMs = Date.now(); // Dedup if (msg.text === this._lastMessageText) return; const key = msg.messageKey || `${msg.speaker}::${msg.text}`; const lastSeen = this._recentMessageKeyTimestamps.get(key); if (lastSeen && (nowMs - lastSeen) < 120000) return; this._recentMessageKeyTimestamps.set(key, nowMs); if (this._recentMessageKeyTimestamps.size > 400) { const cutoff = nowMs - 10 * 60 * 1000; for (const [k, ts] of this._recentMessageKeyTimestamps.entries()) { if (ts < cutoff) this._recentMessageKeyTimestamps.delete(k); } } this._lastMessageText = msg.text; // Classify as history (sent before bot joined) vs live const messageTimestamp = new Date(msg.teamsTimestamp || msg.timestamp); const messageMs = messageTimestamp.getTime(); const isHistory = this._botJoinedAtMs > 0 && messageMs < this._botJoinedAtMs; this._logger.info( `Chat${isHistory ? ' [HISTORY]' : ''}: [${msg.speaker}] ${msg.text}` + (msg.teamsTimestamp ? ` (teams_ts=${msg.teamsTimestamp})` : ''), ); this._onChatMessage({ speaker: msg.speaker, text: msg.text, timestamp: messageTimestamp, source: 'chat', isHistory, }); } /** * Send a chat message in the meeting. * Finds the chat input (with retry), types the message, and sends it. */ async sendChatMessage(text: string): Promise { this._logger.info(`Sending chat message: ${text.substring(0, 60)}...`); const inputSelectors = [ '[data-tid="ckeditor-replyConversation"]', 'div[role="textbox"][data-tid*="chat"]', 'div[role="textbox"][data-tid*="message"]', 'div[role="textbox"][aria-label*="message" i]', 'div[role="textbox"][aria-label*="Nachricht" i]', 'div[contenteditable="true"][data-tid*="message"]', 'div[contenteditable="true"][data-tid*="chat"]', 'div[contenteditable="true"][aria-label*="message" i]', '[aria-label*="Type a new message" i]', '[aria-label="Type a message"]', '[aria-label*="Neue Nachricht eingeben" i]', '[placeholder*="Type a message" i]', '[placeholder*="Nachricht" i]', 'div[contenteditable="true"][role="textbox"]', ]; const maxAttempts = 5; const retryDelayMs = 600; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { let input: any = null; for (const selector of inputSelectors) { input = await this._page.$(selector); if (input) { const isVisible = await input.isVisible().catch(() => false); if (isVisible) break; await input.dispose(); input = null; } } if (input) { await input.click(); await this._page.waitForTimeout(200); await this._page.keyboard.type(text, { delay: 10 }); await this._page.waitForTimeout(200); await this._page.keyboard.press('Enter'); this._logger.info('Chat message sent'); return true; } if (attempt < maxAttempts) { this._logger.info(`Chat input not found, retry ${attempt}/${maxAttempts} in ${retryDelayMs}ms`); await this._page.waitForTimeout(retryDelayMs); } } catch (error) { this._logger.error(`Error sending chat message (attempt ${attempt}):`, error); if (attempt < maxAttempts) { await this._page.waitForTimeout(retryDelayMs); } else { return false; } } } this._logger.warn('Could not find chat input field'); return false; } /** * Stop monitoring chat messages. */ async unsubscribe(): Promise { this._isSubscribed = false; if (this._scanIntervalId) { clearInterval(this._scanIntervalId); this._scanIntervalId = null; } try { await this._page.evaluate(() => { if ((window as any).__chatObserver) { (window as any).__chatObserver.disconnect(); } }); } catch { // Page might be closed } this._logger.info('Unsubscribed from chat messages'); } }