service-teams-browser-bot/src/bot/chatProcedure.ts

651 lines
23 KiB
TypeScript

import { Page } from 'playwright';
import { Logger } from 'winston';
export interface ChatMessageEntry {
speaker: string;
text: string;
timestamp: Date;
source: 'chat';
isHistory: boolean;
}
/**
* Handles reading and writing chat messages in a Teams meeting.
*
* Teams meeting chat is a separate panel that can be opened via the chat button.
* Messages are rendered as DOM elements that we observe via MutationObserver.
*/
export class ChatProcedure {
private _page: Page;
private _logger: Logger;
private _onChatMessage: (entry: ChatMessageEntry) => void;
private _isSubscribed: boolean = false;
private _lastMessageText: string = '';
private _botJoinedAtMs: number = 0;
private _recentMessageKeyTimestamps: Map<string, number> = new Map();
private _scanIntervalId: ReturnType<typeof setInterval> | null = null;
constructor(
page: Page,
logger: Logger,
onChatMessage: (entry: ChatMessageEntry) => void
) {
this._page = page;
this._logger = logger;
this._onChatMessage = onChatMessage;
}
/**
* Set the bot's join timestamp so we can classify messages as history vs live.
*/
setBotJoinedAt(ms: number): void {
this._botJoinedAtMs = ms;
this._logger.info(`Chat history cutoff set: messages before ${new Date(ms).toISOString()} are history`);
}
/**
* Open the chat panel and start monitoring messages.
*/
async enableChatMonitoring(): Promise<void> {
this._logger.info('Enabling chat monitoring...');
// Open chat panel
await this._openChatPanel();
// Wait for chat to load
await this._page.waitForTimeout(2000);
this._logger.info('Chat panel opened');
}
/**
* Check if the chat panel is currently visible by probing for known
* UI elements (chat input, message list, or aria-pressed toggle).
*/
private async _isChatPanelOpen(): Promise<boolean> {
return this._page.evaluate(() => {
// 1. Chat button toggle state
const chatBtn = document.querySelector('button[id="chat-button"], button[data-tid="chat-button"]') as HTMLElement | null;
if (chatBtn?.getAttribute('aria-pressed') === 'true') return true;
// 2. Chat input visible (multiple Teams UI variants)
const inputSelectors = [
'[data-tid="ckeditor-replyConversation"]',
'div[role="textbox"][data-tid*="chat"]',
'div[role="textbox"][aria-label*="message" i]',
'div[role="textbox"][aria-label*="Nachricht" i]',
'[contenteditable="true"][aria-label*="message" i]',
'[contenteditable="true"][aria-label*="Nachricht" i]',
'[placeholder*="message" i]',
'[placeholder*="Nachricht" i]',
'div[aria-label="Type a message"]',
];
for (const sel of inputSelectors) {
const el = document.querySelector(sel) as HTMLElement | null;
if (el && el.offsetHeight > 0) return true;
}
// 3. Message list container visible
const listSelectors = [
'[data-tid="message-pane-list"]',
'[data-tid="chat-pane-list"]',
'[data-tid="chat-pane"]',
'.ts-message-list-container',
'[role="log"]',
];
for (const sel of listSelectors) {
const el = document.querySelector(sel) as HTMLElement | null;
if (el && el.offsetHeight > 50) return true;
}
// 4. "Meeting chat" heading visible
const headings = document.querySelectorAll('h2, h3, [role="heading"]');
for (const h of Array.from(headings)) {
const txt = (h as HTMLElement).innerText?.toLowerCase() || '';
if (txt.includes('meeting chat') || txt.includes('besprechungschat')) return true;
}
return false;
});
}
/**
* Open the chat panel by clicking the chat button.
* In authenticated Teams, the chat panel may already be open (meeting loads
* from a chat thread). Clicking again would TOGGLE it closed.
*/
private async _openChatPanel(): Promise<void> {
if (await this._isChatPanelOpen()) {
this._logger.info('Chat panel already open - skipping toggle');
return;
}
const chatButtonSelectors = [
'button[id="chat-button"]',
'button[data-tid="chat-button"]',
'button[aria-label*="Chat" i]',
'button[aria-label*="chat" i]',
'#chat-button',
];
for (const selector of chatButtonSelectors) {
try {
const button = await this._page.$(selector);
if (button) {
await button.click();
this._logger.info(`Opened chat panel: ${selector}`);
return;
}
} catch {
// Continue
}
}
this._logger.warn('Could not find chat button - chat monitoring will not work');
}
/**
* Subscribe to chat messages using MutationObserver.
*/
async subscribeToChatMessages(): Promise<void> {
if (this._isSubscribed) {
this._logger.warn('Already subscribed to chat messages');
return;
}
this._isSubscribed = true;
this._logger.info('Subscribing to chat messages...');
// Expose callbacks from Node.js to browser
try {
await this._page.exposeFunction('__onChatMessageEvent', (msg: {
speaker: string;
text: string;
timestamp: string;
teamsTimestamp?: string;
messageKey?: string;
}) => {
this._handleChatMessage(msg);
});
} catch {
// Function may already be exposed from a previous subscription
}
try {
await this._page.exposeFunction('__onChatDebug', (info: {
tag: string;
tid: string;
text: string;
children: number;
html: string;
}) => {
this._logger.debug(`ChatDOM: <${info.tag} data-tid="${info.tid}"> children=${info.children}, text="${info.text.substring(0, 120)}"`);
});
} catch {
// Already exposed
}
// Find chat container and set up observer
const chatObserverTarget = await this._page.evaluate(() => {
// Noise patterns: system messages, not actual chat
const noisePatterns = [
'meeting ended', 'meeting started', 'was invited', 'left the chat',
'joined the meeting', 'left the meeting', 'doesn\'t have a teams account',
'verify their identity', 'new notification', 'last read',
];
function _isNoise(text: string): boolean {
const lower = text.toLowerCase();
return noisePatterns.some(p => lower.includes(p));
}
function _extractTeamsTimestamp(el: HTMLElement): string | undefined {
const timeSelectors = [
'time[datetime]',
'[data-tid="message-timestamp"] time',
'[data-tid*="timestamp"] time',
'[data-tid="message-timestamp"]',
];
for (const sel of timeSelectors) {
const timeEl = el.querySelector(sel) || el.closest?.('[data-tid*="message"]')?.querySelector(sel);
if (timeEl) {
const dt = timeEl.getAttribute('datetime');
if (dt) return dt;
const title = timeEl.getAttribute('title');
if (title) return title;
}
}
return undefined;
}
function _extractChatMessage(el: HTMLElement): boolean {
const messageSelectors = [
'[data-tid="chat-message"]',
'.fui-ChatMessage',
'[data-tid*="message-body"]',
'[data-tid*="chat-pane-message"]',
];
let messageEl: HTMLElement | null = null;
for (const sel of messageSelectors) {
messageEl = el.matches?.(sel) ? el : el.querySelector(sel);
if (messageEl) break;
}
if (messageEl) {
const authorSelectors = [
'[data-tid="message-author"]',
'[data-tid="message-author-name"]',
'.fui-ChatMessage__author',
'[data-tid*="author"]',
];
let author = 'Unknown';
for (const sel of authorSelectors) {
const authorEl = messageEl.querySelector(sel) || el.querySelector(sel);
if (authorEl?.textContent) {
author = authorEl.textContent.trim();
break;
}
}
const bodySelectors = [
'[data-tid="message-body"]',
'.fui-ChatMessage__body',
'[data-tid="chat-message-text"]',
'[data-tid*="message-body"]',
];
let text = '';
for (const sel of bodySelectors) {
const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel);
if (bodyEl) {
text = (bodyEl as HTMLElement).innerText?.trim() || '';
break;
}
}
if (text && text.length > 0) {
const teamsTs = _extractTeamsTimestamp(messageEl) || _extractTeamsTimestamp(el);
(window as any).__onChatMessageEvent({
speaker: author,
text,
timestamp: teamsTs || new Date().toISOString(),
teamsTimestamp: teamsTs,
messageKey: `${author}::${text}`,
});
return true;
}
}
// Strategy 2: Structural fallback for authenticated Teams chat
// Chat messages typically have: author element + body element as children
const fullText = el.innerText?.trim() || '';
if (!fullText || fullText.length < 2 || _isNoise(fullText)) return false;
// Skip typing indicators, system messages
const tid = el.getAttribute('data-tid') || '';
if (tid === 'typing-indicator') return false;
// Look for elements that look like user messages (have author-like + body-like children)
const children = Array.from(el.children) as HTMLElement[];
if (children.length >= 2) {
// Find an element that looks like a name (short text, no data-tid with "body")
for (let i = 0; i < children.length - 1; i++) {
const candidateName = children[i].innerText?.trim() || '';
const candidateBody = children.slice(i + 1).map(c => c.innerText?.trim()).filter(Boolean).join(' ').trim();
if (
candidateName.length > 1 && candidateName.length < 60 &&
candidateBody.length > 1 &&
!_isNoise(candidateBody) &&
!candidateName.includes('meeting') && !candidateName.includes('Meeting')
) {
// Check if this looks like a time-stamped message (not just any two children)
const hasTid = children[i].getAttribute('data-tid') || '';
if (hasTid.includes('author') || hasTid.includes('name') || hasTid.includes('sender')) {
const teamsTs = _extractTeamsTimestamp(el);
(window as any).__onChatMessageEvent({
speaker: candidateName,
text: candidateBody,
timestamp: teamsTs || new Date().toISOString(),
teamsTimestamp: teamsTs,
messageKey: `${candidateName}::${candidateBody}`,
});
return true;
}
}
}
}
return false;
}
// Teams chat containers - try multiple selectors
const chatContainerSelectors = [
'[data-tid="message-pane-list"]',
'[data-tid="chat-pane"]',
'[data-tid="chat-pane-list"]',
'.ts-message-list-container',
'[role="log"]',
];
let chatContainer: Element | null = null;
let matchedSelector = '';
for (const sel of chatContainerSelectors) {
chatContainer = document.querySelector(sel);
if (chatContainer) {
matchedSelector = sel;
break;
}
}
if (!chatContainer) {
const candidates = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"]');
for (const c of Array.from(candidates)) {
const cTid = c.getAttribute('data-tid') || '';
// Prefer larger containers, not buttons or small elements
if ((c as HTMLElement).offsetHeight > 50 && c.tagName !== 'BUTTON') {
chatContainer = c;
matchedSelector = `[data-tid="${cTid}"]`;
break;
}
}
}
// Use found container or fall back to document.body
const target = chatContainer || document.body;
const observer = new MutationObserver((mutations) => {
for (const mutation of mutations) {
if (mutation.type === 'childList') {
mutation.addedNodes.forEach((node) => {
if (node.nodeType !== Node.ELEMENT_NODE) return;
const el = node as HTMLElement;
const text = el.innerText?.trim() || '';
if (!text || text.length < 2) return;
if (!_extractChatMessage(el)) {
// Log unrecognized elements for debugging (skip noise)
if (!_isNoise(text) && text.length > 3) {
const tid = el.getAttribute('data-tid') || '';
if (tid !== 'typing-indicator') {
(window as any).__onChatDebug?.({
tag: el.tagName,
tid,
text: text.substring(0, 200),
children: el.children?.length || 0,
html: el.innerHTML?.substring(0, 500) || '',
});
}
}
}
});
}
}
});
observer.observe(target, { childList: true, subtree: true });
(window as any).__chatObserver = observer;
return chatContainer ? `container:${matchedSelector}` : 'body-fallback';
});
this._logger.info(`Chat MutationObserver set up (target: ${chatObserverTarget})`);
this._startPeriodicChatScan();
}
/**
* Periodic scan as fallback: participant messages may load async and miss
* the MutationObserver (addedNode with empty placeholder, then content update).
* Also reopens the chat panel if it was closed (e.g. by user or Teams).
*/
private _startPeriodicChatScan(): void {
if (this._scanIntervalId) return;
const intervalMs = 5000;
this._scanIntervalId = setInterval(async () => {
if (!this._isSubscribed || !this._page) return;
try {
if (!(await this._isChatPanelOpen())) {
this._logger.info('Chat panel closed, reopening');
await this._openChatPanel();
await this._page.waitForTimeout(1000);
}
const knownKeys = Array.from(this._recentMessageKeyTimestamps.keys());
const scanResult = await this._page.evaluate((knownKeysArr: string[]) => {
const known = new Set(knownKeysArr);
const noisePatterns = [
'meeting ended', 'meeting started', 'was invited', 'left the chat',
'joined the meeting', 'left the meeting', 'doesn\'t have a teams account',
'verify their identity',
];
function isNoise(t: string) {
const l = t.toLowerCase();
return noisePatterns.some(p => l.includes(p));
}
const results: Array<{ speaker: string; text: string; timestamp: string; teamsTimestamp?: string; messageKey: string }> = [];
const seenThisScan = new Set<string>();
// Strategy 1: known selectors
const containerSelectors = [
'[data-tid="message-pane-list"]', '[data-tid="chat-pane-list"]',
'[data-tid="chat-pane"]', '[role="log"]', '.ts-message-list-container',
];
let container: Element | null = null;
for (const sel of containerSelectors) {
container = document.querySelector(sel);
if (container) break;
}
const messageSelectors = [
'[data-tid="chat-message"]', '.fui-ChatMessage',
'[data-tid*="chat-pane-message"]', '[data-tid*="message-body"]',
];
const target = container || document.body;
const candidates = target.querySelectorAll(messageSelectors.join(', '));
for (const el of Array.from(candidates) as HTMLElement[]) {
const messageEl = el.closest?.('[data-tid="chat-message"], .fui-ChatMessage') || el;
let author = 'Unknown';
const authorSels = [
'[data-tid="message-author"]', '[data-tid="message-author-name"]',
'.fui-ChatMessage__author', '[data-tid*="author"]',
];
for (const sel of authorSels) {
const authorEl = messageEl.querySelector(sel) || el.querySelector(sel);
if (authorEl?.textContent) { author = authorEl.textContent.trim(); break; }
}
const bodySels = [
'[data-tid="message-body"]', '.fui-ChatMessage__body',
'[data-tid="chat-message-text"]', '[data-tid*="message-body"]',
];
let text = '';
for (const sel of bodySels) {
const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel);
if (bodyEl) { text = (bodyEl as HTMLElement).innerText?.trim() || ''; break; }
}
if (!text || text.length < 2 || isNoise(text)) continue;
const key = `${author}::${text}`;
if (known.has(key) || seenThisScan.has(key)) continue;
seenThisScan.add(key);
const timeEl = messageEl.querySelector('time[datetime], [data-tid*="timestamp"] time');
const ts = timeEl?.getAttribute?.('datetime') || new Date().toISOString();
results.push({ speaker: author, text, timestamp: ts, teamsTimestamp: ts, messageKey: key });
}
// Diagnostics (once per 20s, controlled by caller)
let diag: string | undefined;
if ((window as any).__chatScanDiagCounter === undefined) (window as any).__chatScanDiagCounter = 0;
(window as any).__chatScanDiagCounter++;
if ((window as any).__chatScanDiagCounter % 4 === 1) {
const info: string[] = [];
info.push(`container=${container?.tagName || 'body'}[${container?.getAttribute?.('data-tid') || ''}]`);
info.push(`candidates=${candidates.length}`);
// Dump all chat-ish elements for debugging
const allChat = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"], [role="log"], .fui-Chat');
const tags: string[] = [];
for (const c of Array.from(allChat).slice(0, 15)) {
const tid = c.getAttribute('data-tid') || '';
const cls = c.className?.toString?.()?.substring(0, 40) || '';
const h = (c as HTMLElement).offsetHeight || 0;
tags.push(`<${c.tagName} tid="${tid}" cls="${cls}" h=${h}>`);
}
info.push(`domElements=[${tags.join(', ')}]`);
diag = info.join(' | ');
}
return { messages: results, diag };
}, knownKeys);
if (scanResult.diag) {
this._logger.info(`[ChatScan] ${scanResult.diag}`);
}
for (const msg of scanResult.messages) {
this._handleChatMessage(msg);
}
} catch {
// Page may be closed
}
}, intervalMs);
this._logger.info(`Chat periodic scan started (interval: ${intervalMs}ms)`);
}
/**
* Handle a chat message event from the browser.
*/
private _handleChatMessage(msg: {
speaker: string;
text: string;
timestamp: string;
teamsTimestamp?: string;
messageKey?: string;
}): void {
if (!this._isSubscribed || !msg.text) return;
const nowMs = Date.now();
// Dedup
if (msg.text === this._lastMessageText) return;
const key = msg.messageKey || `${msg.speaker}::${msg.text}`;
const lastSeen = this._recentMessageKeyTimestamps.get(key);
if (lastSeen && (nowMs - lastSeen) < 120000) return;
this._recentMessageKeyTimestamps.set(key, nowMs);
if (this._recentMessageKeyTimestamps.size > 400) {
const cutoff = nowMs - 10 * 60 * 1000;
for (const [k, ts] of this._recentMessageKeyTimestamps.entries()) {
if (ts < cutoff) this._recentMessageKeyTimestamps.delete(k);
}
}
this._lastMessageText = msg.text;
// Classify as history (sent before bot joined) vs live
const messageTimestamp = new Date(msg.teamsTimestamp || msg.timestamp);
const messageMs = messageTimestamp.getTime();
const isHistory = this._botJoinedAtMs > 0 && messageMs < this._botJoinedAtMs;
this._logger.info(
`Chat${isHistory ? ' [HISTORY]' : ''}: [${msg.speaker}] ${msg.text}` +
(msg.teamsTimestamp ? ` (teams_ts=${msg.teamsTimestamp})` : ''),
);
this._onChatMessage({
speaker: msg.speaker,
text: msg.text,
timestamp: messageTimestamp,
source: 'chat',
isHistory,
});
}
/**
* Send a chat message in the meeting.
* Finds the chat input (with retry), types the message, and sends it.
*/
async sendChatMessage(text: string): Promise<boolean> {
this._logger.info(`Sending chat message: ${text.substring(0, 60)}...`);
const inputSelectors = [
'[data-tid="ckeditor-replyConversation"]',
'div[role="textbox"][data-tid*="chat"]',
'div[role="textbox"][data-tid*="message"]',
'div[role="textbox"][aria-label*="message" i]',
'div[role="textbox"][aria-label*="Nachricht" i]',
'div[contenteditable="true"][data-tid*="message"]',
'div[contenteditable="true"][data-tid*="chat"]',
'div[contenteditable="true"][aria-label*="message" i]',
'[aria-label*="Type a new message" i]',
'[aria-label="Type a message"]',
'[aria-label*="Neue Nachricht eingeben" i]',
'[placeholder*="Type a message" i]',
'[placeholder*="Nachricht" i]',
'div[contenteditable="true"][role="textbox"]',
];
const maxAttempts = 5;
const retryDelayMs = 600;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
let input: any = null;
for (const selector of inputSelectors) {
input = await this._page.$(selector);
if (input) {
const isVisible = await input.isVisible().catch(() => false);
if (isVisible) break;
await input.dispose();
input = null;
}
}
if (input) {
await input.click();
await this._page.waitForTimeout(200);
await this._page.keyboard.type(text, { delay: 10 });
await this._page.waitForTimeout(200);
await this._page.keyboard.press('Enter');
this._logger.info('Chat message sent');
return true;
}
if (attempt < maxAttempts) {
this._logger.info(`Chat input not found, retry ${attempt}/${maxAttempts} in ${retryDelayMs}ms`);
await this._page.waitForTimeout(retryDelayMs);
}
} catch (error) {
this._logger.error(`Error sending chat message (attempt ${attempt}):`, error);
if (attempt < maxAttempts) {
await this._page.waitForTimeout(retryDelayMs);
} else {
return false;
}
}
}
this._logger.warn('Could not find chat input field');
return false;
}
/**
* Stop monitoring chat messages.
*/
async unsubscribe(): Promise<void> {
this._isSubscribed = false;
if (this._scanIntervalId) {
clearInterval(this._scanIntervalId);
this._scanIntervalId = null;
}
try {
await this._page.evaluate(() => {
if ((window as any).__chatObserver) {
(window as any).__chatObserver.disconnect();
}
});
} catch {
// Page might be closed
}
this._logger.info('Unsubscribed from chat messages');
}
}