From 92f96bbc3e3232bd0e09b9f3c0610979625103fd Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 6 Mar 2026 13:14:43 +0100 Subject: [PATCH] chat: robustere Panel-Erkennung, erweiterte Input-Selektoren, DOM-Diagnostik, periodische Debug-Screenshots Made-with: Cursor --- src/bot/chatProcedure.ts | 163 ++++++++++++++++++++++++++++++--------- src/bot/orchestrator.ts | 10 +++ 2 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/bot/chatProcedure.ts b/src/bot/chatProcedure.ts index dc3ea54..4be8097 100644 --- a/src/bot/chatProcedure.ts +++ b/src/bot/chatProcedure.ts @@ -58,33 +58,64 @@ export class ChatProcedure { this._logger.info('Chat panel opened'); } + /** + * Check if the chat panel is currently visible by probing for known + * UI elements (chat input, message list, or aria-pressed toggle). + */ + private async _isChatPanelOpen(): Promise { + return this._page.evaluate(() => { + // 1. Chat button toggle state + const chatBtn = document.querySelector('button[id="chat-button"], button[data-tid="chat-button"]') as HTMLElement | null; + if (chatBtn?.getAttribute('aria-pressed') === 'true') return true; + + // 2. Chat input visible (multiple Teams UI variants) + const inputSelectors = [ + '[data-tid="ckeditor-replyConversation"]', + 'div[role="textbox"][data-tid*="chat"]', + 'div[role="textbox"][aria-label*="message" i]', + 'div[role="textbox"][aria-label*="Nachricht" i]', + '[contenteditable="true"][aria-label*="message" i]', + '[contenteditable="true"][aria-label*="Nachricht" i]', + '[placeholder*="message" i]', + '[placeholder*="Nachricht" i]', + 'div[aria-label="Type a message"]', + ]; + for (const sel of inputSelectors) { + const el = document.querySelector(sel) as HTMLElement | null; + if (el && el.offsetHeight > 0) return true; + } + + // 3. Message list container visible + const listSelectors = [ + '[data-tid="message-pane-list"]', + '[data-tid="chat-pane-list"]', + '[data-tid="chat-pane"]', + '.ts-message-list-container', + '[role="log"]', + ]; + for (const sel of listSelectors) { + const el = document.querySelector(sel) as HTMLElement | null; + if (el && el.offsetHeight > 50) return true; + } + + // 4. "Meeting chat" heading visible + const headings = document.querySelectorAll('h2, h3, [role="heading"]'); + for (const h of Array.from(headings)) { + const txt = (h as HTMLElement).innerText?.toLowerCase() || ''; + if (txt.includes('meeting chat') || txt.includes('besprechungschat')) return true; + } + + return false; + }); + } + /** * Open the chat panel by clicking the chat button. * In authenticated Teams, the chat panel may already be open (meeting loads * from a chat thread). Clicking again would TOGGLE it closed. */ private async _openChatPanel(): Promise { - // Check if chat panel is already open before clicking (avoid toggle-off) - const alreadyOpen = await this._page.evaluate(() => { - const chatBtn = document.querySelector('button[id="chat-button"], button[data-tid="chat-button"]') as HTMLElement | null; - if (chatBtn) { - // Teams toggle buttons use aria-pressed="true" when active - if (chatBtn.getAttribute('aria-pressed') === 'true') return true; - } - // Also check if chat input or message list is visible (panel is open) - const chatInput = document.querySelector( - '[data-tid="ckeditor-replyConversation"], div[role="textbox"][data-tid*="chat"], div[role="textbox"][aria-label*="message" i]' - ) as HTMLElement | null; - if (chatInput && chatInput.offsetHeight > 0) return true; - // Check if message list container is visible - const messageList = document.querySelector( - '[data-tid="message-pane-list"], [data-tid="chat-pane-list"], [data-tid="chat-pane"]' - ) as HTMLElement | null; - if (messageList && messageList.offsetHeight > 50) return true; - return false; - }); - - if (alreadyOpen) { + if (await this._isChatPanelOpen()) { this._logger.info('Chat panel already open - skipping toggle'); return; } @@ -369,27 +400,23 @@ export class ChatProcedure { */ private _startPeriodicChatScan(): void { if (this._scanIntervalId) return; - const intervalMs = 2500; + const intervalMs = 5000; this._scanIntervalId = setInterval(async () => { if (!this._isSubscribed || !this._page) return; try { - const chatPanelOpen = await this._page.evaluate(() => { - const chatBtn = document.querySelector('button[id="chat-button"], button[data-tid="chat-button"]') as HTMLElement | null; - if (chatBtn?.getAttribute('aria-pressed') === 'true') return true; - const messageList = document.querySelector('[data-tid="message-pane-list"], [data-tid="chat-pane-list"], [data-tid="chat-pane"]') as HTMLElement | null; - return !!(messageList && messageList.offsetHeight > 50); - }); - if (!chatPanelOpen) { + if (!(await this._isChatPanelOpen())) { this._logger.info('Chat panel closed, reopening'); await this._openChatPanel(); + await this._page.waitForTimeout(1000); } const knownKeys = Array.from(this._recentMessageKeyTimestamps.keys()); - const messages = await this._page.evaluate((knownKeysArr: string[]) => { + const scanResult = await this._page.evaluate((knownKeysArr: string[]) => { const known = new Set(knownKeysArr); const noisePatterns = [ 'meeting ended', 'meeting started', 'was invited', 'left the chat', 'joined the meeting', 'left the meeting', 'doesn\'t have a teams account', + 'verify their identity', ]; function isNoise(t: string) { const l = t.toLowerCase(); @@ -397,15 +424,45 @@ export class ChatProcedure { } const results: Array<{ speaker: string; text: string; timestamp: string; teamsTimestamp?: string; messageKey: string }> = []; const seenThisScan = new Set(); - const container = document.querySelector('[data-tid="message-pane-list"], [data-tid="chat-pane-list"], [data-tid="chat-pane"]') || document.body; - const candidates = container.querySelectorAll('[data-tid="chat-message"], .fui-ChatMessage, [data-tid*="chat-pane-message"]'); + + // Strategy 1: known selectors + const containerSelectors = [ + '[data-tid="message-pane-list"]', '[data-tid="chat-pane-list"]', + '[data-tid="chat-pane"]', '[role="log"]', '.ts-message-list-container', + ]; + let container: Element | null = null; + for (const sel of containerSelectors) { + container = document.querySelector(sel); + if (container) break; + } + + const messageSelectors = [ + '[data-tid="chat-message"]', '.fui-ChatMessage', + '[data-tid*="chat-pane-message"]', '[data-tid*="message-body"]', + ]; + const target = container || document.body; + const candidates = target.querySelectorAll(messageSelectors.join(', ')); + for (const el of Array.from(candidates) as HTMLElement[]) { const messageEl = el.closest?.('[data-tid="chat-message"], .fui-ChatMessage') || el; let author = 'Unknown'; - const authorEl = messageEl.querySelector('[data-tid="message-author"], [data-tid="message-author-name"], .fui-ChatMessage__author, [data-tid*="author"]'); - if (authorEl?.textContent) author = authorEl.textContent.trim(); - const bodyEl = messageEl.querySelector('[data-tid="message-body"], .fui-ChatMessage__body, [data-tid*="message-body"]'); - const text = (bodyEl as HTMLElement)?.innerText?.trim() || ''; + const authorSels = [ + '[data-tid="message-author"]', '[data-tid="message-author-name"]', + '.fui-ChatMessage__author', '[data-tid*="author"]', + ]; + for (const sel of authorSels) { + const authorEl = messageEl.querySelector(sel) || el.querySelector(sel); + if (authorEl?.textContent) { author = authorEl.textContent.trim(); break; } + } + const bodySels = [ + '[data-tid="message-body"]', '.fui-ChatMessage__body', + '[data-tid="chat-message-text"]', '[data-tid*="message-body"]', + ]; + let text = ''; + for (const sel of bodySels) { + const bodyEl = messageEl.querySelector(sel) || el.querySelector(sel); + if (bodyEl) { text = (bodyEl as HTMLElement).innerText?.trim() || ''; break; } + } if (!text || text.length < 2 || isNoise(text)) continue; const key = `${author}::${text}`; if (known.has(key) || seenThisScan.has(key)) continue; @@ -414,9 +471,35 @@ export class ChatProcedure { const ts = timeEl?.getAttribute?.('datetime') || new Date().toISOString(); results.push({ speaker: author, text, timestamp: ts, teamsTimestamp: ts, messageKey: key }); } - return results; + + // Diagnostics (once per 20s, controlled by caller) + let diag: string | undefined; + if ((window as any).__chatScanDiagCounter === undefined) (window as any).__chatScanDiagCounter = 0; + (window as any).__chatScanDiagCounter++; + if ((window as any).__chatScanDiagCounter % 4 === 1) { + const info: string[] = []; + info.push(`container=${container?.tagName || 'body'}[${container?.getAttribute?.('data-tid') || ''}]`); + info.push(`candidates=${candidates.length}`); + // Dump all chat-ish elements for debugging + const allChat = document.querySelectorAll('[data-tid*="chat"], [data-tid*="message"], [role="log"], .fui-Chat'); + const tags: string[] = []; + for (const c of Array.from(allChat).slice(0, 15)) { + const tid = c.getAttribute('data-tid') || ''; + const cls = c.className?.toString?.()?.substring(0, 40) || ''; + const h = (c as HTMLElement).offsetHeight || 0; + tags.push(`<${c.tagName} tid="${tid}" cls="${cls}" h=${h}>`); + } + info.push(`domElements=[${tags.join(', ')}]`); + diag = info.join(' | '); + } + + return { messages: results, diag }; }, knownKeys); - for (const msg of messages) { + + if (scanResult.diag) { + this._logger.info(`[ChatScan] ${scanResult.diag}`); + } + for (const msg of scanResult.messages) { this._handleChatMessage(msg); } } catch { @@ -490,7 +573,11 @@ export class ChatProcedure { 'div[contenteditable="true"][data-tid*="chat"]', 'div[contenteditable="true"][aria-label*="message" i]', '[aria-label*="Type a new message" i]', + '[aria-label="Type a message"]', '[aria-label*="Neue Nachricht eingeben" i]', + '[placeholder*="Type a message" i]', + '[placeholder*="Nachricht" i]', + 'div[contenteditable="true"][role="textbox"]', ]; const maxAttempts = 5; diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index d4dcc53..b44cdd5 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -589,12 +589,17 @@ export class BotOrchestrator { * a WebSocket ping. Prevents Teams from detecting the bot as idle * and kicking it from the meeting. */ + private _keepAliveCounter: number = 0; + private _startKeepAlive(): void { if (this._keepAliveInterval) return; + this._keepAliveCounter = 0; this._keepAliveInterval = setInterval(async () => { if (this._isShuttingDown || !this._page) return; + this._keepAliveCounter++; + try { // Small random mouse movement to simulate user activity const x = 640 + Math.floor(Math.random() * 20 - 10); @@ -612,6 +617,11 @@ export class BotOrchestrator { // Connection might be closing } } + + // Periodic debug screenshot every ~45s (every 3rd keepalive cycle) + if (this._isDebugMode && this._keepAliveCounter % 3 === 0) { + await this._takeScreenshot(`debug-periodic-${this._keepAliveCounter}`); + } }, 15000); this._logger.info('Keepalive started (15s interval)');