From dddf1cd97006687ca311a89f01d24149f1a56670 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 18 Feb 2026 21:41:48 +0100 Subject: [PATCH] refactor: central _pollForElement utility (500ms interval), remove all fixed waits and snapshot checks Co-authored-by: Cursor --- src/bot/orchestrator.ts | 318 +++++++++++++--------------------------- 1 file changed, 101 insertions(+), 217 deletions(-) diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index 86c414a..2463c8b 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -1,4 +1,4 @@ -import { Browser, BrowserContext, Page, chromium } from 'playwright'; +import { Browser, BrowserContext, Page, ElementHandle, chromium } from 'playwright'; import { Logger } from 'winston'; import { v4 as uuidv4 } from 'uuid'; import path from 'path'; @@ -86,6 +86,37 @@ export class BotOrchestrator { this._logger = createSessionLogger(sessionId); } + /** + * Poll for a DOM element matching any of the given selectors. + * Checks every 500ms until found or timeout is reached. + * Returns the element handle if found, or null on timeout. + */ + private async _pollForElement( + selectors: string | string[], + timeoutMs: number = 15000, + label?: string, + ): Promise { + const selectorList = Array.isArray(selectors) ? selectors : [selectors]; + const combined = selectorList.join(', '); + const tag = label || combined.substring(0, 60); + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + for (const selector of selectorList) { + try { + const el = await this._page!.$(selector); + if (el) { + this._logger.info(`[poll] Found "${tag}" via: ${selector}`); + return el; + } + } catch { /* page navigated or element detached — retry */ } + } + await this._page!.waitForTimeout(500); + } + this._logger.warn(`[poll] "${tag}" not found within ${timeoutMs}ms`); + return null; + } + get sessionId(): string { return this._sessionId; } @@ -180,11 +211,12 @@ export class BotOrchestrator { /** * Join a meeting as authenticated user (System Bot or User Account). * Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now + * + * Every UI step uses _pollForElement (500ms interval) for both stability and performance: + * no fixed waits, the flow proceeds as soon as each element appears. */ private async _attemptAuthJoin(): Promise { - // Launch browser in headful mode with minimal args (Chromium Minimal) await this._launchBrowser(true); - this._setState('navigating'); // STEP 1: Navigate to teams.microsoft.com to trigger authentication @@ -194,21 +226,14 @@ export class BotOrchestrator { timeout: 30000, }); - // Wait for login redirect - try { - await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 }); - this._logger.info('Redirected to MS login page'); - } catch { - this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`); - } - - // Wait for login page to render - try { - await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', { - timeout: 15000, state: 'visible', - }); - } catch { - this._logger.warn('Login page elements not found'); + // Poll for login page (redirect to login.microsoftonline.com) + const emailInput = await this._pollForElement( + ['input[name="loginfmt"]', 'input[type="email"]'], + 30000, + 'MS login email input', + ); + if (!emailInput) { + this._logger.warn(`No login page found, current URL: ${this._page!.url().substring(0, 150)}`); } // STEP 2: Microsoft Authentication @@ -228,35 +253,23 @@ export class BotOrchestrator { // STEP 3: Wait for Teams to finish loading after auth this._logger.info('Waiting for Teams to load after auth...'); try { - await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 }); + await this._page!.waitForURL( + (url) => url.hostname.includes('teams.microsoft.com') || url.hostname.includes('teams.cloud.microsoft'), + { timeout: 30000 }, + ); } catch { - try { - await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 }); - } catch { - this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`); - } + this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`); } - // Give Teams a moment to initialize (session cookies, service workers) - await this._page!.waitForTimeout(3000); - // STEP 4: Navigate directly to the meeting URL (authenticated session) - // This is the key step: the previous flow waited for a "Join" button in - // the Teams chat header, which only works if the meeting chat happens to - // be visible. Navigating directly to the meeting URL reliably shows the - // pre-join screen for authenticated users. + // STEP 4: Navigate to the meeting URL this._logger.info(`Auth join: navigating to meeting URL: ${this._meetingUrl.substring(0, 80)}...`); await this._page!.goto(this._meetingUrl, { waitUntil: 'domcontentloaded', timeout: 30000, }); + this._logger.info(`Auth join: URL after navigation: ${this._page!.url().substring(0, 150)}`); - // Teams may show interstitials before the pre-join screen. - // Poll for ALL possible next-step buttons simultaneously so we don't - // miss one if it appears slowly. - const currentUrl = this._page!.url(); - this._logger.info(`Auth join: URL after meeting navigation: ${currentUrl.substring(0, 150)}`); - - // Race: wait for whichever button appears first (interstitial OR pre-join) + // STEP 4a: Poll for the first actionable button (interstitial OR pre-join) const interstitialSelectors = [ 'button:has-text("Continue on this browser")', 'button:has-text("In diesem Browser fortfahren")', @@ -269,95 +282,50 @@ export class BotOrchestrator { 'button:has-text("Jetzt teilnehmen")', 'button[data-tid="prejoin-join-button"]', ]; - const allSelectors = [...interstitialSelectors, ...preJoinSelectors].join(', '); - // Poll up to 30s for ANY of these buttons - this._logger.info('Waiting for interstitial or pre-join button...'); - try { - const firstBtn = await this._page!.waitForSelector(allSelectors, { - timeout: 30000, state: 'visible', - }); - if (firstBtn) { - const btnText = await firstBtn.textContent().catch(() => ''); - const btnTid = await firstBtn.getAttribute('data-tid').catch(() => ''); - this._logger.info(`First visible button: "${btnText?.trim()}" (data-tid="${btnTid}")`); + const firstBtn = await this._pollForElement( + [...interstitialSelectors, ...preJoinSelectors], + 30000, + 'interstitial or pre-join button', + ); - // If it's an interstitial button (not pre-join), click it and wait for the next screen - const isPreJoin = preJoinSelectors.some(s => - s.includes(btnTid || '__none__') || (btnText && s.includes(btnText.trim())) - ); - if (!isPreJoin) { - await firstBtn.click(); - this._logger.info('Clicked interstitial button, waiting for pre-join screen...'); - await this._page!.waitForTimeout(3000); - } + if (firstBtn) { + const btnText = (await firstBtn.textContent().catch(() => ''))?.trim() || ''; + const btnTid = (await firstBtn.getAttribute('data-tid').catch(() => '')) || ''; + + const isPreJoin = btnTid === 'prejoin-join-button' + || btnText.toLowerCase().includes('join now') + || btnText.toLowerCase().includes('jetzt teilnehmen'); + + if (!isPreJoin) { + await firstBtn.click(); + this._logger.info(`Clicked interstitial: "${btnText}"`); } - } catch { - this._logger.warn('No interstitial or pre-join button found within 30s'); + } else { await this._takeScreenshot('auth-no-buttons'); } - // STEP 5: Pre-Join screen → Click "Join now" - this._logger.info('Waiting for pre-join screen'); - try { - await this._page!.waitForSelector( - 'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]', - { timeout: 30000, state: 'visible' }, - ); - } catch { - this._logger.warn('"Join now" button not found'); - await this._takeScreenshot('auth-no-join-now'); - } - - this._logger.info('Camera left OFF (video disabled for stability)'); - - // Ensure microphone is ON (required for voice playback) + // STEP 5: Poll for "Join now" on the pre-join screen await this._ensureMicOn(); - await this._page!.waitForTimeout(2000); - - const joinNowSelectors = [ - 'button:has-text("Join now")', - 'button:has-text("Jetzt teilnehmen")', - 'button[data-tid="prejoin-join-button"]', - ]; - - let joinNowClicked = false; - for (const selector of joinNowSelectors) { - try { - const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' }); - if (btn) { - await btn.click(); - joinNowClicked = true; - break; - } - } catch { /* try next */ } - } - - if (!joinNowClicked) { - await this._takeScreenshot('auth-no-join-now-final'); + const joinNowBtn = await this._pollForElement(preJoinSelectors, 30000, 'Join now button'); + if (!joinNowBtn) { + await this._takeScreenshot('auth-no-join-now'); throw new Error('"Join now" button not found on pre-join screen'); } - + await joinNowBtn.click(); this._logger.info('Clicked "Join now", waiting for meeting'); - // Wait for meeting admission (hangup button = in meeting) + // STEP 6: Wait for meeting admission (hangup button = in meeting) await this._waitForMeetingAdmission(); this._setState('in_meeting'); this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`); - // Start keepalive to prevent idle disconnect this._startKeepAlive(); - - // Initialize audio playback await this._audioProcedure!.initialize(); - - // Enable transcript capture (captions or audio based on transferMode) await this._enableTranscriptCapture(); await this._enableChat(); - - // Send greeting in meeting chat await this._sendJoinGreeting(); } @@ -372,52 +340,26 @@ export class BotOrchestrator { */ private async _ensureCameraOn(): Promise { try { - // Primary: the actual switch input (fui-Switch) - let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]'); + const cameraToggle = await this._pollForElement([ + 'input[data-tid="toggle-video"]', + '[data-tid="toggle-video"]', + 'input[role="switch"][title*="camera" i]', + 'input[role="switch"][title*="Camera" i]', + 'input[role="switch"][title*="Video" i]', + ], 10000, 'camera toggle (pre-join)'); - if (!cameraToggle) { - this._logger.info('Primary camera selector not found, trying fallbacks...'); - const fallbacks = [ - '[data-tid="toggle-video"]', - 'input[role="switch"][title*="camera" i]', - 'input[role="switch"][title*="Camera" i]', - 'input[role="switch"][title*="Video" i]', - ]; - for (const sel of fallbacks) { - cameraToggle = await this._page!.$(sel); - if (cameraToggle) { - this._logger.info(`Camera toggle found via fallback: ${sel}`); - break; - } - } - } + if (!cameraToggle) return; - if (!cameraToggle) { - this._logger.warn('Camera toggle not found on pre-join screen'); - return; - } - - // Read current state const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', title: el.getAttribute('title') || '', })); - this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); if (!state.checked) { - // Camera is OFF — click to turn ON await cameraToggle.click(); this._logger.info('Camera toggled ON'); - await this._page!.waitForTimeout(2000); - - // Verify - const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({ - checked: el.checked, - dataCid: el.getAttribute('data-cid') || '', - })); - this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`); } else { this._logger.info('Camera already ON'); } @@ -435,48 +377,22 @@ export class BotOrchestrator { */ private async _ensureCameraOnInMeeting(): Promise { try { - // Wait a moment for meeting controls to render - await this._page!.waitForTimeout(2000); + const videoBtn = await this._pollForElement([ + 'button#video-button', + 'button[data-inp="video-button"]', + 'button[aria-label*="camera" i]', + 'button[aria-label*="Camera" i]', + 'button[aria-label*="Video" i]', + ], 10000, 'in-meeting camera button'); - // Find the in-meeting video button - let videoBtn = await this._page!.$('button#video-button'); + if (!videoBtn) return; - if (!videoBtn) { - // Fallback selectors - const fallbacks = [ - 'button[data-inp="video-button"]', - 'button[id="video-button"]', - 'button[aria-label*="camera" i]', - 'button[aria-label*="Camera" i]', - 'button[aria-label*="Video" i]', - ]; - for (const sel of fallbacks) { - videoBtn = await this._page!.$(sel); - if (videoBtn) { - this._logger.info(`In-meeting video button found via fallback: ${sel}`); - break; - } - } - } - - if (!videoBtn) { - this._logger.warn('In-meeting video button not found'); - return; - } - - // Read current state - const state = await videoBtn.evaluate((el) => ({ + const state = await videoBtn.evaluate((el: HTMLElement) => ({ dataState: el.getAttribute('data-state') || '', ariaLabel: el.getAttribute('aria-label') || '', - id: el.id, })); + this._logger.info(`In-meeting camera: data-state="${state.dataState}", aria-label="${state.ariaLabel}"`); - this._logger.info( - `In-meeting camera: data-state="${state.dataState}", ` + - `aria-label="${state.ariaLabel}", id="${state.id}"`, - ); - - // Camera is off if data-state is "call-video-off" const isOff = state.dataState === 'call-video-off' || state.ariaLabel.toLowerCase().includes('turn camera on') || state.ariaLabel.toLowerCase().includes('kamera einschalten'); @@ -484,17 +400,6 @@ export class BotOrchestrator { if (isOff) { await videoBtn.click(); this._logger.info('In-meeting camera was OFF — clicked to turn ON'); - await this._page!.waitForTimeout(2000); - - // Verify - const afterState = await videoBtn.evaluate((el) => ({ - dataState: el.getAttribute('data-state') || '', - ariaLabel: el.getAttribute('aria-label') || '', - })); - this._logger.info( - `In-meeting camera after toggle: data-state="${afterState.dataState}", ` + - `aria-label="${afterState.ariaLabel}"`, - ); } else { this._logger.info('In-meeting camera already ON'); } @@ -514,48 +419,27 @@ export class BotOrchestrator { */ private async _ensureMicOn(): Promise { try { - let micToggle = await this._page!.$('input[data-tid="toggle-audio"]'); + const micToggle = await this._pollForElement([ + 'input[data-tid="toggle-audio"]', + '[data-tid="toggle-audio"]', + 'input[role="switch"][title*="microphone" i]', + 'input[role="switch"][title*="Mikrofon" i]', + 'input[role="switch"][title*="mic" i]', + 'input[role="switch"][title*="audio" i]', + ], 10000, 'mic toggle (pre-join)'); - if (!micToggle) { - const fallbacks = [ - '[data-tid="toggle-audio"]', - 'input[role="switch"][title*="microphone" i]', - 'input[role="switch"][title*="Mikrofon" i]', - 'input[role="switch"][title*="mic" i]', - 'input[role="switch"][title*="audio" i]', - ]; - for (const sel of fallbacks) { - micToggle = await this._page!.$(sel); - if (micToggle) { - this._logger.info(`Mic toggle found via fallback: ${sel}`); - break; - } - } - } - - if (!micToggle) { - this._logger.warn('Mic toggle not found on pre-join screen'); - return; - } + if (!micToggle) return; const state = await micToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', title: el.getAttribute('title') || '', })); - this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); if (!state.checked) { await micToggle.click(); this._logger.info('Mic toggled ON'); - await this._page!.waitForTimeout(1000); - - const afterState = await micToggle.evaluate((el: HTMLInputElement) => ({ - checked: el.checked, - dataCid: el.getAttribute('data-cid') || '', - })); - this._logger.info(`Mic after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`); } else { this._logger.info('Mic already ON'); }