refactor: central _pollForElement utility (500ms interval), remove all fixed waits and snapshot checks
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG
parent
a981ae6bd8
commit
dddf1cd970
1 changed files with 101 additions and 217 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import { Browser, BrowserContext, Page, chromium } from 'playwright';
|
||||
import { Browser, BrowserContext, Page, ElementHandle, chromium } from 'playwright';
|
||||
import { Logger } from 'winston';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import path from 'path';
|
||||
|
|
@ -86,6 +86,37 @@ export class BotOrchestrator {
|
|||
this._logger = createSessionLogger(sessionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll for a DOM element matching any of the given selectors.
|
||||
* Checks every 500ms until found or timeout is reached.
|
||||
* Returns the element handle if found, or null on timeout.
|
||||
*/
|
||||
private async _pollForElement(
|
||||
selectors: string | string[],
|
||||
timeoutMs: number = 15000,
|
||||
label?: string,
|
||||
): Promise<ElementHandle | null> {
|
||||
const selectorList = Array.isArray(selectors) ? selectors : [selectors];
|
||||
const combined = selectorList.join(', ');
|
||||
const tag = label || combined.substring(0, 60);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
for (const selector of selectorList) {
|
||||
try {
|
||||
const el = await this._page!.$(selector);
|
||||
if (el) {
|
||||
this._logger.info(`[poll] Found "${tag}" via: ${selector}`);
|
||||
return el;
|
||||
}
|
||||
} catch { /* page navigated or element detached — retry */ }
|
||||
}
|
||||
await this._page!.waitForTimeout(500);
|
||||
}
|
||||
this._logger.warn(`[poll] "${tag}" not found within ${timeoutMs}ms`);
|
||||
return null;
|
||||
}
|
||||
|
||||
get sessionId(): string {
|
||||
return this._sessionId;
|
||||
}
|
||||
|
|
@ -180,11 +211,12 @@ export class BotOrchestrator {
|
|||
/**
|
||||
* Join a meeting as authenticated user (System Bot or User Account).
|
||||
* Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now
|
||||
*
|
||||
* Every UI step uses _pollForElement (500ms interval) for both stability and performance:
|
||||
* no fixed waits, the flow proceeds as soon as each element appears.
|
||||
*/
|
||||
private async _attemptAuthJoin(): Promise<void> {
|
||||
// Launch browser in headful mode with minimal args (Chromium Minimal)
|
||||
await this._launchBrowser(true);
|
||||
|
||||
this._setState('navigating');
|
||||
|
||||
// STEP 1: Navigate to teams.microsoft.com to trigger authentication
|
||||
|
|
@ -194,21 +226,14 @@ export class BotOrchestrator {
|
|||
timeout: 30000,
|
||||
});
|
||||
|
||||
// Wait for login redirect
|
||||
try {
|
||||
await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 });
|
||||
this._logger.info('Redirected to MS login page');
|
||||
} catch {
|
||||
this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`);
|
||||
}
|
||||
|
||||
// Wait for login page to render
|
||||
try {
|
||||
await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', {
|
||||
timeout: 15000, state: 'visible',
|
||||
});
|
||||
} catch {
|
||||
this._logger.warn('Login page elements not found');
|
||||
// Poll for login page (redirect to login.microsoftonline.com)
|
||||
const emailInput = await this._pollForElement(
|
||||
['input[name="loginfmt"]', 'input[type="email"]'],
|
||||
30000,
|
||||
'MS login email input',
|
||||
);
|
||||
if (!emailInput) {
|
||||
this._logger.warn(`No login page found, current URL: ${this._page!.url().substring(0, 150)}`);
|
||||
}
|
||||
|
||||
// STEP 2: Microsoft Authentication
|
||||
|
|
@ -228,35 +253,23 @@ export class BotOrchestrator {
|
|||
// STEP 3: Wait for Teams to finish loading after auth
|
||||
this._logger.info('Waiting for Teams to load after auth...');
|
||||
try {
|
||||
await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 });
|
||||
await this._page!.waitForURL(
|
||||
(url) => url.hostname.includes('teams.microsoft.com') || url.hostname.includes('teams.cloud.microsoft'),
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
} catch {
|
||||
try {
|
||||
await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 });
|
||||
} catch {
|
||||
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
|
||||
}
|
||||
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
|
||||
}
|
||||
// Give Teams a moment to initialize (session cookies, service workers)
|
||||
await this._page!.waitForTimeout(3000);
|
||||
|
||||
// STEP 4: Navigate directly to the meeting URL (authenticated session)
|
||||
// This is the key step: the previous flow waited for a "Join" button in
|
||||
// the Teams chat header, which only works if the meeting chat happens to
|
||||
// be visible. Navigating directly to the meeting URL reliably shows the
|
||||
// pre-join screen for authenticated users.
|
||||
// STEP 4: Navigate to the meeting URL
|
||||
this._logger.info(`Auth join: navigating to meeting URL: ${this._meetingUrl.substring(0, 80)}...`);
|
||||
await this._page!.goto(this._meetingUrl, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30000,
|
||||
});
|
||||
this._logger.info(`Auth join: URL after navigation: ${this._page!.url().substring(0, 150)}`);
|
||||
|
||||
// Teams may show interstitials before the pre-join screen.
|
||||
// Poll for ALL possible next-step buttons simultaneously so we don't
|
||||
// miss one if it appears slowly.
|
||||
const currentUrl = this._page!.url();
|
||||
this._logger.info(`Auth join: URL after meeting navigation: ${currentUrl.substring(0, 150)}`);
|
||||
|
||||
// Race: wait for whichever button appears first (interstitial OR pre-join)
|
||||
// STEP 4a: Poll for the first actionable button (interstitial OR pre-join)
|
||||
const interstitialSelectors = [
|
||||
'button:has-text("Continue on this browser")',
|
||||
'button:has-text("In diesem Browser fortfahren")',
|
||||
|
|
@ -269,95 +282,50 @@ export class BotOrchestrator {
|
|||
'button:has-text("Jetzt teilnehmen")',
|
||||
'button[data-tid="prejoin-join-button"]',
|
||||
];
|
||||
const allSelectors = [...interstitialSelectors, ...preJoinSelectors].join(', ');
|
||||
|
||||
// Poll up to 30s for ANY of these buttons
|
||||
this._logger.info('Waiting for interstitial or pre-join button...');
|
||||
try {
|
||||
const firstBtn = await this._page!.waitForSelector(allSelectors, {
|
||||
timeout: 30000, state: 'visible',
|
||||
});
|
||||
if (firstBtn) {
|
||||
const btnText = await firstBtn.textContent().catch(() => '');
|
||||
const btnTid = await firstBtn.getAttribute('data-tid').catch(() => '');
|
||||
this._logger.info(`First visible button: "${btnText?.trim()}" (data-tid="${btnTid}")`);
|
||||
const firstBtn = await this._pollForElement(
|
||||
[...interstitialSelectors, ...preJoinSelectors],
|
||||
30000,
|
||||
'interstitial or pre-join button',
|
||||
);
|
||||
|
||||
// If it's an interstitial button (not pre-join), click it and wait for the next screen
|
||||
const isPreJoin = preJoinSelectors.some(s =>
|
||||
s.includes(btnTid || '__none__') || (btnText && s.includes(btnText.trim()))
|
||||
);
|
||||
if (!isPreJoin) {
|
||||
await firstBtn.click();
|
||||
this._logger.info('Clicked interstitial button, waiting for pre-join screen...');
|
||||
await this._page!.waitForTimeout(3000);
|
||||
}
|
||||
if (firstBtn) {
|
||||
const btnText = (await firstBtn.textContent().catch(() => ''))?.trim() || '';
|
||||
const btnTid = (await firstBtn.getAttribute('data-tid').catch(() => '')) || '';
|
||||
|
||||
const isPreJoin = btnTid === 'prejoin-join-button'
|
||||
|| btnText.toLowerCase().includes('join now')
|
||||
|| btnText.toLowerCase().includes('jetzt teilnehmen');
|
||||
|
||||
if (!isPreJoin) {
|
||||
await firstBtn.click();
|
||||
this._logger.info(`Clicked interstitial: "${btnText}"`);
|
||||
}
|
||||
} catch {
|
||||
this._logger.warn('No interstitial or pre-join button found within 30s');
|
||||
} else {
|
||||
await this._takeScreenshot('auth-no-buttons');
|
||||
}
|
||||
|
||||
// STEP 5: Pre-Join screen → Click "Join now"
|
||||
this._logger.info('Waiting for pre-join screen');
|
||||
try {
|
||||
await this._page!.waitForSelector(
|
||||
'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]',
|
||||
{ timeout: 30000, state: 'visible' },
|
||||
);
|
||||
} catch {
|
||||
this._logger.warn('"Join now" button not found');
|
||||
await this._takeScreenshot('auth-no-join-now');
|
||||
}
|
||||
|
||||
this._logger.info('Camera left OFF (video disabled for stability)');
|
||||
|
||||
// Ensure microphone is ON (required for voice playback)
|
||||
// STEP 5: Poll for "Join now" on the pre-join screen
|
||||
await this._ensureMicOn();
|
||||
|
||||
await this._page!.waitForTimeout(2000);
|
||||
|
||||
const joinNowSelectors = [
|
||||
'button:has-text("Join now")',
|
||||
'button:has-text("Jetzt teilnehmen")',
|
||||
'button[data-tid="prejoin-join-button"]',
|
||||
];
|
||||
|
||||
let joinNowClicked = false;
|
||||
for (const selector of joinNowSelectors) {
|
||||
try {
|
||||
const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' });
|
||||
if (btn) {
|
||||
await btn.click();
|
||||
joinNowClicked = true;
|
||||
break;
|
||||
}
|
||||
} catch { /* try next */ }
|
||||
}
|
||||
|
||||
if (!joinNowClicked) {
|
||||
await this._takeScreenshot('auth-no-join-now-final');
|
||||
const joinNowBtn = await this._pollForElement(preJoinSelectors, 30000, 'Join now button');
|
||||
if (!joinNowBtn) {
|
||||
await this._takeScreenshot('auth-no-join-now');
|
||||
throw new Error('"Join now" button not found on pre-join screen');
|
||||
}
|
||||
|
||||
await joinNowBtn.click();
|
||||
this._logger.info('Clicked "Join now", waiting for meeting');
|
||||
|
||||
// Wait for meeting admission (hangup button = in meeting)
|
||||
// STEP 6: Wait for meeting admission (hangup button = in meeting)
|
||||
await this._waitForMeetingAdmission();
|
||||
|
||||
this._setState('in_meeting');
|
||||
this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`);
|
||||
|
||||
// Start keepalive to prevent idle disconnect
|
||||
this._startKeepAlive();
|
||||
|
||||
// Initialize audio playback
|
||||
await this._audioProcedure!.initialize();
|
||||
|
||||
// Enable transcript capture (captions or audio based on transferMode)
|
||||
await this._enableTranscriptCapture();
|
||||
await this._enableChat();
|
||||
|
||||
// Send greeting in meeting chat
|
||||
await this._sendJoinGreeting();
|
||||
}
|
||||
|
||||
|
|
@ -372,52 +340,26 @@ export class BotOrchestrator {
|
|||
*/
|
||||
private async _ensureCameraOn(): Promise<void> {
|
||||
try {
|
||||
// Primary: the actual switch input (fui-Switch)
|
||||
let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]');
|
||||
const cameraToggle = await this._pollForElement([
|
||||
'input[data-tid="toggle-video"]',
|
||||
'[data-tid="toggle-video"]',
|
||||
'input[role="switch"][title*="camera" i]',
|
||||
'input[role="switch"][title*="Camera" i]',
|
||||
'input[role="switch"][title*="Video" i]',
|
||||
], 10000, 'camera toggle (pre-join)');
|
||||
|
||||
if (!cameraToggle) {
|
||||
this._logger.info('Primary camera selector not found, trying fallbacks...');
|
||||
const fallbacks = [
|
||||
'[data-tid="toggle-video"]',
|
||||
'input[role="switch"][title*="camera" i]',
|
||||
'input[role="switch"][title*="Camera" i]',
|
||||
'input[role="switch"][title*="Video" i]',
|
||||
];
|
||||
for (const sel of fallbacks) {
|
||||
cameraToggle = await this._page!.$(sel);
|
||||
if (cameraToggle) {
|
||||
this._logger.info(`Camera toggle found via fallback: ${sel}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!cameraToggle) return;
|
||||
|
||||
if (!cameraToggle) {
|
||||
this._logger.warn('Camera toggle not found on pre-join screen');
|
||||
return;
|
||||
}
|
||||
|
||||
// Read current state
|
||||
const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({
|
||||
checked: el.checked,
|
||||
dataCid: el.getAttribute('data-cid') || '',
|
||||
title: el.getAttribute('title') || '',
|
||||
}));
|
||||
|
||||
this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
|
||||
|
||||
if (!state.checked) {
|
||||
// Camera is OFF — click to turn ON
|
||||
await cameraToggle.click();
|
||||
this._logger.info('Camera toggled ON');
|
||||
await this._page!.waitForTimeout(2000);
|
||||
|
||||
// Verify
|
||||
const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({
|
||||
checked: el.checked,
|
||||
dataCid: el.getAttribute('data-cid') || '',
|
||||
}));
|
||||
this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
|
||||
} else {
|
||||
this._logger.info('Camera already ON');
|
||||
}
|
||||
|
|
@ -435,48 +377,22 @@ export class BotOrchestrator {
|
|||
*/
|
||||
private async _ensureCameraOnInMeeting(): Promise<void> {
|
||||
try {
|
||||
// Wait a moment for meeting controls to render
|
||||
await this._page!.waitForTimeout(2000);
|
||||
const videoBtn = await this._pollForElement([
|
||||
'button#video-button',
|
||||
'button[data-inp="video-button"]',
|
||||
'button[aria-label*="camera" i]',
|
||||
'button[aria-label*="Camera" i]',
|
||||
'button[aria-label*="Video" i]',
|
||||
], 10000, 'in-meeting camera button');
|
||||
|
||||
// Find the in-meeting video button
|
||||
let videoBtn = await this._page!.$('button#video-button');
|
||||
if (!videoBtn) return;
|
||||
|
||||
if (!videoBtn) {
|
||||
// Fallback selectors
|
||||
const fallbacks = [
|
||||
'button[data-inp="video-button"]',
|
||||
'button[id="video-button"]',
|
||||
'button[aria-label*="camera" i]',
|
||||
'button[aria-label*="Camera" i]',
|
||||
'button[aria-label*="Video" i]',
|
||||
];
|
||||
for (const sel of fallbacks) {
|
||||
videoBtn = await this._page!.$(sel);
|
||||
if (videoBtn) {
|
||||
this._logger.info(`In-meeting video button found via fallback: ${sel}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!videoBtn) {
|
||||
this._logger.warn('In-meeting video button not found');
|
||||
return;
|
||||
}
|
||||
|
||||
// Read current state
|
||||
const state = await videoBtn.evaluate((el) => ({
|
||||
const state = await videoBtn.evaluate((el: HTMLElement) => ({
|
||||
dataState: el.getAttribute('data-state') || '',
|
||||
ariaLabel: el.getAttribute('aria-label') || '',
|
||||
id: el.id,
|
||||
}));
|
||||
this._logger.info(`In-meeting camera: data-state="${state.dataState}", aria-label="${state.ariaLabel}"`);
|
||||
|
||||
this._logger.info(
|
||||
`In-meeting camera: data-state="${state.dataState}", ` +
|
||||
`aria-label="${state.ariaLabel}", id="${state.id}"`,
|
||||
);
|
||||
|
||||
// Camera is off if data-state is "call-video-off"
|
||||
const isOff = state.dataState === 'call-video-off'
|
||||
|| state.ariaLabel.toLowerCase().includes('turn camera on')
|
||||
|| state.ariaLabel.toLowerCase().includes('kamera einschalten');
|
||||
|
|
@ -484,17 +400,6 @@ export class BotOrchestrator {
|
|||
if (isOff) {
|
||||
await videoBtn.click();
|
||||
this._logger.info('In-meeting camera was OFF — clicked to turn ON');
|
||||
await this._page!.waitForTimeout(2000);
|
||||
|
||||
// Verify
|
||||
const afterState = await videoBtn.evaluate((el) => ({
|
||||
dataState: el.getAttribute('data-state') || '',
|
||||
ariaLabel: el.getAttribute('aria-label') || '',
|
||||
}));
|
||||
this._logger.info(
|
||||
`In-meeting camera after toggle: data-state="${afterState.dataState}", ` +
|
||||
`aria-label="${afterState.ariaLabel}"`,
|
||||
);
|
||||
} else {
|
||||
this._logger.info('In-meeting camera already ON');
|
||||
}
|
||||
|
|
@ -514,48 +419,27 @@ export class BotOrchestrator {
|
|||
*/
|
||||
private async _ensureMicOn(): Promise<void> {
|
||||
try {
|
||||
let micToggle = await this._page!.$('input[data-tid="toggle-audio"]');
|
||||
const micToggle = await this._pollForElement([
|
||||
'input[data-tid="toggle-audio"]',
|
||||
'[data-tid="toggle-audio"]',
|
||||
'input[role="switch"][title*="microphone" i]',
|
||||
'input[role="switch"][title*="Mikrofon" i]',
|
||||
'input[role="switch"][title*="mic" i]',
|
||||
'input[role="switch"][title*="audio" i]',
|
||||
], 10000, 'mic toggle (pre-join)');
|
||||
|
||||
if (!micToggle) {
|
||||
const fallbacks = [
|
||||
'[data-tid="toggle-audio"]',
|
||||
'input[role="switch"][title*="microphone" i]',
|
||||
'input[role="switch"][title*="Mikrofon" i]',
|
||||
'input[role="switch"][title*="mic" i]',
|
||||
'input[role="switch"][title*="audio" i]',
|
||||
];
|
||||
for (const sel of fallbacks) {
|
||||
micToggle = await this._page!.$(sel);
|
||||
if (micToggle) {
|
||||
this._logger.info(`Mic toggle found via fallback: ${sel}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!micToggle) {
|
||||
this._logger.warn('Mic toggle not found on pre-join screen');
|
||||
return;
|
||||
}
|
||||
if (!micToggle) return;
|
||||
|
||||
const state = await micToggle.evaluate((el: HTMLInputElement) => ({
|
||||
checked: el.checked,
|
||||
dataCid: el.getAttribute('data-cid') || '',
|
||||
title: el.getAttribute('title') || '',
|
||||
}));
|
||||
|
||||
this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
|
||||
|
||||
if (!state.checked) {
|
||||
await micToggle.click();
|
||||
this._logger.info('Mic toggled ON');
|
||||
await this._page!.waitForTimeout(1000);
|
||||
|
||||
const afterState = await micToggle.evaluate((el: HTMLInputElement) => ({
|
||||
checked: el.checked,
|
||||
dataCid: el.getAttribute('data-cid') || '',
|
||||
}));
|
||||
this._logger.info(`Mic after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
|
||||
} else {
|
||||
this._logger.info('Mic already ON');
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue