PowerOn/service-teams-browser-bot
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: central _pollForElement utility (500ms interval), remove all fixed waits and snapshot checks

Browse source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG 2026-02-18 21:41:48 +01:00
parent a981ae6bd8
commit dddf1cd970
1 changed files with 101 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

286
src/bot/orchestrator.ts
View file

@ -1,4 +1,4 @@
import { Browser, BrowserContext, Page, chromium } from 'playwright'; import { Browser, BrowserContext, Page, ElementHandle, chromium } from 'playwright';
import { Logger } from 'winston'; import { Logger } from 'winston';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import path from 'path'; import path from 'path';
@ -86,6 +86,37 @@ export class BotOrchestrator {
this._logger = createSessionLogger(sessionId); this._logger = createSessionLogger(sessionId);
} }
/**
* Poll for a DOM element matching any of the given selectors.
* Checks every 500ms until found or timeout is reached.
* Returns the element handle if found, or null on timeout.
*/
private async _pollForElement(
selectors: string | string[],
timeoutMs: number = 15000,
label?: string,
): Promise<ElementHandle | null> {
const selectorList = Array.isArray(selectors) ? selectors : [selectors];
const combined = selectorList.join(', ');
const tag = label || combined.substring(0, 60);
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
for (const selector of selectorList) {
try {
const el = await this._page!.$(selector);
if (el) {
this._logger.info(`[poll] Found "${tag}" via: ${selector}`);
return el;
}
} catch { /* page navigated or element detached — retry */ }
}
await this._page!.waitForTimeout(500);
}
this._logger.warn(`[poll] "${tag}" not found within ${timeoutMs}ms`);
return null;
}
get sessionId(): string { get sessionId(): string {
return this._sessionId; return this._sessionId;
} }
@ -180,11 +211,12 @@ export class BotOrchestrator {
/** /**
* Join a meeting as authenticated user (System Bot or User Account). * Join a meeting as authenticated user (System Bot or User Account).
* Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now * Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now
*
* Every UI step uses _pollForElement (500ms interval) for both stability and performance:
* no fixed waits, the flow proceeds as soon as each element appears.
*/ */
private async _attemptAuthJoin(): Promise<void> { private async _attemptAuthJoin(): Promise<void> {
// Launch browser in headful mode with minimal args (Chromium Minimal)
await this._launchBrowser(true); await this._launchBrowser(true);
this._setState('navigating'); this._setState('navigating');
// STEP 1: Navigate to teams.microsoft.com to trigger authentication // STEP 1: Navigate to teams.microsoft.com to trigger authentication
@ -194,21 +226,14 @@ export class BotOrchestrator {
timeout: 30000, timeout: 30000,
}); });
// Wait for login redirect // Poll for login page (redirect to login.microsoftonline.com)
try { const emailInput = await this._pollForElement(
await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 }); ['input[name="loginfmt"]', 'input[type="email"]'],
this._logger.info('Redirected to MS login page'); 30000,
} catch { 'MS login email input',
this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`); );
} if (!emailInput) {
this._logger.warn(`No login page found, current URL: ${this._page!.url().substring(0, 150)}`);
// Wait for login page to render
try {
await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', {
timeout: 15000, state: 'visible',
});
} catch {
this._logger.warn('Login page elements not found');
} }
// STEP 2: Microsoft Authentication // STEP 2: Microsoft Authentication
@ -228,35 +253,23 @@ export class BotOrchestrator {
// STEP 3: Wait for Teams to finish loading after auth // STEP 3: Wait for Teams to finish loading after auth
this._logger.info('Waiting for Teams to load after auth...'); this._logger.info('Waiting for Teams to load after auth...');
try { try {
await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 }); await this._page!.waitForURL(
} catch { (url) => url.hostname.includes('teams.microsoft.com') || url.hostname.includes('teams.cloud.microsoft'),
try { { timeout: 30000 },
await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 }); );
} catch { } catch {
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`); this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
} }
}
// Give Teams a moment to initialize (session cookies, service workers)
await this._page!.waitForTimeout(3000);
// STEP 4: Navigate directly to the meeting URL (authenticated session) // STEP 4: Navigate to the meeting URL
// This is the key step: the previous flow waited for a "Join" button in
// the Teams chat header, which only works if the meeting chat happens to
// be visible. Navigating directly to the meeting URL reliably shows the
// pre-join screen for authenticated users.
this._logger.info(`Auth join: navigating to meeting URL: ${this._meetingUrl.substring(0, 80)}...`); this._logger.info(`Auth join: navigating to meeting URL: ${this._meetingUrl.substring(0, 80)}...`);
await this._page!.goto(this._meetingUrl, { await this._page!.goto(this._meetingUrl, {
waitUntil: 'domcontentloaded', waitUntil: 'domcontentloaded',
timeout: 30000, timeout: 30000,
}); });
this._logger.info(`Auth join: URL after navigation: ${this._page!.url().substring(0, 150)}`);
// Teams may show interstitials before the pre-join screen. // STEP 4a: Poll for the first actionable button (interstitial OR pre-join)
// Poll for ALL possible next-step buttons simultaneously so we don't
// miss one if it appears slowly.
const currentUrl = this._page!.url();
this._logger.info(`Auth join: URL after meeting navigation: ${currentUrl.substring(0, 150)}`);
// Race: wait for whichever button appears first (interstitial OR pre-join)
const interstitialSelectors = [ const interstitialSelectors = [
'button:has-text("Continue on this browser")', 'button:has-text("Continue on this browser")',
'button:has-text("In diesem Browser fortfahren")', 'button:has-text("In diesem Browser fortfahren")',
@ -269,95 +282,50 @@ export class BotOrchestrator {
'button:has-text("Jetzt teilnehmen")', 'button:has-text("Jetzt teilnehmen")',
'button[data-tid="prejoin-join-button"]', 'button[data-tid="prejoin-join-button"]',
]; ];
const allSelectors = [...interstitialSelectors, ...preJoinSelectors].join(', ');
// Poll up to 30s for ANY of these buttons const firstBtn = await this._pollForElement(
this._logger.info('Waiting for interstitial or pre-join button...'); [...interstitialSelectors, ...preJoinSelectors],
try { 30000,
const firstBtn = await this._page!.waitForSelector(allSelectors, { 'interstitial or pre-join button',
timeout: 30000, state: 'visible',
});
if (firstBtn) {
const btnText = await firstBtn.textContent().catch(() => '');
const btnTid = await firstBtn.getAttribute('data-tid').catch(() => '');
this._logger.info(`First visible button: "${btnText?.trim()}" (data-tid="${btnTid}")`);
// If it's an interstitial button (not pre-join), click it and wait for the next screen
const isPreJoin = preJoinSelectors.some(s =>
s.includes(btnTid || '__none__') || (btnText && s.includes(btnText.trim()))
); );
if (firstBtn) {
const btnText = (await firstBtn.textContent().catch(() => ''))?.trim() || '';
const btnTid = (await firstBtn.getAttribute('data-tid').catch(() => '')) || '';
const isPreJoin = btnTid === 'prejoin-join-button'
|| btnText.toLowerCase().includes('join now')
|| btnText.toLowerCase().includes('jetzt teilnehmen');
if (!isPreJoin) { if (!isPreJoin) {
await firstBtn.click(); await firstBtn.click();
this._logger.info('Clicked interstitial button, waiting for pre-join screen...'); this._logger.info(`Clicked interstitial: "${btnText}"`);
await this._page!.waitForTimeout(3000);
} }
} } else {
} catch {
this._logger.warn('No interstitial or pre-join button found within 30s');
await this._takeScreenshot('auth-no-buttons'); await this._takeScreenshot('auth-no-buttons');
} }
// STEP 5: Pre-Join screen → Click "Join now" // STEP 5: Poll for "Join now" on the pre-join screen
this._logger.info('Waiting for pre-join screen');
try {
await this._page!.waitForSelector(
'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]',
{ timeout: 30000, state: 'visible' },
);
} catch {
this._logger.warn('"Join now" button not found');
await this._takeScreenshot('auth-no-join-now');
}
this._logger.info('Camera left OFF (video disabled for stability)');
// Ensure microphone is ON (required for voice playback)
await this._ensureMicOn(); await this._ensureMicOn();
await this._page!.waitForTimeout(2000); const joinNowBtn = await this._pollForElement(preJoinSelectors, 30000, 'Join now button');
if (!joinNowBtn) {
const joinNowSelectors = [ await this._takeScreenshot('auth-no-join-now');
'button:has-text("Join now")',
'button:has-text("Jetzt teilnehmen")',
'button[data-tid="prejoin-join-button"]',
];
let joinNowClicked = false;
for (const selector of joinNowSelectors) {
try {
const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' });
if (btn) {
await btn.click();
joinNowClicked = true;
break;
}
} catch { /* try next */ }
}
if (!joinNowClicked) {
await this._takeScreenshot('auth-no-join-now-final');
throw new Error('"Join now" button not found on pre-join screen'); throw new Error('"Join now" button not found on pre-join screen');
} }
await joinNowBtn.click();
this._logger.info('Clicked "Join now", waiting for meeting'); this._logger.info('Clicked "Join now", waiting for meeting');
// Wait for meeting admission (hangup button = in meeting) // STEP 6: Wait for meeting admission (hangup button = in meeting)
await this._waitForMeetingAdmission(); await this._waitForMeetingAdmission();
this._setState('in_meeting'); this._setState('in_meeting');
this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`); this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`);
// Start keepalive to prevent idle disconnect
this._startKeepAlive(); this._startKeepAlive();
// Initialize audio playback
await this._audioProcedure!.initialize(); await this._audioProcedure!.initialize();
// Enable transcript capture (captions or audio based on transferMode)
await this._enableTranscriptCapture(); await this._enableTranscriptCapture();
await this._enableChat(); await this._enableChat();
// Send greeting in meeting chat
await this._sendJoinGreeting(); await this._sendJoinGreeting();
} }
@ -372,52 +340,26 @@ export class BotOrchestrator {
*/ */
private async _ensureCameraOn(): Promise<void> { private async _ensureCameraOn(): Promise<void> {
try { try {
// Primary: the actual switch input (fui-Switch) const cameraToggle = await this._pollForElement([
let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]'); 'input[data-tid="toggle-video"]',
if (!cameraToggle) {
this._logger.info('Primary camera selector not found, trying fallbacks...');
const fallbacks = [
'[data-tid="toggle-video"]', '[data-tid="toggle-video"]',
'input[role="switch"][title*="camera" i]', 'input[role="switch"][title*="camera" i]',
'input[role="switch"][title*="Camera" i]', 'input[role="switch"][title*="Camera" i]',
'input[role="switch"][title*="Video" i]', 'input[role="switch"][title*="Video" i]',
]; ], 10000, 'camera toggle (pre-join)');
for (const sel of fallbacks) {
cameraToggle = await this._page!.$(sel);
if (cameraToggle) {
this._logger.info(`Camera toggle found via fallback: ${sel}`);
break;
}
}
}
if (!cameraToggle) { if (!cameraToggle) return;
this._logger.warn('Camera toggle not found on pre-join screen');
return;
}
// Read current state
const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({ const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked, checked: el.checked,
dataCid: el.getAttribute('data-cid') || '', dataCid: el.getAttribute('data-cid') || '',
title: el.getAttribute('title') || '', title: el.getAttribute('title') || '',
})); }));
this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
if (!state.checked) { if (!state.checked) {
// Camera is OFF — click to turn ON
await cameraToggle.click(); await cameraToggle.click();
this._logger.info('Camera toggled ON'); this._logger.info('Camera toggled ON');
await this._page!.waitForTimeout(2000);
// Verify
const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
}));
this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
} else { } else {
this._logger.info('Camera already ON'); this._logger.info('Camera already ON');
} }
@ -435,48 +377,22 @@ export class BotOrchestrator {
*/ */
private async _ensureCameraOnInMeeting(): Promise<void> { private async _ensureCameraOnInMeeting(): Promise<void> {
try { try {
// Wait a moment for meeting controls to render const videoBtn = await this._pollForElement([
await this._page!.waitForTimeout(2000); 'button#video-button',
// Find the in-meeting video button
let videoBtn = await this._page!.$('button#video-button');
if (!videoBtn) {
// Fallback selectors
const fallbacks = [
'button[data-inp="video-button"]', 'button[data-inp="video-button"]',
'button[id="video-button"]',
'button[aria-label*="camera" i]', 'button[aria-label*="camera" i]',
'button[aria-label*="Camera" i]', 'button[aria-label*="Camera" i]',
'button[aria-label*="Video" i]', 'button[aria-label*="Video" i]',
]; ], 10000, 'in-meeting camera button');
for (const sel of fallbacks) {
videoBtn = await this._page!.$(sel);
if (videoBtn) {
this._logger.info(`In-meeting video button found via fallback: ${sel}`);
break;
}
}
}
if (!videoBtn) { if (!videoBtn) return;
this._logger.warn('In-meeting video button not found');
return;
}
// Read current state const state = await videoBtn.evaluate((el: HTMLElement) => ({
const state = await videoBtn.evaluate((el) => ({
dataState: el.getAttribute('data-state') || '', dataState: el.getAttribute('data-state') || '',
ariaLabel: el.getAttribute('aria-label') || '', ariaLabel: el.getAttribute('aria-label') || '',
id: el.id,
})); }));
this._logger.info(`In-meeting camera: data-state="${state.dataState}", aria-label="${state.ariaLabel}"`);
this._logger.info(
`In-meeting camera: data-state="${state.dataState}", ` +
`aria-label="${state.ariaLabel}", id="${state.id}"`,
);
// Camera is off if data-state is "call-video-off"
const isOff = state.dataState === 'call-video-off' const isOff = state.dataState === 'call-video-off'
|| state.ariaLabel.toLowerCase().includes('turn camera on') || state.ariaLabel.toLowerCase().includes('turn camera on')
|| state.ariaLabel.toLowerCase().includes('kamera einschalten'); || state.ariaLabel.toLowerCase().includes('kamera einschalten');
@ -484,17 +400,6 @@ export class BotOrchestrator {
if (isOff) { if (isOff) {
await videoBtn.click(); await videoBtn.click();
this._logger.info('In-meeting camera was OFF — clicked to turn ON'); this._logger.info('In-meeting camera was OFF — clicked to turn ON');
await this._page!.waitForTimeout(2000);
// Verify
const afterState = await videoBtn.evaluate((el) => ({
dataState: el.getAttribute('data-state') || '',
ariaLabel: el.getAttribute('aria-label') || '',
}));
this._logger.info(
`In-meeting camera after toggle: data-state="${afterState.dataState}", ` +
`aria-label="${afterState.ariaLabel}"`,
);
} else { } else {
this._logger.info('In-meeting camera already ON'); this._logger.info('In-meeting camera already ON');
} }
@ -514,48 +419,27 @@ export class BotOrchestrator {
*/ */
private async _ensureMicOn(): Promise<void> { private async _ensureMicOn(): Promise<void> {
try { try {
let micToggle = await this._page!.$('input[data-tid="toggle-audio"]'); const micToggle = await this._pollForElement([
'input[data-tid="toggle-audio"]',
if (!micToggle) {
const fallbacks = [
'[data-tid="toggle-audio"]', '[data-tid="toggle-audio"]',
'input[role="switch"][title*="microphone" i]', 'input[role="switch"][title*="microphone" i]',
'input[role="switch"][title*="Mikrofon" i]', 'input[role="switch"][title*="Mikrofon" i]',
'input[role="switch"][title*="mic" i]', 'input[role="switch"][title*="mic" i]',
'input[role="switch"][title*="audio" i]', 'input[role="switch"][title*="audio" i]',
]; ], 10000, 'mic toggle (pre-join)');
for (const sel of fallbacks) {
micToggle = await this._page!.$(sel);
if (micToggle) {
this._logger.info(`Mic toggle found via fallback: ${sel}`);
break;
}
}
}
if (!micToggle) { if (!micToggle) return;
this._logger.warn('Mic toggle not found on pre-join screen');
return;
}
const state = await micToggle.evaluate((el: HTMLInputElement) => ({ const state = await micToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked, checked: el.checked,
dataCid: el.getAttribute('data-cid') || '', dataCid: el.getAttribute('data-cid') || '',
title: el.getAttribute('title') || '', title: el.getAttribute('title') || '',
})); }));
this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
if (!state.checked) { if (!state.checked) {
await micToggle.click(); await micToggle.click();
this._logger.info('Mic toggled ON'); this._logger.info('Mic toggled ON');
await this._page!.waitForTimeout(1000);
const afterState = await micToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
}));
this._logger.info(`Mic after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
} else { } else {
this._logger.info('Mic already ON'); this._logger.info('Mic already ON');
} }