PowerOn/service-teams-browser-bot
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: central _pollForElement utility (500ms interval), remove all fixed waits and snapshot checks

Browse source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG 2026-02-18 21:41:48 +01:00
parent a981ae6bd8
commit dddf1cd970
1 changed files with 101 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

286
src/bot/orchestrator.ts
View file

@ -1,4 +1,4 @@
import { Browser, BrowserContext, Page, chromium } from 'playwright';
import { Browser, BrowserContext, Page, ElementHandle, chromium } from 'playwright';
import { Logger } from 'winston';
import { v4 as uuidv4 } from 'uuid';
import path from 'path';
@ -86,6 +86,37 @@ export class BotOrchestrator {
this._logger = createSessionLogger(sessionId);
}
/**
* Poll for a DOM element matching any of the given selectors.
* Checks every 500ms until found or timeout is reached.
* Returns the element handle if found, or null on timeout.
*/
private async _pollForElement(
selectors: string | string[],
timeoutMs: number = 15000,
label?: string,
): Promise<ElementHandle | null> {
const selectorList = Array.isArray(selectors) ? selectors : [selectors];
const combined = selectorList.join(', ');
const tag = label || combined.substring(0, 60);
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
for (const selector of selectorList) {
try {
const el = await this._page!.$(selector);
if (el) {
this._logger.info(`[poll] Found "${tag}" via: ${selector}`);
return el;
}
} catch { /* page navigated or element detached — retry */ }
}
await this._page!.waitForTimeout(500);
}
this._logger.warn(`[poll] "${tag}" not found within ${timeoutMs}ms`);
return null;
}
get sessionId(): string {
return this._sessionId;
}
@ -180,11 +211,12 @@ export class BotOrchestrator {
/**
* Join a meeting as authenticated user (System Bot or User Account).
* Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now
*
* Every UI step uses _pollForElement (500ms interval) for both stability and performance:
* no fixed waits, the flow proceeds as soon as each element appears.
*/
private async _attemptAuthJoin(): Promise<void> {
// Launch browser in headful mode with minimal args (Chromium Minimal)
await this._launchBrowser(true);
this._setState('navigating');
// STEP 1: Navigate to teams.microsoft.com to trigger authentication
@ -194,21 +226,14 @@ export class BotOrchestrator {
timeout: 30000,
});
// Wait for login redirect
try {
await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 });
this._logger.info('Redirected to MS login page');
} catch {
this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`);
}
// Wait for login page to render
try {
await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', {
timeout: 15000, state: 'visible',
});
} catch {
this._logger.warn('Login page elements not found');
// Poll for login page (redirect to login.microsoftonline.com)
const emailInput = await this._pollForElement(
['input[name="loginfmt"]', 'input[type="email"]'],
30000,
'MS login email input',
);
if (!emailInput) {
this._logger.warn(`No login page found, current URL: ${this._page!.url().substring(0, 150)}`);
}
// STEP 2: Microsoft Authentication
@ -228,35 +253,23 @@ export class BotOrchestrator {
// STEP 3: Wait for Teams to finish loading after auth
this._logger.info('Waiting for Teams to load after auth...');
try {
await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 });
} catch {
try {
await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 });
await this._page!.waitForURL(
(url) => url.hostname.includes('teams.microsoft.com') || url.hostname.includes('teams.cloud.microsoft'),
{ timeout: 30000 },
);
} catch {
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
}
}
// Give Teams a moment to initialize (session cookies, service workers)
await this._page!.waitForTimeout(3000);
// STEP 4: Navigate directly to the meeting URL (authenticated session)
// This is the key step: the previous flow waited for a "Join" button in
// the Teams chat header, which only works if the meeting chat happens to
// be visible. Navigating directly to the meeting URL reliably shows the
// pre-join screen for authenticated users.
// STEP 4: Navigate to the meeting URL
this._logger.info(`Auth join: navigating to meeting URL: ${this._meetingUrl.substring(0, 80)}...`);
await this._page!.goto(this._meetingUrl, {
waitUntil: 'domcontentloaded',
timeout: 30000,
});
this._logger.info(`Auth join: URL after navigation: ${this._page!.url().substring(0, 150)}`);
// Teams may show interstitials before the pre-join screen.
// Poll for ALL possible next-step buttons simultaneously so we don't
// miss one if it appears slowly.
const currentUrl = this._page!.url();
this._logger.info(`Auth join: URL after meeting navigation: ${currentUrl.substring(0, 150)}`);
// Race: wait for whichever button appears first (interstitial OR pre-join)
// STEP 4a: Poll for the first actionable button (interstitial OR pre-join)
const interstitialSelectors = [
'button:has-text("Continue on this browser")',
'button:has-text("In diesem Browser fortfahren")',
@ -269,95 +282,50 @@ export class BotOrchestrator {
'button:has-text("Jetzt teilnehmen")',
'button[data-tid="prejoin-join-button"]',
];
const allSelectors = [...interstitialSelectors, ...preJoinSelectors].join(', ');
// Poll up to 30s for ANY of these buttons
this._logger.info('Waiting for interstitial or pre-join button...');
try {
const firstBtn = await this._page!.waitForSelector(allSelectors, {
timeout: 30000, state: 'visible',
});
if (firstBtn) {
const btnText = await firstBtn.textContent().catch(() => '');
const btnTid = await firstBtn.getAttribute('data-tid').catch(() => '');
this._logger.info(`First visible button: "${btnText?.trim()}" (data-tid="${btnTid}")`);
// If it's an interstitial button (not pre-join), click it and wait for the next screen
const isPreJoin = preJoinSelectors.some(s =>
s.includes(btnTid || '__none__') || (btnText && s.includes(btnText.trim()))
const firstBtn = await this._pollForElement(
[...interstitialSelectors, ...preJoinSelectors],
30000,
'interstitial or pre-join button',
);
if (firstBtn) {
const btnText = (await firstBtn.textContent().catch(() => ''))?.trim() || '';
const btnTid = (await firstBtn.getAttribute('data-tid').catch(() => '')) || '';
const isPreJoin = btnTid === 'prejoin-join-button'
|| btnText.toLowerCase().includes('join now')
|| btnText.toLowerCase().includes('jetzt teilnehmen');
if (!isPreJoin) {
await firstBtn.click();
this._logger.info('Clicked interstitial button, waiting for pre-join screen...');
await this._page!.waitForTimeout(3000);
this._logger.info(`Clicked interstitial: "${btnText}"`);
}
}
} catch {
this._logger.warn('No interstitial or pre-join button found within 30s');
} else {
await this._takeScreenshot('auth-no-buttons');
}
// STEP 5: Pre-Join screen → Click "Join now"
this._logger.info('Waiting for pre-join screen');
try {
await this._page!.waitForSelector(
'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]',
{ timeout: 30000, state: 'visible' },
);
} catch {
this._logger.warn('"Join now" button not found');
await this._takeScreenshot('auth-no-join-now');
}
this._logger.info('Camera left OFF (video disabled for stability)');
// Ensure microphone is ON (required for voice playback)
// STEP 5: Poll for "Join now" on the pre-join screen
await this._ensureMicOn();
await this._page!.waitForTimeout(2000);
const joinNowSelectors = [
'button:has-text("Join now")',
'button:has-text("Jetzt teilnehmen")',
'button[data-tid="prejoin-join-button"]',
];
let joinNowClicked = false;
for (const selector of joinNowSelectors) {
try {
const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' });
if (btn) {
await btn.click();
joinNowClicked = true;
break;
}
} catch { /* try next */ }
}
if (!joinNowClicked) {
await this._takeScreenshot('auth-no-join-now-final');
const joinNowBtn = await this._pollForElement(preJoinSelectors, 30000, 'Join now button');
if (!joinNowBtn) {
await this._takeScreenshot('auth-no-join-now');
throw new Error('"Join now" button not found on pre-join screen');
}
await joinNowBtn.click();
this._logger.info('Clicked "Join now", waiting for meeting');
// Wait for meeting admission (hangup button = in meeting)
// STEP 6: Wait for meeting admission (hangup button = in meeting)
await this._waitForMeetingAdmission();
this._setState('in_meeting');
this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`);
// Start keepalive to prevent idle disconnect
this._startKeepAlive();
// Initialize audio playback
await this._audioProcedure!.initialize();
// Enable transcript capture (captions or audio based on transferMode)
await this._enableTranscriptCapture();
await this._enableChat();
// Send greeting in meeting chat
await this._sendJoinGreeting();
}
@ -372,52 +340,26 @@ export class BotOrchestrator {
*/
private async _ensureCameraOn(): Promise<void> {
try {
// Primary: the actual switch input (fui-Switch)
let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]');
if (!cameraToggle) {
this._logger.info('Primary camera selector not found, trying fallbacks...');
const fallbacks = [
const cameraToggle = await this._pollForElement([
'input[data-tid="toggle-video"]',
'[data-tid="toggle-video"]',
'input[role="switch"][title*="camera" i]',
'input[role="switch"][title*="Camera" i]',
'input[role="switch"][title*="Video" i]',
];
for (const sel of fallbacks) {
cameraToggle = await this._page!.$(sel);
if (cameraToggle) {
this._logger.info(`Camera toggle found via fallback: ${sel}`);
break;
}
}
}
], 10000, 'camera toggle (pre-join)');
if (!cameraToggle) {
this._logger.warn('Camera toggle not found on pre-join screen');
return;
}
if (!cameraToggle) return;
// Read current state
const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
title: el.getAttribute('title') || '',
}));
this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
if (!state.checked) {
// Camera is OFF — click to turn ON
await cameraToggle.click();
this._logger.info('Camera toggled ON');
await this._page!.waitForTimeout(2000);
// Verify
const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
}));
this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
} else {
this._logger.info('Camera already ON');
}
@ -435,48 +377,22 @@ export class BotOrchestrator {
*/
private async _ensureCameraOnInMeeting(): Promise<void> {
try {
// Wait a moment for meeting controls to render
await this._page!.waitForTimeout(2000);
// Find the in-meeting video button
let videoBtn = await this._page!.$('button#video-button');
if (!videoBtn) {
// Fallback selectors
const fallbacks = [
const videoBtn = await this._pollForElement([
'button#video-button',
'button[data-inp="video-button"]',
'button[id="video-button"]',
'button[aria-label*="camera" i]',
'button[aria-label*="Camera" i]',
'button[aria-label*="Video" i]',
];
for (const sel of fallbacks) {
videoBtn = await this._page!.$(sel);
if (videoBtn) {
this._logger.info(`In-meeting video button found via fallback: ${sel}`);
break;
}
}
}
], 10000, 'in-meeting camera button');
if (!videoBtn) {
this._logger.warn('In-meeting video button not found');
return;
}
if (!videoBtn) return;
// Read current state
const state = await videoBtn.evaluate((el) => ({
const state = await videoBtn.evaluate((el: HTMLElement) => ({
dataState: el.getAttribute('data-state') || '',
ariaLabel: el.getAttribute('aria-label') || '',
id: el.id,
}));
this._logger.info(`In-meeting camera: data-state="${state.dataState}", aria-label="${state.ariaLabel}"`);
this._logger.info(
`In-meeting camera: data-state="${state.dataState}", ` +
`aria-label="${state.ariaLabel}", id="${state.id}"`,
);
// Camera is off if data-state is "call-video-off"
const isOff = state.dataState === 'call-video-off'
|| state.ariaLabel.toLowerCase().includes('turn camera on')
|| state.ariaLabel.toLowerCase().includes('kamera einschalten');
@ -484,17 +400,6 @@ export class BotOrchestrator {
if (isOff) {
await videoBtn.click();
this._logger.info('In-meeting camera was OFF — clicked to turn ON');
await this._page!.waitForTimeout(2000);
// Verify
const afterState = await videoBtn.evaluate((el) => ({
dataState: el.getAttribute('data-state') || '',
ariaLabel: el.getAttribute('aria-label') || '',
}));
this._logger.info(
`In-meeting camera after toggle: data-state="${afterState.dataState}", ` +
`aria-label="${afterState.ariaLabel}"`,
);
} else {
this._logger.info('In-meeting camera already ON');
}
@ -514,48 +419,27 @@ export class BotOrchestrator {
*/
private async _ensureMicOn(): Promise<void> {
try {
let micToggle = await this._page!.$('input[data-tid="toggle-audio"]');
if (!micToggle) {
const fallbacks = [
const micToggle = await this._pollForElement([
'input[data-tid="toggle-audio"]',
'[data-tid="toggle-audio"]',
'input[role="switch"][title*="microphone" i]',
'input[role="switch"][title*="Mikrofon" i]',
'input[role="switch"][title*="mic" i]',
'input[role="switch"][title*="audio" i]',
];
for (const sel of fallbacks) {
micToggle = await this._page!.$(sel);
if (micToggle) {
this._logger.info(`Mic toggle found via fallback: ${sel}`);
break;
}
}
}
], 10000, 'mic toggle (pre-join)');
if (!micToggle) {
this._logger.warn('Mic toggle not found on pre-join screen');
return;
}
if (!micToggle) return;
const state = await micToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
title: el.getAttribute('title') || '',
}));
this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
if (!state.checked) {
await micToggle.click();
this._logger.info('Mic toggled ON');
await this._page!.waitForTimeout(1000);
const afterState = await micToggle.evaluate((el: HTMLInputElement) => ({
checked: el.checked,
dataCid: el.getAttribute('data-cid') || '',
}));
this._logger.info(`Mic after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
} else {
this._logger.info('Mic already ON');
}