import { Browser, BrowserContext, Page, chromium } from 'playwright'; import { Logger } from 'winston'; import { v4 as uuidv4 } from 'uuid'; import path from 'path'; import fs from 'fs'; import os from 'os'; import WebSocket from 'ws'; import { config } from '../config'; import { createSessionLogger } from '../utils/logger'; import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage } from '../types'; import { JoinProcedure } from './joinProcedure'; import { CaptionsProcedure } from './captionsProcedure'; import { AudioProcedure } from './audioProcedure'; import { AudioCaptureProcedure } from './audioCaptureProcedure'; import { ChatProcedure, ChatMessageEntry } from './chatProcedure'; import { AuthProcedure } from './authProcedure'; import { isValidMeetingUrl } from './meetingUrlParser'; /** * Generate a solid-white Y4M video file for use as fake camera input. * Chromium loops this single frame at 30fps, so participants see a static white image. * Later this can be replaced with a custom image (avatar/background). */ function _generateFakeVideoFile(): string { const width = 1280; const height = 720; const filePath = path.join(os.tmpdir(), 'bot-video-white.y4m'); if (fs.existsSync(filePath)) return filePath; const header = `YUV4MPEG2 W${width} H${height} F30:1 Ip A0:0 C420jpeg\n`; const frameHeader = 'FRAME\n'; // White in YUV: Y=235, U=128, V=128 const yPlane = Buffer.alloc(width * height, 235); const uvSize = (width / 2) * (height / 2); const uPlane = Buffer.alloc(uvSize, 128); const vPlane = Buffer.alloc(uvSize, 128); const fd = fs.openSync(filePath, 'w'); fs.writeSync(fd, header); fs.writeSync(fd, frameHeader); fs.writeSync(fd, yPlane); fs.writeSync(fd, uPlane); fs.writeSync(fd, vPlane); fs.closeSync(fd); return filePath; } export interface OrchestratorCallbacks { onStateChange: (state: BotState, message?: string) => void; onTranscript: (entry: TranscriptEntry) => void; onError: (error: Error) => void; } export interface OrchestratorOptions { gatewayWsUrl: string; instanceId: string; language?: string; botAccountEmail?: string; botAccountPassword?: string; transferMode?: string; } /** * Orchestrates the entire bot lifecycle: * - Connects to Gateway via WebSocket * - Launches browser * - Joins meeting * - Enables captions * - Sends transcripts to Gateway * - Handles audio playback from Gateway * - Leaves meeting */ export class BotOrchestrator { private _sessionId: string; private _meetingUrl: string; private _botName: string; private _logger: Logger; private _callbacks: OrchestratorCallbacks; private _options: OrchestratorOptions; private _browser: Browser | null = null; private _context: BrowserContext | null = null; private _page: Page | null = null; private _gatewayWs: WebSocket | null = null; private _useHttpFallback: boolean = false; private _httpBaseUrl: string = ''; private _joinProcedure: JoinProcedure | null = null; private _captionsProcedure: CaptionsProcedure | null = null; private _audioProcedure: AudioProcedure | null = null; private _audioCaptureProcedure: AudioCaptureProcedure | null = null; private _chatProcedure: ChatProcedure | null = null; private _state: BotState = 'idle'; private _isShuttingDown: boolean = false; private _keepAliveInterval: NodeJS.Timeout | null = null; constructor( sessionId: string, meetingUrl: string, botName: string, callbacks: OrchestratorCallbacks, options: OrchestratorOptions ) { this._sessionId = sessionId; this._meetingUrl = meetingUrl; this._botName = botName || config.botName; this._callbacks = callbacks; this._options = options; this._logger = createSessionLogger(sessionId); } get sessionId(): string { return this._sessionId; } get state(): BotState { return this._state; } /** * Start the bot - connect to Gateway, launch browser, join meeting, enable captions. * Chooses between anonymous join and authenticated join based on credentials. */ async start(): Promise { if (!isValidMeetingUrl(this._meetingUrl)) { throw new Error(`Invalid meeting URL: ${this._meetingUrl}`); } try { this._setState('launching'); // Connect to Gateway WebSocket first await this._connectToGateway(); // Choose join method based on credentials const hasCredentials = !!(this._options.botAccountEmail && this._options.botAccountPassword); if (hasCredentials) { this._logger.info(`Authenticated join as: ${this._options.botAccountEmail}`); await this._attemptAuthJoin(); } else { this._logger.info('Anonymous join with bot name: ' + this._botName); await this._attemptJoin(); } } catch (error) { this._logger.error('Error starting bot:', error); this._setState('error', (error as Error).message); await this._takeScreenshot('error'); throw error; } } /** * Join a meeting as anonymous guest with the configured bot name. */ private async _attemptJoin(): Promise { // Launch browser await this._launchBrowser(); this._setState('navigating'); // STEP 1: Navigate to meeting URL and click "Continue on this browser" await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl); // STEP 2: Enter bot name and click "Join now" await this._joinProcedure!.joinMeetingLobbyFlow(); // Check if we're in lobby const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 10 }); if (inLobby) { this._setState('in_lobby'); this._logger.info('Bot is in lobby, waiting to be admitted...'); } // Wait to be admitted to the meeting await this._waitForMeetingAdmission(); this._setState('in_meeting'); this._logger.info(`Bot joined the meeting as "${this._botName}"`); // Start keepalive to prevent idle disconnect this._startKeepAlive(); // Dismiss any post-join permission modals (e.g. "Manage windows on all displays") await this._joinProcedure!.dismissBrowserPermissionModals(); // Verify camera is on in the meeting await this._ensureCameraOnInMeeting(); // Initialize audio playback await this._audioProcedure!.initialize(); // Enable transcript capture (captions or audio based on transferMode) await this._enableTranscriptCapture(); // Enable chat monitoring await this._enableChat(); } /** * Join a meeting as authenticated user (System Bot or User Account). * Flow: teams.microsoft.com → MS Login → Teams Chat → Join → Pre-Join → Join now */ private async _attemptAuthJoin(): Promise { // Launch browser in headful mode with minimal args (Chromium Minimal) await this._launchBrowser(true); this._setState('navigating'); // STEP 1: Navigate to teams.microsoft.com this._logger.info('Auth join: navigating to teams.microsoft.com'); await this._page!.goto('https://teams.microsoft.com', { waitUntil: 'domcontentloaded', timeout: 30000, }); // Wait for login redirect try { await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 }); this._logger.info('Redirected to MS login page'); } catch { this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`); } // Wait for login page to render try { await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', { timeout: 15000, state: 'visible', }); } catch { this._logger.warn('Login page elements not found'); } // STEP 2: Microsoft Authentication this._logger.info(`Authenticating as ${this._options.botAccountEmail}`); const authProcedure = new AuthProcedure(this._page!, this._logger); const authSuccess = await authProcedure.authenticateWithMicrosoft( this._options.botAccountEmail!, this._options.botAccountPassword!, true, ); if (!authSuccess) { throw new Error('Microsoft authentication failed'); } this._logger.info('Authentication successful'); // STEP 3: Wait for Teams chat page (landing page after auth) try { await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 }); } catch { // Also accept teams.cloud.microsoft try { await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 }); } catch { this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`); } } // Wait for "Join" button in chat header try { await this._page!.waitForSelector( 'button[data-tid="chat-join-button"], button[data-tid="join-call-button"]', { timeout: 30000, state: 'visible' }, ); this._logger.info('Teams chat page loaded, "Join" button found'); } catch { this._logger.warn('"Join" button not found in chat header'); await this._takeScreenshot('auth-no-join-button'); } // STEP 4: Click "Join" in chat header → Pre-Join screen this._logger.info('Clicking "Join" in chat header'); const chatJoinSelectors = [ 'button[data-tid="chat-join-button"]', 'button[data-tid="join-call-button"]', ]; let chatJoinClicked = false; for (const selector of chatJoinSelectors) { try { const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' }); if (btn) { await btn.click(); chatJoinClicked = true; break; } } catch { /* try next */ } } if (!chatJoinClicked) { throw new Error('"Join" button in chat header not found'); } // STEP 5: Pre-Join screen → Click "Join now" this._logger.info('Waiting for pre-join screen'); try { await this._page!.waitForSelector( 'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]', { timeout: 30000, state: 'visible' }, ); } catch { this._logger.warn('"Join now" button not found'); await this._takeScreenshot('auth-no-join-now'); } // Activate camera toggle if it's off (so background image is visible) await this._ensureCameraOn(); await this._page!.waitForTimeout(2000); const joinNowSelectors = [ 'button:has-text("Join now")', 'button:has-text("Jetzt teilnehmen")', 'button[data-tid="prejoin-join-button"]', ]; let joinNowClicked = false; for (const selector of joinNowSelectors) { try { const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' }); if (btn) { await btn.click(); joinNowClicked = true; break; } } catch { /* try next */ } } if (!joinNowClicked) { throw new Error('"Join now" button not found on pre-join screen'); } this._logger.info('Clicked "Join now", waiting for meeting'); // Wait for meeting admission (hangup button = in meeting) await this._waitForMeetingAdmission(); this._setState('in_meeting'); this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`); // Start keepalive to prevent idle disconnect this._startKeepAlive(); // Verify camera is on in the meeting await this._ensureCameraOnInMeeting(); // Initialize audio playback await this._audioProcedure!.initialize(); // Enable transcript capture (captions or audio based on transferMode) await this._enableTranscriptCapture(); await this._enableChat(); } /** * Ensure the camera is turned on in the pre-join screen. * When camera is on, Teams shows the profile/background image. * * Teams pre-join uses a fui-Switch input: * * - checked present = camera ON (data-cid="toggle-video-true", title="Turn camera off") * - checked absent = camera OFF (data-cid="toggle-video-false", title="Turn camera on") */ private async _ensureCameraOn(): Promise { try { // Primary: the actual switch input (fui-Switch) let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]'); if (!cameraToggle) { this._logger.info('Primary camera selector not found, trying fallbacks...'); const fallbacks = [ '[data-tid="toggle-video"]', 'input[role="switch"][title*="camera" i]', 'input[role="switch"][title*="Camera" i]', 'input[role="switch"][title*="Video" i]', ]; for (const sel of fallbacks) { cameraToggle = await this._page!.$(sel); if (cameraToggle) { this._logger.info(`Camera toggle found via fallback: ${sel}`); break; } } } if (!cameraToggle) { this._logger.warn('Camera toggle not found on pre-join screen'); return; } // Read current state const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', title: el.getAttribute('title') || '', })); this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); if (!state.checked) { // Camera is OFF — click to turn ON await cameraToggle.click(); this._logger.info('Camera toggled ON'); await this._page!.waitForTimeout(2000); // Verify const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', })); this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`); } else { this._logger.info('Camera already ON'); } } catch (err) { this._logger.warn(`Could not toggle camera: ${err}`); } } /** * Verify camera is on after joining the meeting, and turn it on if not. * * In-meeting camera button (from Teams DOM): *