import { Browser, BrowserContext, Page, chromium } from 'playwright'; import { Logger } from 'winston'; import { v4 as uuidv4 } from 'uuid'; import path from 'path'; import fs from 'fs'; import WebSocket from 'ws'; import { config } from '../config'; import { createSessionLogger } from '../utils/logger'; import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage } from '../types'; import { JoinProcedure } from './joinProcedure'; import { CaptionsProcedure } from './captionsProcedure'; import { AudioProcedure } from './audioProcedure'; import { AudioCaptureProcedure } from './audioCaptureProcedure'; import { ChatProcedure, ChatMessageEntry } from './chatProcedure'; import { AuthProcedure } from './authProcedure'; import { isValidMeetingUrl } from './meetingUrlParser'; export interface OrchestratorCallbacks { onStateChange: (state: BotState, message?: string) => void; onTranscript: (entry: TranscriptEntry) => void; onError: (error: Error) => void; } export interface OrchestratorOptions { gatewayWsUrl: string; instanceId: string; language?: string; botAccountEmail?: string; botAccountPassword?: string; transferMode?: string; } /** * Orchestrates the entire bot lifecycle: * - Connects to Gateway via WebSocket * - Launches browser * - Joins meeting * - Enables captions * - Sends transcripts to Gateway * - Handles audio playback from Gateway * - Leaves meeting */ export class BotOrchestrator { private _sessionId: string; private _meetingUrl: string; private _botName: string; private _logger: Logger; private _callbacks: OrchestratorCallbacks; private _options: OrchestratorOptions; private _browser: Browser | null = null; private _context: BrowserContext | null = null; private _page: Page | null = null; private _gatewayWs: WebSocket | null = null; private _useHttpFallback: boolean = false; private _httpBaseUrl: string = ''; private _joinProcedure: JoinProcedure | null = null; private _captionsProcedure: CaptionsProcedure | null = null; private _audioProcedure: AudioProcedure | null = null; private _audioCaptureProcedure: AudioCaptureProcedure | null = null; private _chatProcedure: ChatProcedure | null = null; private _state: BotState = 'idle'; private _isShuttingDown: boolean = false; private _keepAliveInterval: NodeJS.Timeout | null = null; constructor( sessionId: string, meetingUrl: string, botName: string, callbacks: OrchestratorCallbacks, options: OrchestratorOptions ) { this._sessionId = sessionId; this._meetingUrl = meetingUrl; this._botName = botName || config.botName; this._callbacks = callbacks; this._options = options; this._logger = createSessionLogger(sessionId); } get sessionId(): string { return this._sessionId; } get state(): BotState { return this._state; } /** * Start the bot - connect to Gateway, launch browser, join meeting, enable captions. * Chooses between anonymous join and authenticated join based on credentials. */ async start(): Promise { if (!isValidMeetingUrl(this._meetingUrl)) { throw new Error(`Invalid meeting URL: ${this._meetingUrl}`); } try { this._setState('launching'); // Connect to Gateway WebSocket first await this._connectToGateway(); // Choose join method based on credentials const hasCredentials = !!(this._options.botAccountEmail && this._options.botAccountPassword); if (hasCredentials) { this._logger.info(`Authenticated join as: ${this._options.botAccountEmail}`); await this._attemptAuthJoin(); } else { this._logger.info('Anonymous join with bot name: ' + this._botName); await this._attemptJoin(); } } catch (error) { this._logger.error('Error starting bot:', error); this._setState('error', (error as Error).message); await this._takeScreenshot('error'); throw error; } } /** * Join a meeting as anonymous guest with the configured bot name. */ private async _attemptJoin(): Promise { // Launch browser await this._launchBrowser(); this._setState('navigating'); // STEP 1: Navigate to meeting URL and click "Continue on this browser" await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl); // STEP 2: Enter bot name and click "Join now" await this._joinProcedure!.joinMeetingLobbyFlow(); // Check if we're in lobby const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 10 }); if (inLobby) { this._setState('in_lobby'); this._logger.info('Bot is in lobby, waiting to be admitted...'); } // Wait to be admitted to the meeting await this._waitForMeetingAdmission(); this._setState('in_meeting'); this._logger.info(`Bot joined the meeting as "${this._botName}"`); // Start keepalive to prevent idle disconnect this._startKeepAlive(); // Dismiss any post-join permission modals (e.g. "Manage windows on all displays") await this._joinProcedure!.dismissBrowserPermissionModals(); // Initialize audio playback await this._audioProcedure!.initialize(); // Enable transcript capture (captions or audio based on transferMode) await this._enableTranscriptCapture(); // Enable chat monitoring await this._enableChat(); } /** * Join a meeting as authenticated user (System Bot or User Account). * Flow: teams.microsoft.com → MS Login → Teams Chat → Join → Pre-Join → Join now */ private async _attemptAuthJoin(): Promise { // Launch browser in headful mode with minimal args (Chromium Minimal) await this._launchBrowser(true); this._setState('navigating'); // STEP 1: Navigate to teams.microsoft.com this._logger.info('Auth join: navigating to teams.microsoft.com'); await this._page!.goto('https://teams.microsoft.com', { waitUntil: 'domcontentloaded', timeout: 30000, }); // Wait for login redirect try { await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 }); this._logger.info('Redirected to MS login page'); } catch { this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`); } // Wait for login page to render try { await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', { timeout: 15000, state: 'visible', }); } catch { this._logger.warn('Login page elements not found'); } // STEP 2: Microsoft Authentication this._logger.info(`Authenticating as ${this._options.botAccountEmail}`); const authProcedure = new AuthProcedure(this._page!, this._logger); const authSuccess = await authProcedure.authenticateWithMicrosoft( this._options.botAccountEmail!, this._options.botAccountPassword!, true, ); if (!authSuccess) { throw new Error('Microsoft authentication failed'); } this._logger.info('Authentication successful'); // STEP 3: Wait for Teams chat page (landing page after auth) try { await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 }); } catch { // Also accept teams.cloud.microsoft try { await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 }); } catch { this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`); } } // Wait for "Join" button in chat header try { await this._page!.waitForSelector( 'button[data-tid="chat-join-button"], button[data-tid="join-call-button"]', { timeout: 30000, state: 'visible' }, ); this._logger.info('Teams chat page loaded, "Join" button found'); } catch { this._logger.warn('"Join" button not found in chat header'); await this._takeScreenshot('auth-no-join-button'); } // STEP 4: Click "Join" in chat header → Pre-Join screen this._logger.info('Clicking "Join" in chat header'); const chatJoinSelectors = [ 'button[data-tid="chat-join-button"]', 'button[data-tid="join-call-button"]', ]; let chatJoinClicked = false; for (const selector of chatJoinSelectors) { try { const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' }); if (btn) { await btn.click(); chatJoinClicked = true; break; } } catch { /* try next */ } } if (!chatJoinClicked) { throw new Error('"Join" button in chat header not found'); } // STEP 5: Pre-Join screen → Click "Join now" this._logger.info('Waiting for pre-join screen'); try { await this._page!.waitForSelector( 'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]', { timeout: 30000, state: 'visible' }, ); } catch { this._logger.warn('"Join now" button not found'); await this._takeScreenshot('auth-no-join-now'); } // Activate camera toggle if it's off (so background image is visible) await this._ensureCameraOn(); await this._page!.waitForTimeout(2000); const joinNowSelectors = [ 'button:has-text("Join now")', 'button:has-text("Jetzt teilnehmen")', 'button[data-tid="prejoin-join-button"]', ]; let joinNowClicked = false; for (const selector of joinNowSelectors) { try { const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' }); if (btn) { await btn.click(); joinNowClicked = true; break; } } catch { /* try next */ } } if (!joinNowClicked) { throw new Error('"Join now" button not found on pre-join screen'); } this._logger.info('Clicked "Join now", waiting for meeting'); // Wait for meeting admission (hangup button = in meeting) await this._waitForMeetingAdmission(); this._setState('in_meeting'); this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`); // Start keepalive to prevent idle disconnect this._startKeepAlive(); // Initialize audio playback await this._audioProcedure!.initialize(); // Enable transcript capture (captions or audio based on transferMode) await this._enableTranscriptCapture(); await this._enableChat(); } /** * Ensure the camera is turned on in the pre-join screen. * When camera is on, Teams shows the profile/background image. * * Teams pre-join uses a fui-Switch input: * * - checked present = camera ON (data-cid="toggle-video-true", title="Turn camera off") * - checked absent = camera OFF (data-cid="toggle-video-false", title="Turn camera on") */ private async _ensureCameraOn(): Promise { try { // Primary: the actual switch input (fui-Switch) let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]'); if (!cameraToggle) { this._logger.info('Primary camera selector not found, trying fallbacks...'); const fallbacks = [ '[data-tid="toggle-video"]', 'input[role="switch"][title*="camera" i]', 'input[role="switch"][title*="Camera" i]', 'input[role="switch"][title*="Video" i]', ]; for (const sel of fallbacks) { cameraToggle = await this._page!.$(sel); if (cameraToggle) { this._logger.info(`Camera toggle found via fallback: ${sel}`); break; } } } if (!cameraToggle) { this._logger.warn('Camera toggle not found on pre-join screen'); return; } // Read current state const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', title: el.getAttribute('title') || '', })); this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`); if (!state.checked) { // Camera is OFF — click to turn ON await cameraToggle.click(); this._logger.info('Camera toggled ON'); await this._page!.waitForTimeout(2000); // Verify const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({ checked: el.checked, dataCid: el.getAttribute('data-cid') || '', })); this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`); } else { this._logger.info('Camera already ON'); } } catch (err) { this._logger.warn(`Could not toggle camera: ${err}`); } } /** * Start a keepalive timer that periodically moves the mouse and sends * a WebSocket ping. Prevents Teams from detecting the bot as idle * and kicking it from the meeting. */ private _startKeepAlive(): void { if (this._keepAliveInterval) return; this._keepAliveInterval = setInterval(async () => { if (this._isShuttingDown || !this._page) return; try { // Small random mouse movement to simulate user activity const x = 640 + Math.floor(Math.random() * 20 - 10); const y = 360 + Math.floor(Math.random() * 20 - 10); await this._page.mouse.move(x, y); } catch { // Page might be closed } // WebSocket heartbeat if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) { try { this._gatewayWs.send(JSON.stringify({ type: 'ping', sessionId: this._sessionId })); } catch { // Connection might be closing } } }, 30000); this._logger.info('Keepalive started (30s interval)'); } /** * Stop the keepalive timer. */ private _stopKeepAlive(): void { if (this._keepAliveInterval) { clearInterval(this._keepAliveInterval); this._keepAliveInterval = null; this._logger.info('Keepalive stopped'); } } /** * Connect to the Gateway WebSocket for this session. */ private async _connectToGateway(): Promise { // gatewayWsUrl is the full WebSocket URL provided by the Gateway // It already includes instanceId and sessionId const wsUrl = this._options.gatewayWsUrl; this._logger.info(`Connecting to Gateway: ${wsUrl}`); // Derive HTTP base URL from WebSocket URL for fallback this._httpBaseUrl = wsUrl .replace('wss://', 'https://') .replace('ws://', 'http://') .replace(/\/bot\/ws\/.*$/, ''); return new Promise((resolve, reject) => { this._gatewayWs = new WebSocket(wsUrl); const wsTimeout = setTimeout(() => { if (this._gatewayWs?.readyState !== WebSocket.OPEN) { this._logger.warn('WebSocket connection timeout - switching to HTTP fallback'); this._useHttpFallback = true; this._gatewayWs?.close(); this._gatewayWs = null; resolve(); // Continue with HTTP fallback instead of failing } }, 10000); this._gatewayWs.on('open', () => { clearTimeout(wsTimeout); this._logger.info('Connected to Gateway via WebSocket'); this._useHttpFallback = false; resolve(); }); this._gatewayWs.on('message', (data) => { this._handleGatewayMessage(data.toString()); }); this._gatewayWs.on('close', (code, reason) => { this._logger.warn(`Gateway WebSocket closed: ${code} - ${reason}`); if (!this._isShuttingDown && !this._useHttpFallback) { this._logger.info('Switching to HTTP fallback for transcript delivery'); this._useHttpFallback = true; } }); this._gatewayWs.on('error', (error) => { clearTimeout(wsTimeout); this._logger.error('Gateway WebSocket error:', error); this._logger.info('Switching to HTTP fallback for transcript delivery'); this._useHttpFallback = true; this._gatewayWs = null; resolve(); // Continue with HTTP fallback }); }); } /** * Handle incoming messages from the Gateway. */ private _handleGatewayMessage(data: string): void { try { const message = JSON.parse(data); switch (message.type) { case 'playAudio': const audioMsg = message as PlayAudioMessage; this.playAudio(audioMsg.audio.data, audioMsg.audio.format); break; case 'sendChatMessage': const chatMsg = message as SendChatMessage; this.sendChatMessageToMeeting(chatMsg.text); break; case 'stopAudio': this._logger.info('Stop audio command received from Gateway'); if (this._audioProcedure) { this._audioProcedure.stopAllAudio(); } break; case 'pong': // Heartbeat response break; default: this._logger.debug('Unknown Gateway message type:', message.type); } } catch (error) { this._logger.error('Error parsing Gateway message:', error); } } /** * Send a message to the Gateway (WebSocket or HTTP fallback). */ private _sendToGateway(message: object): void { if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) { try { this._gatewayWs.send(JSON.stringify(message)); return; } catch (error) { this._logger.error('WebSocket send error, falling back to HTTP:', error); this._useHttpFallback = true; } } // HTTP fallback if (this._useHttpFallback) { this._sendViaHttp(message); } else { this._logger.warn('Cannot send to Gateway - no WebSocket and no HTTP fallback'); } } /** * Send a message via HTTP POST (fallback when WebSocket unavailable). */ private async _sendViaHttp(message: any): Promise { const msgType = message.type; let url = ''; if (msgType === 'transcript') { url = `${this._httpBaseUrl}/bot/transcript/${this._sessionId}`; } else if (msgType === 'status') { url = `${this._httpBaseUrl}/bot/status/${this._sessionId}`; } else if (msgType === 'audioChunk') { // Audio chunks are too frequent for HTTP — only send via WebSocket return; } else { this._logger.debug(`HTTP fallback: unsupported message type ${msgType}`); return; } try { const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(message), }); if (!response.ok) { this._logger.warn(`HTTP fallback response: ${response.status} ${response.statusText}`); } } catch (error) { this._logger.error(`HTTP fallback error for ${msgType}:`, error); } } /** * Send a transcript to the Gateway. */ private _sendTranscript(speaker: string, text: string, isFinal: boolean): void { const message: TranscriptMessage = { type: 'transcript', sessionId: this._sessionId, transcript: { speaker, text, timestamp: new Date().toISOString(), isFinal, }, }; this._sendToGateway(message); } /** * Send a status update to the Gateway. */ private _sendStatus(status: StatusMessage['status'], message?: string): void { const statusMessage: StatusMessage = { type: 'status', sessionId: this._sessionId, status, message, }; this._sendToGateway(statusMessage); } /** * Stop the bot - leave meeting, close browser, disconnect from Gateway. */ async stop(): Promise { if (this._isShuttingDown) { return; } this._isShuttingDown = true; this._logger.info('Stopping bot...'); // Stop keepalive first this._stopKeepAlive(); try { this._setState('leaving'); // Stop audio capture if (this._audioCaptureProcedure) { await this._audioCaptureProcedure.stopCapture(); } // Unsubscribe from captions and chat if (this._captionsProcedure) { await this._captionsProcedure.unsubscribe(); } if (this._chatProcedure) { await this._chatProcedure.unsubscribe(); } // Clean up audio playback if (this._audioProcedure) { await this._audioProcedure.cleanup(); } // Leave the meeting if (this._joinProcedure && this._state !== 'error') { await this._joinProcedure.leaveMeetingFlow(); } } catch (error) { this._logger.error('Error during shutdown:', error); } finally { // Close browser await this._closeBrowser(); // Close Gateway connection if (this._gatewayWs) { this._gatewayWs.close(1000, 'Bot stopping'); this._gatewayWs = null; } this._setState('disconnected'); } } /** * Play audio in the meeting. */ async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise { if (this._isShuttingDown) { this._logger.debug('Ignoring playAudio - bot is shutting down'); return; } if (this._state !== 'in_meeting' || !this._audioProcedure) { this._logger.warn('Cannot play audio - not in meeting'); return; } await this._audioProcedure.playAudio(audioData, format); } /** * Launch the browser and create a new page. * @param authMode - If true, use headful + minimal args (Chromium Minimal, proven to work for auth) */ private async _launchBrowser(authMode: boolean = false): Promise { this._logger.info(`Launching browser (authMode=${authMode})...`); const args = authMode ? [ // Chromium Minimal: only --no-sandbox + fake media (proven to work for authenticated join) '--no-sandbox', '--use-fake-ui-for-media-stream', '--use-fake-device-for-media-stream', ] : [ '--use-fake-ui-for-media-stream', '--use-fake-device-for-media-stream', '--disable-web-security', '--disable-features=IsolateOrigins,site-per-process', '--autoplay-policy=no-user-gesture-required', '--disable-blink-features=AutomationControlled', ]; this._browser = await chromium.launch({ headless: authMode ? false : config.botHeadless, args, }); this._context = await this._browser.newContext({ permissions: ['microphone', 'camera'], viewport: { width: 1280, height: 720 }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0', }); this._page = await this._context.newPage(); // Stealth: Override browser properties that reveal automation. // Teams checks these to detect headless/automated browsers and // blocks the /v2/ authenticated experience, falling back to light-meetings. await this._page.addInitScript(() => { // 1. Remove navigator.webdriver flag (primary detection signal) Object.defineProperty(navigator, 'webdriver', { get: () => false }); // 2. Add realistic plugins (headless has empty plugins array) Object.defineProperty(navigator, 'plugins', { get: () => [ { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }, ], }); // 3. Add realistic languages Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en', 'de'] }); // 4. Override permissions query to not reveal automation const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions); // @ts-ignore window.navigator.permissions.query = (parameters: any) => { if (parameters.name === 'notifications') { return Promise.resolve({ state: Notification.permission } as PermissionStatus); } return originalQuery(parameters); }; // 5. Add chrome runtime (missing in headless) // @ts-ignore if (!window.chrome) { window.chrome = {}; } // @ts-ignore if (!window.chrome.runtime) { window.chrome.runtime = {}; } }); // Initialize procedures this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName); this._audioCaptureProcedure = new AudioCaptureProcedure( this._page, this._logger, (base64Data, sampleRate) => { this._sendAudioChunk(base64Data, sampleRate); }, ); this._captionsProcedure = new CaptionsProcedure( this._page, this._logger, (entry) => { // Send transcript to Gateway this._sendTranscript(entry.speaker, entry.text, entry.isFinal); // Also notify local callbacks this._callbacks.onTranscript(entry); }, this._options.language ); this._audioProcedure = new AudioProcedure(this._page, this._logger); this._chatProcedure = new ChatProcedure( this._page, this._logger, (entry: ChatMessageEntry) => { // Send chat message to Gateway as a special transcript this._sendChatMessage(entry.speaker, entry.text); this._callbacks.onTranscript({ speaker: entry.speaker, text: entry.text, timestamp: entry.timestamp, isFinal: true, }); } ); // Inject audio getUserMedia override BEFORE any navigation // This ensures Teams gets our controlled audio stream when it calls getUserMedia await this._audioProcedure.injectAudioOverride(); // Inject audio capture (WebRTC interception) if transfer mode requires it const transferMode = this._getEffectiveTransferMode(); if (transferMode === 'audio') { await this._audioCaptureProcedure!.injectCaptureOverride(); } // Handle page errors this._page.on('pageerror', (error) => { this._logger.error('Page error:', error); }); // Handle page close this._page.on('close', () => { if (!this._isShuttingDown) { this._logger.warn('Page closed unexpectedly'); this._setState('disconnected'); } }); this._logger.info('Browser launched'); } /** * Close the browser. */ private async _closeBrowser(): Promise { try { if (this._page) { await this._page.close(); } if (this._context) { await this._context.close(); } if (this._browser) { await this._browser.close(); } } catch (error) { this._logger.error('Error closing browser:', error); } this._page = null; this._context = null; this._browser = null; this._logger.info('Browser closed'); } /** * Wait for the bot to be admitted from the lobby. */ private async _waitForMeetingAdmission(): Promise { const startTime = Date.now(); const timeout = config.timeouts.lobbyWait; let consecutiveNoSignal = 0; const maxNoSignal = 5; // Allow several cycles with no lobby/meeting signal before giving up while (Date.now() - startTime < timeout) { // Check if we're in the meeting const inMeeting = await this._joinProcedure!.isInMeeting({ waitForSeconds: 5 }); if (inMeeting) { return; } // Check if still in lobby const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 2 }); if (inLobby) { consecutiveNoSignal = 0; this._logger.info('Still waiting in lobby...'); continue; } // Neither in meeting nor in lobby — this can happen legitimately: // - Authenticated users skip lobby, but meeting UI takes seconds to load // - Page is transitioning between states // Only give up after several consecutive cycles with no signal consecutiveNoSignal++; const currentUrl = this._page?.url() || 'unknown'; this._logger.info(`No lobby/meeting signal detected (attempt ${consecutiveNoSignal}/${maxNoSignal}), URL: ${currentUrl}`); if (consecutiveNoSignal >= maxNoSignal) { // Take a screenshot and log page content for debugging before giving up await this._takeScreenshot('no-meeting-signal'); try { const bodySnippet = await this._page?.evaluate(() => document.body?.innerText?.substring(0, 500) || '(empty)' ); this._logger.warn(`Page content before giving up: ${bodySnippet}`); } catch { /* ignore */ } throw new Error('Bot was removed from lobby or meeting ended'); } } throw new Error('Timeout waiting to be admitted from lobby'); } /** * Determine the effective transfer mode based on config and join mode. * auto: anonymous → audio, authenticated → caption */ private _getEffectiveTransferMode(): 'caption' | 'audio' { const mode = this._options.transferMode || 'auto'; if (mode === 'caption') return 'caption'; if (mode === 'audio') return 'audio'; // auto: use audio for anonymous (Teams only provides English captions), caption for auth const isAuth = !!(this._options.botAccountEmail && this._options.botAccountPassword); return isAuth ? 'caption' : 'audio'; } /** * Enable captions and start scraping. */ private async _enableCaptions(): Promise { try { await this._captionsProcedure!.enableCaptionsFlow(); await this._captionsProcedure!.subscribeToCaptions(); this._logger.info('Captions enabled and subscribed'); } catch (error) { this._logger.warn('Could not enable captions:', error); } } /** * Enable audio capture from meeting participants. */ private async _enableAudioCapture(): Promise { if (!this._audioCaptureProcedure) { this._logger.warn('Audio capture procedure not initialized'); return; } try { await this._audioCaptureProcedure.startCapture(); this._logger.info('Audio capture started (PCM16 16kHz mono)'); } catch (error) { this._logger.warn('Could not start audio capture:', error); } } /** * Enable transcript capture (captions or audio) based on transfer mode. */ private async _enableTranscriptCapture(): Promise { const transferMode = this._getEffectiveTransferMode(); this._logger.info(`Transfer mode: ${transferMode} (configured: ${this._options.transferMode || 'auto'})`); if (transferMode === 'caption') { await this._enableCaptions(); } else { await this._enableAudioCapture(); } } /** * Enable chat monitoring. */ private async _enableChat(): Promise { try { await this._chatProcedure!.enableChatMonitoring(); await this._chatProcedure!.subscribeToChatMessages(); this._logger.info('Chat monitoring enabled and subscribed'); } catch (error) { this._logger.warn('Could not enable chat monitoring:', error); // Continue without chat - not a fatal error } } /** * Send a chat message event to the Gateway. */ private _sendChatMessage(speaker: string, text: string): void { const message: ChatMessage = { type: 'chatMessage', sessionId: this._sessionId, chat: { speaker, text, timestamp: new Date().toISOString(), }, }; this._sendToGateway(message); } /** * Send an audio chunk to the Gateway for STT processing. */ private _sendAudioChunk(base64Data: string, sampleRate: number): void { const message: AudioChunkMessage = { type: 'audioChunk', sessionId: this._sessionId, audio: { format: 'pcm16', sampleRate, data: base64Data, timestamp: new Date().toISOString(), }, }; this._sendToGateway(message); } /** * Send a text message to the meeting chat. */ async sendChatMessageToMeeting(text: string): Promise { if (this._isShuttingDown || this._state !== 'in_meeting' || !this._chatProcedure) { this._logger.warn('Cannot send chat message - not in meeting'); return; } await this._chatProcedure.sendChatMessage(text); } /** * Update the bot state and notify callbacks + Gateway. */ private _setState(state: BotState, message?: string): void { this._state = state; this._logger.info(`State changed: ${state}${message ? ` - ${message}` : ''}`); this._callbacks.onStateChange(state, message); // Send status to Gateway const statusMap: Record = { 'idle': 'connecting', 'launching': 'connecting', 'navigating': 'connecting', 'in_lobby': 'in_lobby', 'in_meeting': 'joined', 'leaving': 'left', 'error': 'error', 'disconnected': 'left', }; this._sendStatus(statusMap[state], message); } /** * Take a screenshot for debugging. * Logs screenshot as base64 for easy viewing from Azure logs. */ private async _takeScreenshot(name: string): Promise { if (!config.screenshotOnError || !this._page) { return; } try { // Save to file const screenshotDir = config.screenshotDir; if (!fs.existsSync(screenshotDir)) { fs.mkdirSync(screenshotDir, { recursive: true }); } const filename = `${this._sessionId}-${name}-${Date.now()}.png`; const filepath = path.join(screenshotDir, filename); const buffer = await this._page.screenshot({ fullPage: true }); fs.writeFileSync(filepath, buffer); this._logger.info(`Screenshot saved: ${filepath}`); // Also log as base64 for Azure logs (truncated for readability) const base64 = buffer.toString('base64'); this._logger.info(`SCREENSHOT_BASE64_START:${name}`); // Log in chunks to avoid log line limits const chunkSize = 50000; for (let i = 0; i < base64.length; i += chunkSize) { this._logger.info(`SCREENSHOT_CHUNK:${base64.substring(i, i + chunkSize)}`); } this._logger.info(`SCREENSHOT_BASE64_END:${name}`); } catch (error) { this._logger.error('Error taking screenshot:', error); } } }