1060 lines
34 KiB
TypeScript
1060 lines
34 KiB
TypeScript
import { Browser, BrowserContext, Page, chromium } from 'playwright';
|
|
import { Logger } from 'winston';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
import path from 'path';
|
|
import fs from 'fs';
|
|
import WebSocket from 'ws';
|
|
|
|
import { config } from '../config';
|
|
import { createSessionLogger } from '../utils/logger';
|
|
import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage } from '../types';
|
|
import { JoinProcedure } from './joinProcedure';
|
|
import { CaptionsProcedure } from './captionsProcedure';
|
|
import { AudioProcedure } from './audioProcedure';
|
|
import { AudioCaptureProcedure } from './audioCaptureProcedure';
|
|
import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
|
|
import { AuthProcedure } from './authProcedure';
|
|
import { isValidMeetingUrl } from './meetingUrlParser';
|
|
|
|
export interface OrchestratorCallbacks {
|
|
onStateChange: (state: BotState, message?: string) => void;
|
|
onTranscript: (entry: TranscriptEntry) => void;
|
|
onError: (error: Error) => void;
|
|
}
|
|
|
|
export interface OrchestratorOptions {
|
|
gatewayWsUrl: string;
|
|
instanceId: string;
|
|
language?: string;
|
|
botAccountEmail?: string;
|
|
botAccountPassword?: string;
|
|
transferMode?: string;
|
|
}
|
|
|
|
/**
|
|
* Orchestrates the entire bot lifecycle:
|
|
* - Connects to Gateway via WebSocket
|
|
* - Launches browser
|
|
* - Joins meeting
|
|
* - Enables captions
|
|
* - Sends transcripts to Gateway
|
|
* - Handles audio playback from Gateway
|
|
* - Leaves meeting
|
|
*/
|
|
export class BotOrchestrator {
|
|
private _sessionId: string;
|
|
private _meetingUrl: string;
|
|
private _botName: string;
|
|
private _logger: Logger;
|
|
private _callbacks: OrchestratorCallbacks;
|
|
private _options: OrchestratorOptions;
|
|
|
|
private _browser: Browser | null = null;
|
|
private _context: BrowserContext | null = null;
|
|
private _page: Page | null = null;
|
|
private _gatewayWs: WebSocket | null = null;
|
|
private _useHttpFallback: boolean = false;
|
|
private _httpBaseUrl: string = '';
|
|
|
|
private _joinProcedure: JoinProcedure | null = null;
|
|
private _captionsProcedure: CaptionsProcedure | null = null;
|
|
private _audioProcedure: AudioProcedure | null = null;
|
|
private _audioCaptureProcedure: AudioCaptureProcedure | null = null;
|
|
private _chatProcedure: ChatProcedure | null = null;
|
|
|
|
private _state: BotState = 'idle';
|
|
private _isShuttingDown: boolean = false;
|
|
private _keepAliveInterval: NodeJS.Timeout | null = null;
|
|
|
|
constructor(
|
|
sessionId: string,
|
|
meetingUrl: string,
|
|
botName: string,
|
|
callbacks: OrchestratorCallbacks,
|
|
options: OrchestratorOptions
|
|
) {
|
|
this._sessionId = sessionId;
|
|
this._meetingUrl = meetingUrl;
|
|
this._botName = botName || config.botName;
|
|
this._callbacks = callbacks;
|
|
this._options = options;
|
|
this._logger = createSessionLogger(sessionId);
|
|
}
|
|
|
|
get sessionId(): string {
|
|
return this._sessionId;
|
|
}
|
|
|
|
get state(): BotState {
|
|
return this._state;
|
|
}
|
|
|
|
/**
|
|
* Start the bot - connect to Gateway, launch browser, join meeting, enable captions.
|
|
* Chooses between anonymous join and authenticated join based on credentials.
|
|
*/
|
|
async start(): Promise<void> {
|
|
if (!isValidMeetingUrl(this._meetingUrl)) {
|
|
throw new Error(`Invalid meeting URL: ${this._meetingUrl}`);
|
|
}
|
|
|
|
try {
|
|
this._setState('launching');
|
|
|
|
// Connect to Gateway WebSocket first
|
|
await this._connectToGateway();
|
|
|
|
// Choose join method based on credentials
|
|
const hasCredentials = !!(this._options.botAccountEmail && this._options.botAccountPassword);
|
|
if (hasCredentials) {
|
|
this._logger.info(`Authenticated join as: ${this._options.botAccountEmail}`);
|
|
await this._attemptAuthJoin();
|
|
} else {
|
|
this._logger.info('Anonymous join with bot name: ' + this._botName);
|
|
await this._attemptJoin();
|
|
}
|
|
|
|
} catch (error) {
|
|
this._logger.error('Error starting bot:', error);
|
|
this._setState('error', (error as Error).message);
|
|
await this._takeScreenshot('error');
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Join a meeting as anonymous guest with the configured bot name.
|
|
*/
|
|
private async _attemptJoin(): Promise<void> {
|
|
// Launch browser
|
|
await this._launchBrowser();
|
|
|
|
this._setState('navigating');
|
|
|
|
// STEP 1: Navigate to meeting URL and click "Continue on this browser"
|
|
await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl);
|
|
|
|
// STEP 2: Enter bot name and click "Join now"
|
|
await this._joinProcedure!.joinMeetingLobbyFlow();
|
|
|
|
// Check if we're in lobby
|
|
const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 10 });
|
|
if (inLobby) {
|
|
this._setState('in_lobby');
|
|
this._logger.info('Bot is in lobby, waiting to be admitted...');
|
|
}
|
|
|
|
// Wait to be admitted to the meeting
|
|
await this._waitForMeetingAdmission();
|
|
|
|
this._setState('in_meeting');
|
|
this._logger.info(`Bot joined the meeting as "${this._botName}"`);
|
|
|
|
// Start keepalive to prevent idle disconnect
|
|
this._startKeepAlive();
|
|
|
|
// Dismiss any post-join permission modals (e.g. "Manage windows on all displays")
|
|
await this._joinProcedure!.dismissBrowserPermissionModals();
|
|
|
|
// Initialize audio playback
|
|
await this._audioProcedure!.initialize();
|
|
|
|
// Enable transcript capture (captions or audio based on transferMode)
|
|
await this._enableTranscriptCapture();
|
|
|
|
// Enable chat monitoring
|
|
await this._enableChat();
|
|
}
|
|
|
|
/**
|
|
* Join a meeting as authenticated user (System Bot or User Account).
|
|
* Flow: teams.microsoft.com → MS Login → Teams Chat → Join → Pre-Join → Join now
|
|
*/
|
|
private async _attemptAuthJoin(): Promise<void> {
|
|
// Launch browser in headful mode with minimal args (Chromium Minimal)
|
|
await this._launchBrowser(true);
|
|
|
|
this._setState('navigating');
|
|
|
|
// STEP 1: Navigate to teams.microsoft.com
|
|
this._logger.info('Auth join: navigating to teams.microsoft.com');
|
|
await this._page!.goto('https://teams.microsoft.com', {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: 30000,
|
|
});
|
|
|
|
// Wait for login redirect
|
|
try {
|
|
await this._page!.waitForURL('**/login.microsoftonline.com/**', { timeout: 30000 });
|
|
this._logger.info('Redirected to MS login page');
|
|
} catch {
|
|
this._logger.warn(`No login redirect, current URL: ${this._page!.url().substring(0, 150)}`);
|
|
}
|
|
|
|
// Wait for login page to render
|
|
try {
|
|
await this._page!.waitForSelector('input[name="loginfmt"], input[type="email"]', {
|
|
timeout: 15000, state: 'visible',
|
|
});
|
|
} catch {
|
|
this._logger.warn('Login page elements not found');
|
|
}
|
|
|
|
// STEP 2: Microsoft Authentication
|
|
this._logger.info(`Authenticating as ${this._options.botAccountEmail}`);
|
|
const authProcedure = new AuthProcedure(this._page!, this._logger);
|
|
const authSuccess = await authProcedure.authenticateWithMicrosoft(
|
|
this._options.botAccountEmail!,
|
|
this._options.botAccountPassword!,
|
|
true,
|
|
);
|
|
|
|
if (!authSuccess) {
|
|
throw new Error('Microsoft authentication failed');
|
|
}
|
|
this._logger.info('Authentication successful');
|
|
|
|
// STEP 3: Wait for Teams chat page (landing page after auth)
|
|
try {
|
|
await this._page!.waitForURL('**/teams.microsoft.com/**', { timeout: 30000 });
|
|
} catch {
|
|
// Also accept teams.cloud.microsoft
|
|
try {
|
|
await this._page!.waitForURL('**/teams.cloud.microsoft/**', { timeout: 10000 });
|
|
} catch {
|
|
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
|
|
}
|
|
}
|
|
|
|
// Wait for "Join" button in chat header
|
|
try {
|
|
await this._page!.waitForSelector(
|
|
'button[data-tid="chat-join-button"], button[data-tid="join-call-button"]',
|
|
{ timeout: 30000, state: 'visible' },
|
|
);
|
|
this._logger.info('Teams chat page loaded, "Join" button found');
|
|
} catch {
|
|
this._logger.warn('"Join" button not found in chat header');
|
|
await this._takeScreenshot('auth-no-join-button');
|
|
}
|
|
|
|
// STEP 4: Click "Join" in chat header → Pre-Join screen
|
|
this._logger.info('Clicking "Join" in chat header');
|
|
const chatJoinSelectors = [
|
|
'button[data-tid="chat-join-button"]',
|
|
'button[data-tid="join-call-button"]',
|
|
];
|
|
|
|
let chatJoinClicked = false;
|
|
for (const selector of chatJoinSelectors) {
|
|
try {
|
|
const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' });
|
|
if (btn) {
|
|
await btn.click();
|
|
chatJoinClicked = true;
|
|
break;
|
|
}
|
|
} catch { /* try next */ }
|
|
}
|
|
|
|
if (!chatJoinClicked) {
|
|
throw new Error('"Join" button in chat header not found');
|
|
}
|
|
|
|
// STEP 5: Pre-Join screen → Click "Join now"
|
|
this._logger.info('Waiting for pre-join screen');
|
|
try {
|
|
await this._page!.waitForSelector(
|
|
'button:has-text("Join now"), button:has-text("Jetzt teilnehmen"), button[data-tid="prejoin-join-button"]',
|
|
{ timeout: 30000, state: 'visible' },
|
|
);
|
|
} catch {
|
|
this._logger.warn('"Join now" button not found');
|
|
await this._takeScreenshot('auth-no-join-now');
|
|
}
|
|
|
|
// Activate camera toggle if it's off (so background image is visible)
|
|
await this._ensureCameraOn();
|
|
|
|
await this._page!.waitForTimeout(2000);
|
|
|
|
const joinNowSelectors = [
|
|
'button:has-text("Join now")',
|
|
'button:has-text("Jetzt teilnehmen")',
|
|
'button[data-tid="prejoin-join-button"]',
|
|
];
|
|
|
|
let joinNowClicked = false;
|
|
for (const selector of joinNowSelectors) {
|
|
try {
|
|
const btn = await this._page!.waitForSelector(selector, { timeout: 5000, state: 'visible' });
|
|
if (btn) {
|
|
await btn.click();
|
|
joinNowClicked = true;
|
|
break;
|
|
}
|
|
} catch { /* try next */ }
|
|
}
|
|
|
|
if (!joinNowClicked) {
|
|
throw new Error('"Join now" button not found on pre-join screen');
|
|
}
|
|
|
|
this._logger.info('Clicked "Join now", waiting for meeting');
|
|
|
|
// Wait for meeting admission (hangup button = in meeting)
|
|
await this._waitForMeetingAdmission();
|
|
|
|
this._setState('in_meeting');
|
|
this._logger.info(`Bot joined the meeting (authenticated as ${this._options.botAccountEmail})`);
|
|
|
|
// Start keepalive to prevent idle disconnect
|
|
this._startKeepAlive();
|
|
|
|
// Initialize audio playback
|
|
await this._audioProcedure!.initialize();
|
|
|
|
// Enable transcript capture (captions or audio based on transferMode)
|
|
await this._enableTranscriptCapture();
|
|
await this._enableChat();
|
|
}
|
|
|
|
/**
|
|
* Ensure the camera is turned on in the pre-join screen.
|
|
* When camera is on, Teams shows the profile/background image.
|
|
*
|
|
* Teams pre-join uses a fui-Switch input:
|
|
* <input data-tid="toggle-video" role="switch" type="checkbox" checked>
|
|
* - checked present = camera ON (data-cid="toggle-video-true", title="Turn camera off")
|
|
* - checked absent = camera OFF (data-cid="toggle-video-false", title="Turn camera on")
|
|
*/
|
|
private async _ensureCameraOn(): Promise<void> {
|
|
try {
|
|
// Primary: the actual switch input (fui-Switch)
|
|
let cameraToggle = await this._page!.$('input[data-tid="toggle-video"]');
|
|
|
|
if (!cameraToggle) {
|
|
this._logger.info('Primary camera selector not found, trying fallbacks...');
|
|
const fallbacks = [
|
|
'[data-tid="toggle-video"]',
|
|
'input[role="switch"][title*="camera" i]',
|
|
'input[role="switch"][title*="Camera" i]',
|
|
'input[role="switch"][title*="Video" i]',
|
|
];
|
|
for (const sel of fallbacks) {
|
|
cameraToggle = await this._page!.$(sel);
|
|
if (cameraToggle) {
|
|
this._logger.info(`Camera toggle found via fallback: ${sel}`);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!cameraToggle) {
|
|
this._logger.warn('Camera toggle not found on pre-join screen');
|
|
return;
|
|
}
|
|
|
|
// Read current state
|
|
const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({
|
|
checked: el.checked,
|
|
dataCid: el.getAttribute('data-cid') || '',
|
|
title: el.getAttribute('title') || '',
|
|
}));
|
|
|
|
this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
|
|
|
|
if (!state.checked) {
|
|
// Camera is OFF — click to turn ON
|
|
await cameraToggle.click();
|
|
this._logger.info('Camera toggled ON');
|
|
await this._page!.waitForTimeout(2000);
|
|
|
|
// Verify
|
|
const afterState = await cameraToggle.evaluate((el: HTMLInputElement) => ({
|
|
checked: el.checked,
|
|
dataCid: el.getAttribute('data-cid') || '',
|
|
}));
|
|
this._logger.info(`Camera after toggle: checked=${afterState.checked}, data-cid="${afterState.dataCid}"`);
|
|
} else {
|
|
this._logger.info('Camera already ON');
|
|
}
|
|
} catch (err) {
|
|
this._logger.warn(`Could not toggle camera: ${err}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start a keepalive timer that periodically moves the mouse and sends
|
|
* a WebSocket ping. Prevents Teams from detecting the bot as idle
|
|
* and kicking it from the meeting.
|
|
*/
|
|
private _startKeepAlive(): void {
|
|
if (this._keepAliveInterval) return;
|
|
|
|
this._keepAliveInterval = setInterval(async () => {
|
|
if (this._isShuttingDown || !this._page) return;
|
|
|
|
try {
|
|
// Small random mouse movement to simulate user activity
|
|
const x = 640 + Math.floor(Math.random() * 20 - 10);
|
|
const y = 360 + Math.floor(Math.random() * 20 - 10);
|
|
await this._page.mouse.move(x, y);
|
|
} catch {
|
|
// Page might be closed
|
|
}
|
|
|
|
// WebSocket heartbeat
|
|
if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) {
|
|
try {
|
|
this._gatewayWs.send(JSON.stringify({ type: 'ping', sessionId: this._sessionId }));
|
|
} catch {
|
|
// Connection might be closing
|
|
}
|
|
}
|
|
}, 30000);
|
|
|
|
this._logger.info('Keepalive started (30s interval)');
|
|
}
|
|
|
|
/**
|
|
* Stop the keepalive timer.
|
|
*/
|
|
private _stopKeepAlive(): void {
|
|
if (this._keepAliveInterval) {
|
|
clearInterval(this._keepAliveInterval);
|
|
this._keepAliveInterval = null;
|
|
this._logger.info('Keepalive stopped');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Connect to the Gateway WebSocket for this session.
|
|
*/
|
|
private async _connectToGateway(): Promise<void> {
|
|
// gatewayWsUrl is the full WebSocket URL provided by the Gateway
|
|
// It already includes instanceId and sessionId
|
|
const wsUrl = this._options.gatewayWsUrl;
|
|
this._logger.info(`Connecting to Gateway: ${wsUrl}`);
|
|
|
|
// Derive HTTP base URL from WebSocket URL for fallback
|
|
this._httpBaseUrl = wsUrl
|
|
.replace('wss://', 'https://')
|
|
.replace('ws://', 'http://')
|
|
.replace(/\/bot\/ws\/.*$/, '');
|
|
|
|
return new Promise((resolve, reject) => {
|
|
this._gatewayWs = new WebSocket(wsUrl);
|
|
|
|
const wsTimeout = setTimeout(() => {
|
|
if (this._gatewayWs?.readyState !== WebSocket.OPEN) {
|
|
this._logger.warn('WebSocket connection timeout - switching to HTTP fallback');
|
|
this._useHttpFallback = true;
|
|
this._gatewayWs?.close();
|
|
this._gatewayWs = null;
|
|
resolve(); // Continue with HTTP fallback instead of failing
|
|
}
|
|
}, 10000);
|
|
|
|
this._gatewayWs.on('open', () => {
|
|
clearTimeout(wsTimeout);
|
|
this._logger.info('Connected to Gateway via WebSocket');
|
|
this._useHttpFallback = false;
|
|
resolve();
|
|
});
|
|
|
|
this._gatewayWs.on('message', (data) => {
|
|
this._handleGatewayMessage(data.toString());
|
|
});
|
|
|
|
this._gatewayWs.on('close', (code, reason) => {
|
|
this._logger.warn(`Gateway WebSocket closed: ${code} - ${reason}`);
|
|
if (!this._isShuttingDown && !this._useHttpFallback) {
|
|
this._logger.info('Switching to HTTP fallback for transcript delivery');
|
|
this._useHttpFallback = true;
|
|
}
|
|
});
|
|
|
|
this._gatewayWs.on('error', (error) => {
|
|
clearTimeout(wsTimeout);
|
|
this._logger.error('Gateway WebSocket error:', error);
|
|
this._logger.info('Switching to HTTP fallback for transcript delivery');
|
|
this._useHttpFallback = true;
|
|
this._gatewayWs = null;
|
|
resolve(); // Continue with HTTP fallback
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Handle incoming messages from the Gateway.
|
|
*/
|
|
private _handleGatewayMessage(data: string): void {
|
|
try {
|
|
const message = JSON.parse(data);
|
|
|
|
switch (message.type) {
|
|
case 'playAudio':
|
|
const audioMsg = message as PlayAudioMessage;
|
|
this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
|
|
break;
|
|
|
|
case 'sendChatMessage':
|
|
const chatMsg = message as SendChatMessage;
|
|
this.sendChatMessageToMeeting(chatMsg.text);
|
|
break;
|
|
|
|
case 'stopAudio':
|
|
this._logger.info('Stop audio command received from Gateway');
|
|
if (this._audioProcedure) {
|
|
this._audioProcedure.stopAllAudio();
|
|
}
|
|
break;
|
|
|
|
case 'pong':
|
|
// Heartbeat response
|
|
break;
|
|
|
|
default:
|
|
this._logger.debug('Unknown Gateway message type:', message.type);
|
|
}
|
|
} catch (error) {
|
|
this._logger.error('Error parsing Gateway message:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a message to the Gateway (WebSocket or HTTP fallback).
|
|
*/
|
|
private _sendToGateway(message: object): void {
|
|
if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) {
|
|
try {
|
|
this._gatewayWs.send(JSON.stringify(message));
|
|
return;
|
|
} catch (error) {
|
|
this._logger.error('WebSocket send error, falling back to HTTP:', error);
|
|
this._useHttpFallback = true;
|
|
}
|
|
}
|
|
|
|
// HTTP fallback
|
|
if (this._useHttpFallback) {
|
|
this._sendViaHttp(message);
|
|
} else {
|
|
this._logger.warn('Cannot send to Gateway - no WebSocket and no HTTP fallback');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a message via HTTP POST (fallback when WebSocket unavailable).
|
|
*/
|
|
private async _sendViaHttp(message: any): Promise<void> {
|
|
const msgType = message.type;
|
|
let url = '';
|
|
|
|
if (msgType === 'transcript') {
|
|
url = `${this._httpBaseUrl}/bot/transcript/${this._sessionId}`;
|
|
} else if (msgType === 'status') {
|
|
url = `${this._httpBaseUrl}/bot/status/${this._sessionId}`;
|
|
} else if (msgType === 'audioChunk') {
|
|
// Audio chunks are too frequent for HTTP — only send via WebSocket
|
|
return;
|
|
} else {
|
|
this._logger.debug(`HTTP fallback: unsupported message type ${msgType}`);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(message),
|
|
});
|
|
if (!response.ok) {
|
|
this._logger.warn(`HTTP fallback response: ${response.status} ${response.statusText}`);
|
|
}
|
|
} catch (error) {
|
|
this._logger.error(`HTTP fallback error for ${msgType}:`, error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a transcript to the Gateway.
|
|
*/
|
|
private _sendTranscript(speaker: string, text: string, isFinal: boolean): void {
|
|
const message: TranscriptMessage = {
|
|
type: 'transcript',
|
|
sessionId: this._sessionId,
|
|
transcript: {
|
|
speaker,
|
|
text,
|
|
timestamp: new Date().toISOString(),
|
|
isFinal,
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send a status update to the Gateway.
|
|
*/
|
|
private _sendStatus(status: StatusMessage['status'], message?: string): void {
|
|
const statusMessage: StatusMessage = {
|
|
type: 'status',
|
|
sessionId: this._sessionId,
|
|
status,
|
|
message,
|
|
};
|
|
this._sendToGateway(statusMessage);
|
|
}
|
|
|
|
/**
|
|
* Stop the bot - leave meeting, close browser, disconnect from Gateway.
|
|
*/
|
|
async stop(): Promise<void> {
|
|
if (this._isShuttingDown) {
|
|
return;
|
|
}
|
|
|
|
this._isShuttingDown = true;
|
|
this._logger.info('Stopping bot...');
|
|
|
|
// Stop keepalive first
|
|
this._stopKeepAlive();
|
|
|
|
try {
|
|
this._setState('leaving');
|
|
|
|
// Stop audio capture
|
|
if (this._audioCaptureProcedure) {
|
|
await this._audioCaptureProcedure.stopCapture();
|
|
}
|
|
|
|
// Unsubscribe from captions and chat
|
|
if (this._captionsProcedure) {
|
|
await this._captionsProcedure.unsubscribe();
|
|
}
|
|
if (this._chatProcedure) {
|
|
await this._chatProcedure.unsubscribe();
|
|
}
|
|
|
|
// Clean up audio playback
|
|
if (this._audioProcedure) {
|
|
await this._audioProcedure.cleanup();
|
|
}
|
|
|
|
// Leave the meeting
|
|
if (this._joinProcedure && this._state !== 'error') {
|
|
await this._joinProcedure.leaveMeetingFlow();
|
|
}
|
|
|
|
} catch (error) {
|
|
this._logger.error('Error during shutdown:', error);
|
|
} finally {
|
|
// Close browser
|
|
await this._closeBrowser();
|
|
|
|
// Close Gateway connection
|
|
if (this._gatewayWs) {
|
|
this._gatewayWs.close(1000, 'Bot stopping');
|
|
this._gatewayWs = null;
|
|
}
|
|
|
|
this._setState('disconnected');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Play audio in the meeting.
|
|
*/
|
|
async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise<void> {
|
|
if (this._isShuttingDown) {
|
|
this._logger.debug('Ignoring playAudio - bot is shutting down');
|
|
return;
|
|
}
|
|
if (this._state !== 'in_meeting' || !this._audioProcedure) {
|
|
this._logger.warn('Cannot play audio - not in meeting');
|
|
return;
|
|
}
|
|
|
|
await this._audioProcedure.playAudio(audioData, format);
|
|
}
|
|
|
|
/**
|
|
* Launch the browser and create a new page.
|
|
* @param authMode - If true, use headful + minimal args (Chromium Minimal, proven to work for auth)
|
|
*/
|
|
private async _launchBrowser(authMode: boolean = false): Promise<void> {
|
|
this._logger.info(`Launching browser (authMode=${authMode})...`);
|
|
|
|
const args = authMode
|
|
? [
|
|
// Chromium Minimal: only --no-sandbox + fake media (proven to work for authenticated join)
|
|
'--no-sandbox',
|
|
'--use-fake-ui-for-media-stream',
|
|
'--use-fake-device-for-media-stream',
|
|
]
|
|
: [
|
|
'--use-fake-ui-for-media-stream',
|
|
'--use-fake-device-for-media-stream',
|
|
'--disable-web-security',
|
|
'--disable-features=IsolateOrigins,site-per-process',
|
|
'--autoplay-policy=no-user-gesture-required',
|
|
'--disable-blink-features=AutomationControlled',
|
|
];
|
|
|
|
this._browser = await chromium.launch({
|
|
headless: authMode ? false : config.botHeadless,
|
|
args,
|
|
});
|
|
|
|
this._context = await this._browser.newContext({
|
|
permissions: ['microphone', 'camera'],
|
|
viewport: { width: 1280, height: 720 },
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
|
|
});
|
|
|
|
this._page = await this._context.newPage();
|
|
|
|
// Stealth: Override browser properties that reveal automation.
|
|
// Teams checks these to detect headless/automated browsers and
|
|
// blocks the /v2/ authenticated experience, falling back to light-meetings.
|
|
await this._page.addInitScript(() => {
|
|
// 1. Remove navigator.webdriver flag (primary detection signal)
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
|
|
|
// 2. Add realistic plugins (headless has empty plugins array)
|
|
Object.defineProperty(navigator, 'plugins', {
|
|
get: () => [
|
|
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
|
|
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
|
|
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
|
|
],
|
|
});
|
|
|
|
// 3. Add realistic languages
|
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en', 'de'] });
|
|
|
|
// 4. Override permissions query to not reveal automation
|
|
const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
|
|
// @ts-ignore
|
|
window.navigator.permissions.query = (parameters: any) => {
|
|
if (parameters.name === 'notifications') {
|
|
return Promise.resolve({ state: Notification.permission } as PermissionStatus);
|
|
}
|
|
return originalQuery(parameters);
|
|
};
|
|
|
|
// 5. Add chrome runtime (missing in headless)
|
|
// @ts-ignore
|
|
if (!window.chrome) { window.chrome = {}; }
|
|
// @ts-ignore
|
|
if (!window.chrome.runtime) { window.chrome.runtime = {}; }
|
|
});
|
|
|
|
// Initialize procedures
|
|
this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName);
|
|
this._audioCaptureProcedure = new AudioCaptureProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(base64Data, sampleRate) => {
|
|
this._sendAudioChunk(base64Data, sampleRate);
|
|
},
|
|
);
|
|
this._captionsProcedure = new CaptionsProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(entry) => {
|
|
// Send transcript to Gateway
|
|
this._sendTranscript(entry.speaker, entry.text, entry.isFinal);
|
|
// Also notify local callbacks
|
|
this._callbacks.onTranscript(entry);
|
|
},
|
|
this._options.language
|
|
);
|
|
this._audioProcedure = new AudioProcedure(this._page, this._logger);
|
|
this._chatProcedure = new ChatProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(entry: ChatMessageEntry) => {
|
|
// Send chat message to Gateway as a special transcript
|
|
this._sendChatMessage(entry.speaker, entry.text);
|
|
this._callbacks.onTranscript({
|
|
speaker: entry.speaker,
|
|
text: entry.text,
|
|
timestamp: entry.timestamp,
|
|
isFinal: true,
|
|
});
|
|
}
|
|
);
|
|
|
|
// Inject audio getUserMedia override BEFORE any navigation
|
|
// This ensures Teams gets our controlled audio stream when it calls getUserMedia
|
|
await this._audioProcedure.injectAudioOverride();
|
|
|
|
// Inject audio capture (WebRTC interception) if transfer mode requires it
|
|
const transferMode = this._getEffectiveTransferMode();
|
|
if (transferMode === 'audio') {
|
|
await this._audioCaptureProcedure!.injectCaptureOverride();
|
|
}
|
|
|
|
// Handle page errors
|
|
this._page.on('pageerror', (error) => {
|
|
this._logger.error('Page error:', error);
|
|
});
|
|
|
|
// Handle page close
|
|
this._page.on('close', () => {
|
|
if (!this._isShuttingDown) {
|
|
this._logger.warn('Page closed unexpectedly');
|
|
this._setState('disconnected');
|
|
}
|
|
});
|
|
|
|
this._logger.info('Browser launched');
|
|
}
|
|
|
|
/**
|
|
* Close the browser.
|
|
*/
|
|
private async _closeBrowser(): Promise<void> {
|
|
try {
|
|
if (this._page) {
|
|
await this._page.close();
|
|
}
|
|
if (this._context) {
|
|
await this._context.close();
|
|
}
|
|
if (this._browser) {
|
|
await this._browser.close();
|
|
}
|
|
} catch (error) {
|
|
this._logger.error('Error closing browser:', error);
|
|
}
|
|
|
|
this._page = null;
|
|
this._context = null;
|
|
this._browser = null;
|
|
this._logger.info('Browser closed');
|
|
}
|
|
|
|
/**
|
|
* Wait for the bot to be admitted from the lobby.
|
|
*/
|
|
private async _waitForMeetingAdmission(): Promise<void> {
|
|
const startTime = Date.now();
|
|
const timeout = config.timeouts.lobbyWait;
|
|
let consecutiveNoSignal = 0;
|
|
const maxNoSignal = 5; // Allow several cycles with no lobby/meeting signal before giving up
|
|
|
|
while (Date.now() - startTime < timeout) {
|
|
// Check if we're in the meeting
|
|
const inMeeting = await this._joinProcedure!.isInMeeting({ waitForSeconds: 5 });
|
|
if (inMeeting) {
|
|
return;
|
|
}
|
|
|
|
// Check if still in lobby
|
|
const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 2 });
|
|
if (inLobby) {
|
|
consecutiveNoSignal = 0;
|
|
this._logger.info('Still waiting in lobby...');
|
|
continue;
|
|
}
|
|
|
|
// Neither in meeting nor in lobby — this can happen legitimately:
|
|
// - Authenticated users skip lobby, but meeting UI takes seconds to load
|
|
// - Page is transitioning between states
|
|
// Only give up after several consecutive cycles with no signal
|
|
consecutiveNoSignal++;
|
|
const currentUrl = this._page?.url() || 'unknown';
|
|
this._logger.info(`No lobby/meeting signal detected (attempt ${consecutiveNoSignal}/${maxNoSignal}), URL: ${currentUrl}`);
|
|
|
|
if (consecutiveNoSignal >= maxNoSignal) {
|
|
// Take a screenshot and log page content for debugging before giving up
|
|
await this._takeScreenshot('no-meeting-signal');
|
|
try {
|
|
const bodySnippet = await this._page?.evaluate(() =>
|
|
document.body?.innerText?.substring(0, 500) || '(empty)'
|
|
);
|
|
this._logger.warn(`Page content before giving up: ${bodySnippet}`);
|
|
} catch { /* ignore */ }
|
|
throw new Error('Bot was removed from lobby or meeting ended');
|
|
}
|
|
}
|
|
|
|
throw new Error('Timeout waiting to be admitted from lobby');
|
|
}
|
|
|
|
/**
|
|
* Determine the effective transfer mode based on config and join mode.
|
|
* auto: anonymous → audio, authenticated → caption
|
|
*/
|
|
private _getEffectiveTransferMode(): 'caption' | 'audio' {
|
|
const mode = this._options.transferMode || 'auto';
|
|
if (mode === 'caption') return 'caption';
|
|
if (mode === 'audio') return 'audio';
|
|
// auto: use audio for anonymous (Teams only provides English captions), caption for auth
|
|
const isAuth = !!(this._options.botAccountEmail && this._options.botAccountPassword);
|
|
return isAuth ? 'caption' : 'audio';
|
|
}
|
|
|
|
/**
|
|
* Enable captions and start scraping.
|
|
*/
|
|
private async _enableCaptions(): Promise<void> {
|
|
try {
|
|
await this._captionsProcedure!.enableCaptionsFlow();
|
|
await this._captionsProcedure!.subscribeToCaptions();
|
|
this._logger.info('Captions enabled and subscribed');
|
|
} catch (error) {
|
|
this._logger.warn('Could not enable captions:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enable audio capture from meeting participants.
|
|
*/
|
|
private async _enableAudioCapture(): Promise<void> {
|
|
if (!this._audioCaptureProcedure) {
|
|
this._logger.warn('Audio capture procedure not initialized');
|
|
return;
|
|
}
|
|
try {
|
|
await this._audioCaptureProcedure.startCapture();
|
|
this._logger.info('Audio capture started (PCM16 16kHz mono)');
|
|
} catch (error) {
|
|
this._logger.warn('Could not start audio capture:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enable transcript capture (captions or audio) based on transfer mode.
|
|
*/
|
|
private async _enableTranscriptCapture(): Promise<void> {
|
|
const transferMode = this._getEffectiveTransferMode();
|
|
this._logger.info(`Transfer mode: ${transferMode} (configured: ${this._options.transferMode || 'auto'})`);
|
|
|
|
if (transferMode === 'caption') {
|
|
await this._enableCaptions();
|
|
} else {
|
|
await this._enableAudioCapture();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enable chat monitoring.
|
|
*/
|
|
private async _enableChat(): Promise<void> {
|
|
try {
|
|
await this._chatProcedure!.enableChatMonitoring();
|
|
await this._chatProcedure!.subscribeToChatMessages();
|
|
this._logger.info('Chat monitoring enabled and subscribed');
|
|
} catch (error) {
|
|
this._logger.warn('Could not enable chat monitoring:', error);
|
|
// Continue without chat - not a fatal error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a chat message event to the Gateway.
|
|
*/
|
|
private _sendChatMessage(speaker: string, text: string): void {
|
|
const message: ChatMessage = {
|
|
type: 'chatMessage',
|
|
sessionId: this._sessionId,
|
|
chat: {
|
|
speaker,
|
|
text,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send an audio chunk to the Gateway for STT processing.
|
|
*/
|
|
private _sendAudioChunk(base64Data: string, sampleRate: number): void {
|
|
const message: AudioChunkMessage = {
|
|
type: 'audioChunk',
|
|
sessionId: this._sessionId,
|
|
audio: {
|
|
format: 'pcm16',
|
|
sampleRate,
|
|
data: base64Data,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send a text message to the meeting chat.
|
|
*/
|
|
async sendChatMessageToMeeting(text: string): Promise<void> {
|
|
if (this._isShuttingDown || this._state !== 'in_meeting' || !this._chatProcedure) {
|
|
this._logger.warn('Cannot send chat message - not in meeting');
|
|
return;
|
|
}
|
|
await this._chatProcedure.sendChatMessage(text);
|
|
}
|
|
|
|
/**
|
|
* Update the bot state and notify callbacks + Gateway.
|
|
*/
|
|
private _setState(state: BotState, message?: string): void {
|
|
this._state = state;
|
|
this._logger.info(`State changed: ${state}${message ? ` - ${message}` : ''}`);
|
|
this._callbacks.onStateChange(state, message);
|
|
|
|
// Send status to Gateway
|
|
const statusMap: Record<BotState, StatusMessage['status']> = {
|
|
'idle': 'connecting',
|
|
'launching': 'connecting',
|
|
'navigating': 'connecting',
|
|
'in_lobby': 'in_lobby',
|
|
'in_meeting': 'joined',
|
|
'leaving': 'left',
|
|
'error': 'error',
|
|
'disconnected': 'left',
|
|
};
|
|
this._sendStatus(statusMap[state], message);
|
|
}
|
|
|
|
/**
|
|
* Take a screenshot for debugging.
|
|
* Logs screenshot as base64 for easy viewing from Azure logs.
|
|
*/
|
|
private async _takeScreenshot(name: string): Promise<void> {
|
|
if (!config.screenshotOnError || !this._page) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
// Save to file
|
|
const screenshotDir = config.screenshotDir;
|
|
if (!fs.existsSync(screenshotDir)) {
|
|
fs.mkdirSync(screenshotDir, { recursive: true });
|
|
}
|
|
|
|
const filename = `${this._sessionId}-${name}-${Date.now()}.png`;
|
|
const filepath = path.join(screenshotDir, filename);
|
|
const buffer = await this._page.screenshot({ fullPage: true });
|
|
fs.writeFileSync(filepath, buffer);
|
|
this._logger.info(`Screenshot saved: ${filepath}`);
|
|
|
|
// Also log as base64 for Azure logs (truncated for readability)
|
|
const base64 = buffer.toString('base64');
|
|
this._logger.info(`SCREENSHOT_BASE64_START:${name}`);
|
|
// Log in chunks to avoid log line limits
|
|
const chunkSize = 50000;
|
|
for (let i = 0; i < base64.length; i += chunkSize) {
|
|
this._logger.info(`SCREENSHOT_CHUNK:${base64.substring(i, i + chunkSize)}`);
|
|
}
|
|
this._logger.info(`SCREENSHOT_BASE64_END:${name}`);
|
|
} catch (error) {
|
|
this._logger.error('Error taking screenshot:', error);
|
|
}
|
|
}
|
|
}
|