1369 lines
46 KiB
TypeScript
1369 lines
46 KiB
TypeScript
import { Browser, BrowserContext, Page, ElementHandle, chromium } from 'playwright';
|
|
import { Logger } from 'winston';
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
import path from 'path';
|
|
import fs from 'fs';
|
|
import WebSocket from 'ws';
|
|
|
|
import { config } from '../config';
|
|
import { createSessionLogger } from '../utils/logger';
|
|
import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage, TtsPlaybackAckMessage } from '../types';
|
|
import { JoinProcedure } from './joinProcedure';
|
|
import { CaptionsProcedure } from './captionsProcedure';
|
|
import { AudioProcedure } from './audioProcedure';
|
|
import { AudioCaptureProcedure } from './audioCaptureProcedure';
|
|
import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
|
|
import { AuthProcedure } from './authProcedure';
|
|
import { TeamsActionsService } from './teamsActionsService';
|
|
import { isValidMeetingUrl, getMeetingLaunchUrl, resolveLaunchUrl } from './meetingUrlParser';
|
|
|
|
// Camera / fake video injection is disabled for now to focus on stability.
|
|
// The Y4M fake video file was causing browser crashes when audio started flowing.
|
|
|
|
export interface OrchestratorCallbacks {
|
|
onStateChange: (state: BotState, message?: string) => void;
|
|
onTranscript: (entry: TranscriptEntry) => void;
|
|
onError: (error: Error) => void;
|
|
}
|
|
|
|
export interface OrchestratorOptions {
|
|
gatewayWsUrl: string;
|
|
instanceId: string;
|
|
language?: string;
|
|
botAccountEmail?: string;
|
|
botAccountPassword?: string;
|
|
transferMode?: string;
|
|
debugMode?: boolean;
|
|
}
|
|
|
|
/**
|
|
* Orchestrates the entire bot lifecycle:
|
|
* - Connects to Gateway via WebSocket
|
|
* - Launches browser
|
|
* - Joins meeting
|
|
* - Enables captions
|
|
* - Sends transcripts to Gateway
|
|
* - Handles audio playback from Gateway
|
|
* - Leaves meeting
|
|
*/
|
|
export class BotOrchestrator {
|
|
private _sessionId: string;
|
|
private _meetingUrl: string;
|
|
private _botName: string;
|
|
private _logger: Logger;
|
|
private _callbacks: OrchestratorCallbacks;
|
|
private _options: OrchestratorOptions;
|
|
|
|
private _browser: Browser | null = null;
|
|
private _context: BrowserContext | null = null;
|
|
private _page: Page | null = null;
|
|
private _gatewayWs: WebSocket | null = null;
|
|
private _useHttpFallback: boolean = false;
|
|
private _httpBaseUrl: string = '';
|
|
|
|
private _joinProcedure: JoinProcedure | null = null;
|
|
private _captionsProcedure: CaptionsProcedure | null = null;
|
|
private _audioProcedure: AudioProcedure | null = null;
|
|
private _audioCaptureProcedure: AudioCaptureProcedure | null = null;
|
|
private _chatProcedure: ChatProcedure | null = null;
|
|
private _teamsActions: TeamsActionsService | null = null;
|
|
|
|
private _state: BotState = 'idle';
|
|
private _isShuttingDown: boolean = false;
|
|
private _isDebugMode: boolean = false;
|
|
private _keepAliveInterval: NodeJS.Timeout | null = null;
|
|
|
|
constructor(
|
|
sessionId: string,
|
|
meetingUrl: string,
|
|
botName: string,
|
|
callbacks: OrchestratorCallbacks,
|
|
options: OrchestratorOptions
|
|
) {
|
|
this._sessionId = sessionId;
|
|
this._meetingUrl = meetingUrl;
|
|
this._botName = botName || config.botName;
|
|
this._callbacks = callbacks;
|
|
this._options = options;
|
|
this._isDebugMode = !!options.debugMode;
|
|
this._logger = createSessionLogger(sessionId);
|
|
}
|
|
|
|
/**
|
|
* Poll for a DOM element matching any of the given selectors.
|
|
* Checks every 500ms until found or timeout is reached.
|
|
* Returns the element handle if found, or null on timeout.
|
|
*/
|
|
private async _pollForElement(
|
|
selectors: string | string[],
|
|
timeoutMs: number = 15000,
|
|
label?: string,
|
|
): Promise<ElementHandle | null> {
|
|
const selectorList = Array.isArray(selectors) ? selectors : [selectors];
|
|
const combined = selectorList.join(', ');
|
|
const tag = label || combined.substring(0, 60);
|
|
const deadline = Date.now() + timeoutMs;
|
|
|
|
while (Date.now() < deadline) {
|
|
for (const selector of selectorList) {
|
|
try {
|
|
const el = await this._page!.$(selector);
|
|
if (el) {
|
|
this._logger.info(`[poll] Found "${tag}" via: ${selector}`);
|
|
return el;
|
|
}
|
|
} catch { /* page navigated or element detached — retry */ }
|
|
}
|
|
await this._page!.waitForTimeout(500);
|
|
}
|
|
this._logger.warn(`[poll] "${tag}" not found within ${timeoutMs}ms`);
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Teams launcher commonly embeds meeting params in hash-routed paths:
|
|
* /_#/meet/<id>?p=...&anon=true
|
|
* In this shape, "anon" is in the hash query (not URL.search).
|
|
*/
|
|
private _stripAnonFromInnerMeetingUrl(innerUrlPath: string): string {
|
|
try {
|
|
const innerUrl = new URL(innerUrlPath, 'https://teams.microsoft.com');
|
|
innerUrl.searchParams.delete('anon');
|
|
|
|
const hash = innerUrl.hash || '';
|
|
if (hash.includes('?')) {
|
|
const [hashRoute, hashQuery] = hash.split('?', 2);
|
|
const hashParams = new URLSearchParams(hashQuery || '');
|
|
hashParams.delete('anon');
|
|
const cleanedHashQuery = hashParams.toString();
|
|
innerUrl.hash = cleanedHashQuery ? `${hashRoute}?${cleanedHashQuery}` : hashRoute;
|
|
}
|
|
|
|
return `${innerUrl.pathname}${innerUrl.search}${innerUrl.hash}`;
|
|
} catch {
|
|
return innerUrlPath;
|
|
}
|
|
}
|
|
|
|
get sessionId(): string {
|
|
return this._sessionId;
|
|
}
|
|
|
|
get state(): BotState {
|
|
return this._state;
|
|
}
|
|
|
|
/**
|
|
* Start the bot - connect to Gateway, launch browser, join meeting, enable captions.
|
|
* Chooses between anonymous join and authenticated join based on credentials.
|
|
*/
|
|
async start(): Promise<void> {
|
|
if (!isValidMeetingUrl(this._meetingUrl)) {
|
|
throw new Error(`Invalid meeting URL: ${this._meetingUrl}`);
|
|
}
|
|
|
|
try {
|
|
this._setState('launching');
|
|
|
|
// Connect to Gateway WebSocket first
|
|
await this._connectToGateway();
|
|
|
|
// Choose join method based on credentials
|
|
const hasCredentials = !!(this._options.botAccountEmail && this._options.botAccountPassword);
|
|
if (hasCredentials) {
|
|
this._logger.info(`Authenticated join as: ${this._options.botAccountEmail}`);
|
|
await this._attemptAuthJoin();
|
|
} else {
|
|
this._logger.info('Anonymous join with bot name: ' + this._botName);
|
|
await this._attemptJoin();
|
|
}
|
|
|
|
} catch (error) {
|
|
this._logger.error('Error starting bot:', error);
|
|
this._setState('error', (error as Error).message);
|
|
await this._takeScreenshot('error');
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Join a meeting as anonymous guest with the configured bot name.
|
|
*/
|
|
private async _attemptJoin(): Promise<void> {
|
|
// Launch browser
|
|
await this._launchBrowser();
|
|
|
|
this._setState('navigating');
|
|
|
|
// STEP 1: Navigate to meeting URL and click "Continue on this browser"
|
|
await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl);
|
|
|
|
// Ensure microphone is ON (required for voice playback)
|
|
await this._ensureMicOn();
|
|
|
|
// STEP 2: Enter bot name and click "Join now"
|
|
await this._joinProcedure!.joinMeetingLobbyFlow();
|
|
|
|
// Check if we're in lobby
|
|
const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 10 });
|
|
if (inLobby) {
|
|
this._setState('in_lobby');
|
|
this._logger.info('Bot is in lobby, waiting to be admitted...');
|
|
}
|
|
|
|
// Wait to be admitted to the meeting
|
|
await this._waitForMeetingAdmission();
|
|
|
|
this._setState('in_meeting');
|
|
this._logger.info(`Bot joined the meeting as "${this._botName}"`);
|
|
|
|
// Start keepalive to prevent idle disconnect
|
|
this._startKeepAlive();
|
|
|
|
// Dismiss any post-join permission modals (e.g. "Manage windows on all displays")
|
|
await this._joinProcedure!.dismissBrowserPermissionModals();
|
|
|
|
// Initialize audio playback
|
|
await this._audioProcedure!.initialize();
|
|
|
|
// Enable transcript capture (captions or audio based on transferMode)
|
|
await this._enableTranscriptCapture();
|
|
|
|
// Enable chat monitoring
|
|
await this._enableChat();
|
|
|
|
// Send greeting in meeting chat
|
|
await this._sendJoinGreeting();
|
|
}
|
|
|
|
/**
|
|
* Join a meeting as authenticated user (System Bot or User Account).
|
|
* Flow: teams.microsoft.com → MS Login → Navigate to meeting URL → Pre-Join → Join now
|
|
*
|
|
* Every UI step uses _pollForElement (500ms interval) for both stability and performance:
|
|
* no fixed waits, the flow proceeds as soon as each element appears.
|
|
*/
|
|
private async _attemptAuthJoin(): Promise<void> {
|
|
await this._launchBrowser(true);
|
|
this._setState('navigating');
|
|
|
|
// STEP 1: Navigate to teams.microsoft.com to trigger authentication
|
|
this._logger.info('STEP 1: navigating to teams.microsoft.com');
|
|
await this._page!.goto('https://teams.microsoft.com', {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: 30000,
|
|
});
|
|
|
|
const emailInput = await this._pollForElement(
|
|
['input[name="loginfmt"]', 'input[type="email"]'],
|
|
30000,
|
|
'MS login email input',
|
|
);
|
|
if (!emailInput) {
|
|
this._logger.warn(`No login page found, current URL: ${this._page!.url().substring(0, 150)}`);
|
|
await this._takeScreenshot('step1-no-login-page', this._isDebugMode);
|
|
}
|
|
|
|
// STEP 2: Microsoft Authentication
|
|
this._logger.info(`STEP 2: authenticating as ${this._options.botAccountEmail}`);
|
|
const authProcedure = new AuthProcedure(this._page!, this._logger);
|
|
const authSuccess = await authProcedure.authenticateWithMicrosoft(
|
|
this._options.botAccountEmail!,
|
|
this._options.botAccountPassword!,
|
|
true,
|
|
);
|
|
|
|
if (!authSuccess) {
|
|
await this._takeScreenshot('step2-auth-failed', this._isDebugMode);
|
|
throw new Error('Microsoft authentication failed');
|
|
}
|
|
this._logger.info('STEP 2: authentication successful');
|
|
await this._takeScreenshot('step2-auth-done', this._isDebugMode);
|
|
|
|
// STEP 3: Wait for Teams to load after auth
|
|
this._logger.info('STEP 3: waiting for Teams to load after auth...');
|
|
try {
|
|
await this._page!.waitForURL(
|
|
(url) => url.hostname.includes('teams.microsoft.com') || url.hostname.includes('teams.cloud.microsoft'),
|
|
{ timeout: 30000 },
|
|
);
|
|
} catch {
|
|
this._logger.warn(`Unexpected URL after auth: ${this._page!.url().substring(0, 150)}`);
|
|
}
|
|
await this._takeScreenshot('step3-teams-loaded', this._isDebugMode);
|
|
|
|
// STEP 4: Navigate to the meeting URL with proper launch params.
|
|
// CRITICAL: The suppress params (msLaunch, suppressPrompt, directDl) must
|
|
// be on the LAUNCHER URL itself, NOT inside the encoded meeting URL parameter.
|
|
// resolveLaunchUrl follows redirects first (meeting URL → launcher URL),
|
|
// then adds the params to the RESOLVED launcher URL. getMeetingLaunchUrl
|
|
// adds params to the raw meeting URL — they end up encoded inside the
|
|
// launcher's url= parameter and have no effect on the launcher behavior.
|
|
let launchUrl: string;
|
|
try {
|
|
launchUrl = await resolveLaunchUrl(this._meetingUrl);
|
|
} catch (error) {
|
|
this._logger.warn(`Could not resolve launch URL, using fallback: ${error}`);
|
|
launchUrl = getMeetingLaunchUrl(this._meetingUrl);
|
|
}
|
|
// Remove anon=true since the user is authenticated
|
|
try {
|
|
const urlObj = new URL(launchUrl);
|
|
urlObj.searchParams.delete('anon');
|
|
// Some Teams launcher URLs carry the real meeting path in an encoded "url" param.
|
|
// In auth mode that inner URL can still contain anon=true, which forces guest-like behavior.
|
|
const encodedInnerUrl = urlObj.searchParams.get('url');
|
|
if (encodedInnerUrl) {
|
|
const innerPath = this._stripAnonFromInnerMeetingUrl(encodedInnerUrl);
|
|
urlObj.searchParams.set('url', innerPath);
|
|
}
|
|
launchUrl = urlObj.toString();
|
|
} catch { /* keep as-is */ }
|
|
|
|
this._logger.info(`STEP 4: navigating to launch URL: ${launchUrl.substring(0, 120)}...`);
|
|
this._logger.info(`STEP 4: launch URL contains anon=true? ${launchUrl.includes('anon=true')}`);
|
|
await this._page!.goto(launchUrl, {
|
|
waitUntil: 'domcontentloaded',
|
|
timeout: 30000,
|
|
});
|
|
this._logger.info(`STEP 4: URL after navigation: ${this._page!.url().substring(0, 150)}`);
|
|
await this._takeScreenshot('step4-meeting-url-loaded', this._isDebugMode);
|
|
|
|
// STEP 4a: Poll for first actionable button (interstitial OR pre-join)
|
|
const interstitialSelectors = [
|
|
'button[data-tid="joinOnWeb"]',
|
|
'button:has-text("Continue on this browser")',
|
|
'button:has-text("In diesem Browser fortfahren")',
|
|
'button:has-text("Weiter in diesem Browser")',
|
|
'button:has-text("Join on the web instead")',
|
|
'button:has-text("Use web app instead")',
|
|
'button[data-tid="chat-join-button"]',
|
|
'button[data-tid="join-call-button"]',
|
|
];
|
|
const preJoinSelectors = [
|
|
'button:has-text("Join now")',
|
|
'button:has-text("Jetzt teilnehmen")',
|
|
'button[data-tid="prejoin-join-button"]',
|
|
];
|
|
|
|
const firstBtn = await this._pollForElement(
|
|
[...interstitialSelectors, ...preJoinSelectors],
|
|
30000,
|
|
'interstitial or pre-join button',
|
|
);
|
|
|
|
if (firstBtn) {
|
|
const btnText = (await firstBtn.textContent().catch(() => ''))?.trim() || '';
|
|
const btnTid = (await firstBtn.getAttribute('data-tid').catch(() => '')) || '';
|
|
|
|
const isPreJoin = btnTid === 'prejoin-join-button'
|
|
|| btnText.toLowerCase().includes('join now')
|
|
|| btnText.toLowerCase().includes('jetzt teilnehmen');
|
|
|
|
if (!isPreJoin) {
|
|
await firstBtn.click();
|
|
this._logger.info(`STEP 4a: clicked interstitial: "${btnText}" (data-tid="${btnTid}")`);
|
|
await this._takeScreenshot('step4a-after-interstitial', this._isDebugMode);
|
|
} else {
|
|
this._logger.info(`STEP 4a: pre-join button already visible: "${btnText}"`);
|
|
}
|
|
} else {
|
|
await this._takeScreenshot('step4a-no-buttons-found', this._isDebugMode);
|
|
}
|
|
|
|
// Ensure microphone is ON before joining (required for voice playback)
|
|
await this._ensureMicOn();
|
|
|
|
// STEP 5: Poll for "Join now" on the pre-join screen
|
|
await this._takeScreenshot('step5-before-join-now', this._isDebugMode);
|
|
|
|
const joinNowBtn = await this._pollForElement(preJoinSelectors, 30000, 'Join now button');
|
|
if (!joinNowBtn) {
|
|
await this._takeScreenshot('step5-no-join-now', this._isDebugMode);
|
|
throw new Error('"Join now" button not found on pre-join screen');
|
|
}
|
|
await joinNowBtn.click();
|
|
this._logger.info('STEP 5: clicked "Join now", waiting for meeting');
|
|
await this._takeScreenshot('step5-join-now-clicked', this._isDebugMode);
|
|
|
|
// STEP 6: Wait for meeting admission (hangup button = in meeting)
|
|
await this._waitForMeetingAdmission();
|
|
|
|
this._setState('in_meeting');
|
|
this._logger.info(`STEP 6: bot joined the meeting (authenticated as ${this._options.botAccountEmail})`);
|
|
await this._takeScreenshot('step6-in-meeting', this._isDebugMode);
|
|
|
|
this._startKeepAlive();
|
|
await this._audioProcedure!.initialize();
|
|
await this._enableTranscriptCapture();
|
|
await this._enableChat();
|
|
await this._sendJoinGreeting();
|
|
}
|
|
|
|
/**
|
|
* Ensure the camera is turned on in the pre-join screen.
|
|
* When camera is on, Teams shows the profile/background image.
|
|
*
|
|
* Teams pre-join uses a fui-Switch input:
|
|
* <input data-tid="toggle-video" role="switch" type="checkbox" checked>
|
|
* - checked present = camera ON (data-cid="toggle-video-true", title="Turn camera off")
|
|
* - checked absent = camera OFF (data-cid="toggle-video-false", title="Turn camera on")
|
|
*/
|
|
private async _ensureCameraOn(): Promise<void> {
|
|
try {
|
|
const cameraToggle = await this._pollForElement([
|
|
'input[data-tid="toggle-video"]',
|
|
'[data-tid="toggle-video"]',
|
|
'input[role="switch"][title*="camera" i]',
|
|
'input[role="switch"][title*="Camera" i]',
|
|
'input[role="switch"][title*="Video" i]',
|
|
], 10000, 'camera toggle (pre-join)');
|
|
|
|
if (!cameraToggle) return;
|
|
|
|
const state = await cameraToggle.evaluate((el: HTMLInputElement) => ({
|
|
checked: el.checked,
|
|
dataCid: el.getAttribute('data-cid') || '',
|
|
title: el.getAttribute('title') || '',
|
|
}));
|
|
this._logger.info(`Camera state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
|
|
|
|
if (!state.checked) {
|
|
await cameraToggle.click();
|
|
this._logger.info('Camera toggled ON');
|
|
} else {
|
|
this._logger.info('Camera already ON');
|
|
}
|
|
} catch (err) {
|
|
this._logger.warn(`Could not toggle camera: ${err}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Verify camera is on after joining the meeting, and turn it on if not.
|
|
*
|
|
* In-meeting camera button (from Teams DOM):
|
|
* <button id="video-button" data-state="call-video-off" aria-label="Turn camera on">
|
|
* <button id="video-button" data-state="call-video-on" aria-label="Turn camera off">
|
|
*/
|
|
private async _ensureCameraOnInMeeting(): Promise<void> {
|
|
try {
|
|
const videoBtn = await this._pollForElement([
|
|
'button#video-button',
|
|
'button[data-inp="video-button"]',
|
|
'button[aria-label*="camera" i]',
|
|
'button[aria-label*="Camera" i]',
|
|
'button[aria-label*="Video" i]',
|
|
], 10000, 'in-meeting camera button');
|
|
|
|
if (!videoBtn) return;
|
|
|
|
const state = await videoBtn.evaluate((el: HTMLElement) => ({
|
|
dataState: el.getAttribute('data-state') || '',
|
|
ariaLabel: el.getAttribute('aria-label') || '',
|
|
}));
|
|
this._logger.info(`In-meeting camera: data-state="${state.dataState}", aria-label="${state.ariaLabel}"`);
|
|
|
|
const isOff = state.dataState === 'call-video-off'
|
|
|| state.ariaLabel.toLowerCase().includes('turn camera on')
|
|
|| state.ariaLabel.toLowerCase().includes('kamera einschalten');
|
|
|
|
if (isOff) {
|
|
await videoBtn.click();
|
|
this._logger.info('In-meeting camera was OFF — clicked to turn ON');
|
|
} else {
|
|
this._logger.info('In-meeting camera already ON');
|
|
}
|
|
} catch (err) {
|
|
this._logger.warn(`Could not verify in-meeting camera: ${err}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ensure the microphone is turned on in the pre-join screen.
|
|
* Required for voice playback (TTS audio is injected into the mic stream).
|
|
*
|
|
* Teams pre-join uses a fui-Switch input:
|
|
* <input data-tid="toggle-audio" role="switch" type="checkbox" checked>
|
|
* - checked present = mic ON
|
|
* - checked absent = mic OFF
|
|
*/
|
|
private async _ensureMicOn(): Promise<void> {
|
|
try {
|
|
const micToggle = await this._pollForElement([
|
|
'input[data-tid="toggle-audio"]',
|
|
'[data-tid="toggle-audio"]',
|
|
'input[role="switch"][title*="microphone" i]',
|
|
'input[role="switch"][title*="Mikrofon" i]',
|
|
'input[role="switch"][title*="mic" i]',
|
|
'input[role="switch"][title*="audio" i]',
|
|
], 10000, 'mic toggle (pre-join)');
|
|
|
|
if (!micToggle) return;
|
|
|
|
const state = await micToggle.evaluate((el: HTMLInputElement) => ({
|
|
checked: el.checked,
|
|
dataCid: el.getAttribute('data-cid') || '',
|
|
title: el.getAttribute('title') || '',
|
|
}));
|
|
this._logger.info(`Mic state: checked=${state.checked}, data-cid="${state.dataCid}", title="${state.title}"`);
|
|
|
|
if (!state.checked) {
|
|
await micToggle.click();
|
|
this._logger.info('Mic toggled ON');
|
|
} else {
|
|
this._logger.info('Mic already ON');
|
|
}
|
|
} catch (err) {
|
|
this._logger.warn(`Could not toggle mic: ${err}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start a keepalive timer that periodically moves the mouse and sends
|
|
* a WebSocket ping. Prevents Teams from detecting the bot as idle
|
|
* and kicking it from the meeting.
|
|
*/
|
|
private _startKeepAlive(): void {
|
|
if (this._keepAliveInterval) return;
|
|
|
|
this._keepAliveInterval = setInterval(async () => {
|
|
if (this._isShuttingDown || !this._page) return;
|
|
|
|
try {
|
|
// Small random mouse movement to simulate user activity
|
|
const x = 640 + Math.floor(Math.random() * 20 - 10);
|
|
const y = 360 + Math.floor(Math.random() * 20 - 10);
|
|
await this._page.mouse.move(x, y);
|
|
} catch {
|
|
// Page might be closed
|
|
}
|
|
|
|
// WebSocket heartbeat
|
|
if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) {
|
|
try {
|
|
this._gatewayWs.send(JSON.stringify({ type: 'ping', sessionId: this._sessionId }));
|
|
} catch {
|
|
// Connection might be closing
|
|
}
|
|
}
|
|
}, 15000);
|
|
|
|
this._logger.info('Keepalive started (15s interval)');
|
|
}
|
|
|
|
/**
|
|
* Stop the keepalive timer.
|
|
*/
|
|
private _stopKeepAlive(): void {
|
|
if (this._keepAliveInterval) {
|
|
clearInterval(this._keepAliveInterval);
|
|
this._keepAliveInterval = null;
|
|
this._logger.info('Keepalive stopped');
|
|
}
|
|
}
|
|
|
|
private _wsReconnectAttempts: number = 0;
|
|
private _wsMaxReconnectAttempts: number = 10;
|
|
private _wsReconnecting: boolean = false;
|
|
|
|
/**
|
|
* Connect to the Gateway WebSocket for this session.
|
|
*/
|
|
private async _connectToGateway(): Promise<void> {
|
|
const wsUrl = this._options.gatewayWsUrl;
|
|
this._logger.info(`Connecting to Gateway: ${wsUrl}`);
|
|
|
|
this._httpBaseUrl = wsUrl
|
|
.replace('wss://', 'https://')
|
|
.replace('ws://', 'http://')
|
|
.replace(/\/bot\/ws\/.*$/, '');
|
|
|
|
return this._createWsConnection(wsUrl, true);
|
|
}
|
|
|
|
/**
|
|
* Create (or recreate) the WebSocket connection.
|
|
* On initial connect, `isInitial` = true and the promise resolves/rejects.
|
|
* On reconnect, the promise resolves immediately (fire-and-forget).
|
|
*/
|
|
private _createWsConnection(wsUrl: string, isInitial: boolean): Promise<void> {
|
|
return new Promise((resolve) => {
|
|
this._gatewayWs = new WebSocket(wsUrl);
|
|
|
|
const wsTimeout = setTimeout(() => {
|
|
if (this._gatewayWs?.readyState !== WebSocket.OPEN) {
|
|
this._logger.warn('WebSocket connection timeout - switching to HTTP fallback');
|
|
this._useHttpFallback = true;
|
|
this._gatewayWs?.close();
|
|
this._gatewayWs = null;
|
|
resolve();
|
|
}
|
|
}, 10000);
|
|
|
|
this._gatewayWs.on('open', () => {
|
|
clearTimeout(wsTimeout);
|
|
this._logger.info('Connected to Gateway via WebSocket');
|
|
this._useHttpFallback = false;
|
|
this._wsReconnectAttempts = 0;
|
|
this._wsReconnecting = false;
|
|
resolve();
|
|
});
|
|
|
|
this._gatewayWs.on('message', (data) => {
|
|
this._handleGatewayMessage(data.toString()).catch((err) => {
|
|
this._logger.error('Unhandled error in gateway message handler:', err);
|
|
});
|
|
});
|
|
|
|
this._gatewayWs.on('close', (code, reason) => {
|
|
this._logger.warn(`Gateway WebSocket closed: ${code} - ${reason}`);
|
|
if (!this._isShuttingDown) {
|
|
this._useHttpFallback = true;
|
|
this._scheduleReconnect(wsUrl);
|
|
}
|
|
});
|
|
|
|
this._gatewayWs.on('error', (error) => {
|
|
clearTimeout(wsTimeout);
|
|
this._logger.error('Gateway WebSocket error:', error);
|
|
this._useHttpFallback = true;
|
|
this._gatewayWs = null;
|
|
if (isInitial) resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Schedule a WebSocket reconnection with exponential backoff.
|
|
*/
|
|
private _scheduleReconnect(wsUrl: string): void {
|
|
if (this._isShuttingDown || this._wsReconnecting) return;
|
|
if (this._wsReconnectAttempts >= this._wsMaxReconnectAttempts) {
|
|
this._logger.warn(`WebSocket reconnect limit reached (${this._wsMaxReconnectAttempts}), staying on HTTP fallback`);
|
|
return;
|
|
}
|
|
|
|
this._wsReconnecting = true;
|
|
this._wsReconnectAttempts++;
|
|
const delayMs = Math.min(2000 * Math.pow(1.5, this._wsReconnectAttempts - 1), 30000);
|
|
this._logger.info(`WebSocket reconnect attempt ${this._wsReconnectAttempts}/${this._wsMaxReconnectAttempts} in ${(delayMs / 1000).toFixed(1)}s`);
|
|
|
|
setTimeout(() => {
|
|
if (this._isShuttingDown) return;
|
|
this._createWsConnection(wsUrl, false).catch((err) => {
|
|
this._logger.error('WebSocket reconnect failed:', err);
|
|
});
|
|
}, delayMs);
|
|
}
|
|
|
|
/**
|
|
* Handle incoming messages from the Gateway.
|
|
* Async operations are awaited to ensure proper error handling
|
|
* and serialized execution (e.g. audio + chat don't interfere).
|
|
*/
|
|
private async _handleGatewayMessage(data: string): Promise<void> {
|
|
try {
|
|
const message = JSON.parse(data);
|
|
|
|
switch (message.type) {
|
|
case 'playAudio':
|
|
const audioMsg = message as PlayAudioMessage;
|
|
await this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
|
|
break;
|
|
|
|
case 'sendChatMessage':
|
|
const chatMsg = message as SendChatMessage;
|
|
this._logger.info(`Gateway sendChatMessage received: ${chatMsg.text?.substring(0, 60)}...`);
|
|
try {
|
|
await this.sendChatMessageToMeeting(chatMsg.text);
|
|
} catch (chatErr) {
|
|
this._logger.error(`Failed to send chat message to meeting: ${chatErr}`);
|
|
}
|
|
break;
|
|
|
|
case 'stopAudio':
|
|
this._logger.info('Stop audio command received from Gateway');
|
|
if (this._audioProcedure) {
|
|
this._audioProcedure.stopAllAudio();
|
|
}
|
|
break;
|
|
|
|
case 'botCommand':
|
|
await this._handleBotCommand(message.command, message.params || {});
|
|
break;
|
|
|
|
case 'pong':
|
|
break;
|
|
|
|
default:
|
|
this._logger.debug('Unknown Gateway message type:', message.type);
|
|
}
|
|
} catch (error) {
|
|
this._logger.error('Error handling Gateway message:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a message to the Gateway (WebSocket or HTTP fallback).
|
|
*/
|
|
private _sendToGateway(message: object): void {
|
|
if (this._gatewayWs && this._gatewayWs.readyState === WebSocket.OPEN) {
|
|
try {
|
|
this._gatewayWs.send(JSON.stringify(message));
|
|
return;
|
|
} catch (error) {
|
|
this._logger.error('WebSocket send error, falling back to HTTP:', error);
|
|
this._useHttpFallback = true;
|
|
}
|
|
}
|
|
|
|
// HTTP fallback
|
|
if (this._useHttpFallback) {
|
|
this._sendViaHttp(message);
|
|
} else {
|
|
this._logger.warn('Cannot send to Gateway - no WebSocket and no HTTP fallback');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a message via HTTP POST (fallback when WebSocket unavailable).
|
|
*/
|
|
private async _sendViaHttp(message: any): Promise<void> {
|
|
const msgType = message.type;
|
|
let url = '';
|
|
|
|
if (msgType === 'transcript') {
|
|
url = `${this._httpBaseUrl}/bot/transcript/${this._sessionId}`;
|
|
} else if (msgType === 'status') {
|
|
url = `${this._httpBaseUrl}/bot/status/${this._sessionId}`;
|
|
} else if (msgType === 'audioChunk') {
|
|
// Audio chunks are too frequent for HTTP — only send via WebSocket
|
|
return;
|
|
} else {
|
|
this._logger.debug(`HTTP fallback: unsupported message type ${msgType}`);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(message),
|
|
});
|
|
if (!response.ok) {
|
|
this._logger.warn(`HTTP fallback response: ${response.status} ${response.statusText}`);
|
|
}
|
|
} catch (error) {
|
|
this._logger.error(`HTTP fallback error for ${msgType}:`, error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a transcript to the Gateway.
|
|
*/
|
|
private _sendTranscript(
|
|
speaker: string,
|
|
text: string,
|
|
isFinal: boolean,
|
|
source: 'caption' | 'audioCapture' | 'speakerHint' | 'chat' = 'caption',
|
|
): void {
|
|
const message: TranscriptMessage = {
|
|
type: 'transcript',
|
|
sessionId: this._sessionId,
|
|
transcript: {
|
|
speaker,
|
|
text,
|
|
timestamp: new Date().toISOString(),
|
|
isFinal,
|
|
source,
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send a status update to the Gateway.
|
|
*/
|
|
private _sendStatus(status: StatusMessage['status'], message?: string): void {
|
|
const statusMessage: StatusMessage = {
|
|
type: 'status',
|
|
sessionId: this._sessionId,
|
|
status,
|
|
message,
|
|
};
|
|
this._sendToGateway(statusMessage);
|
|
}
|
|
|
|
/**
|
|
* Stop the bot - leave meeting, close browser, disconnect from Gateway.
|
|
*/
|
|
async stop(): Promise<void> {
|
|
if (this._isShuttingDown) {
|
|
return;
|
|
}
|
|
|
|
this._isShuttingDown = true;
|
|
this._logger.info('Stopping bot...');
|
|
|
|
// Stop keepalive first
|
|
this._stopKeepAlive();
|
|
|
|
try {
|
|
this._setState('leaving');
|
|
|
|
// Stop audio capture
|
|
if (this._audioCaptureProcedure) {
|
|
await this._audioCaptureProcedure.stopCapture();
|
|
}
|
|
|
|
// Unsubscribe from captions and chat
|
|
if (this._captionsProcedure) {
|
|
await this._captionsProcedure.unsubscribe();
|
|
}
|
|
if (this._chatProcedure) {
|
|
await this._chatProcedure.unsubscribe();
|
|
}
|
|
|
|
// Clean up audio playback
|
|
if (this._audioProcedure) {
|
|
await this._audioProcedure.cleanup();
|
|
}
|
|
|
|
// Leave the meeting
|
|
if (this._joinProcedure && this._state !== 'error') {
|
|
await this._joinProcedure.leaveMeetingFlow();
|
|
}
|
|
|
|
} catch (error) {
|
|
this._logger.error('Error during shutdown:', error);
|
|
} finally {
|
|
// Close browser
|
|
await this._closeBrowser();
|
|
|
|
// Close Gateway connection
|
|
if (this._gatewayWs) {
|
|
this._gatewayWs.close(1000, 'Bot stopping');
|
|
this._gatewayWs = null;
|
|
}
|
|
|
|
this._setState('disconnected');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Play audio in the meeting.
|
|
*/
|
|
async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise<void> {
|
|
if (this._isShuttingDown) {
|
|
this._logger.debug('Ignoring playAudio - bot is shutting down');
|
|
return;
|
|
}
|
|
if (this._state !== 'in_meeting' || !this._audioProcedure) {
|
|
this._logger.warn('Cannot play audio - not in meeting');
|
|
return;
|
|
}
|
|
|
|
this._sendTtsPlaybackAck('queued', format, audioData.length, 'Audio queued for playback');
|
|
try {
|
|
await this._audioProcedure.playAudio(audioData, format);
|
|
this._sendTtsPlaybackAck('completed', format, audioData.length, 'Audio playback completed');
|
|
} catch (error) {
|
|
this._sendTtsPlaybackAck('failed', format, audioData.length, String(error));
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private _sendTtsPlaybackAck(
|
|
status: 'queued' | 'completed' | 'failed',
|
|
format: 'mp3' | 'wav' | 'pcm',
|
|
bytesBase64?: number,
|
|
message?: string,
|
|
): void {
|
|
const ack: TtsPlaybackAckMessage = {
|
|
type: 'ttsPlaybackAck',
|
|
sessionId: this._sessionId,
|
|
playback: {
|
|
status,
|
|
format,
|
|
bytesBase64,
|
|
message,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
};
|
|
this._sendToGateway(ack);
|
|
}
|
|
|
|
/**
|
|
* Launch the browser and create a new page.
|
|
* @param authMode - If true, use headful + minimal args (Chromium Minimal, proven to work for auth)
|
|
*/
|
|
private async _launchBrowser(authMode: boolean = false): Promise<void> {
|
|
this._logger.info(`Launching browser (authMode=${authMode})...`);
|
|
|
|
const args = authMode
|
|
? [
|
|
'--no-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-gpu',
|
|
'--use-fake-ui-for-media-stream',
|
|
'--use-fake-device-for-media-stream',
|
|
'--autoplay-policy=no-user-gesture-required',
|
|
]
|
|
: [
|
|
'--disable-dev-shm-usage',
|
|
'--disable-gpu',
|
|
'--use-fake-ui-for-media-stream',
|
|
'--use-fake-device-for-media-stream',
|
|
'--disable-web-security',
|
|
'--disable-features=IsolateOrigins,site-per-process',
|
|
'--autoplay-policy=no-user-gesture-required',
|
|
'--disable-blink-features=AutomationControlled',
|
|
];
|
|
|
|
this._browser = await chromium.launch({
|
|
headless: authMode ? false : config.botHeadless,
|
|
args,
|
|
});
|
|
|
|
this._context = await this._browser.newContext({
|
|
permissions: ['microphone', 'camera'],
|
|
viewport: { width: 1280, height: 720 },
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
|
|
});
|
|
|
|
this._page = await this._context.newPage();
|
|
this._page.on('console', (msg) => {
|
|
const text = msg.text();
|
|
if (text.includes('[AudioCapture]') || text.includes('[AudioPlayback]')) {
|
|
this._logger.info(`[PageConsole] ${text}`);
|
|
}
|
|
});
|
|
|
|
// Stealth: Override browser properties that reveal automation.
|
|
// Teams checks these to detect headless/automated browsers and
|
|
// blocks the /v2/ authenticated experience, falling back to light-meetings.
|
|
await this._page.addInitScript(() => {
|
|
// 1. Remove navigator.webdriver flag (primary detection signal)
|
|
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
|
|
|
// 2. Add realistic plugins (headless has empty plugins array)
|
|
Object.defineProperty(navigator, 'plugins', {
|
|
get: () => [
|
|
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
|
|
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
|
|
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
|
|
],
|
|
});
|
|
|
|
// 3. Add realistic languages
|
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en', 'de'] });
|
|
|
|
// 4. Override permissions query to not reveal automation
|
|
const originalQuery = window.navigator.permissions.query.bind(window.navigator.permissions);
|
|
// @ts-ignore
|
|
window.navigator.permissions.query = (parameters: any) => {
|
|
if (parameters.name === 'notifications') {
|
|
return Promise.resolve({ state: Notification.permission } as PermissionStatus);
|
|
}
|
|
return originalQuery(parameters);
|
|
};
|
|
|
|
// 5. Add chrome runtime (missing in headless)
|
|
// @ts-ignore
|
|
if (!window.chrome) { window.chrome = {}; }
|
|
// @ts-ignore
|
|
if (!window.chrome.runtime) { window.chrome.runtime = {}; }
|
|
});
|
|
|
|
// Initialize procedures
|
|
this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName);
|
|
this._audioCaptureProcedure = new AudioCaptureProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(base64Data, sampleRate, captureDiagnostics) => {
|
|
this._sendAudioChunk(base64Data, sampleRate, captureDiagnostics);
|
|
},
|
|
);
|
|
this._captionsProcedure = new CaptionsProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(entry) => {
|
|
// Aggressive hybrid mode: captions are always speaker hints only.
|
|
// Caption text is never persisted as transcript due to language quality issues.
|
|
this._sendTranscript(entry.speaker, entry.text, entry.isFinal, 'speakerHint');
|
|
},
|
|
this._options.language
|
|
);
|
|
this._audioProcedure = new AudioProcedure(this._page, this._logger);
|
|
this._teamsActions = new TeamsActionsService(this._page, this._logger);
|
|
this._chatProcedure = new ChatProcedure(
|
|
this._page,
|
|
this._logger,
|
|
(entry: ChatMessageEntry) => {
|
|
// Send chat message to Gateway as a special transcript
|
|
this._sendChatMessage(entry.speaker, entry.text);
|
|
this._callbacks.onTranscript({
|
|
speaker: entry.speaker,
|
|
text: entry.text,
|
|
timestamp: entry.timestamp,
|
|
isFinal: true,
|
|
});
|
|
}
|
|
);
|
|
|
|
// Inject audio getUserMedia override BEFORE any navigation
|
|
// This ensures Teams gets our controlled audio stream when it calls getUserMedia
|
|
await this._audioProcedure.injectAudioOverride();
|
|
|
|
// Aggressive hybrid mode: always capture meeting audio as transcript source.
|
|
await this._audioCaptureProcedure!.injectCaptureOverride();
|
|
|
|
// Handle page errors
|
|
this._page.on('pageerror', (error) => {
|
|
this._logger.error('Page error:', error);
|
|
});
|
|
|
|
// Handle page close
|
|
this._page.on('close', () => {
|
|
if (!this._isShuttingDown) {
|
|
this._logger.warn('Page closed unexpectedly');
|
|
this._setState('disconnected');
|
|
}
|
|
});
|
|
|
|
// Handle browser renderer crash (Chromium process segfault)
|
|
this._page.on('crash', () => {
|
|
this._logger.error('BROWSER CRASH: Chromium renderer process crashed!');
|
|
this._setState('error', 'Browser crashed');
|
|
});
|
|
|
|
// Handle browser disconnection (entire browser process dies)
|
|
this._browser.on('disconnected', () => {
|
|
if (!this._isShuttingDown) {
|
|
this._logger.error('BROWSER DISCONNECTED: Browser process died unexpectedly');
|
|
this._setState('error', 'Browser process died');
|
|
}
|
|
});
|
|
|
|
this._logger.info('Browser launched');
|
|
}
|
|
|
|
/**
|
|
* Close the browser.
|
|
*/
|
|
private async _closeBrowser(): Promise<void> {
|
|
try {
|
|
if (this._page) {
|
|
await this._page.close();
|
|
}
|
|
if (this._context) {
|
|
await this._context.close();
|
|
}
|
|
if (this._browser) {
|
|
await this._browser.close();
|
|
}
|
|
} catch (error) {
|
|
this._logger.error('Error closing browser:', error);
|
|
}
|
|
|
|
this._page = null;
|
|
this._context = null;
|
|
this._browser = null;
|
|
this._logger.info('Browser closed');
|
|
}
|
|
|
|
/**
|
|
* Wait for the bot to be admitted from the lobby.
|
|
*/
|
|
private async _waitForMeetingAdmission(): Promise<void> {
|
|
const startTime = Date.now();
|
|
const timeout = config.timeouts.lobbyWait;
|
|
let consecutiveNoSignal = 0;
|
|
const maxNoSignal = 5; // Allow several cycles with no lobby/meeting signal before giving up
|
|
|
|
while (Date.now() - startTime < timeout) {
|
|
// Check if we're in the meeting
|
|
const inMeeting = await this._joinProcedure!.isInMeeting({ waitForSeconds: 5 });
|
|
if (inMeeting) {
|
|
return;
|
|
}
|
|
|
|
// Check if still in lobby
|
|
const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 2 });
|
|
if (inLobby) {
|
|
consecutiveNoSignal = 0;
|
|
this._logger.info('Still waiting in lobby...');
|
|
continue;
|
|
}
|
|
|
|
// Neither in meeting nor in lobby — this can happen legitimately:
|
|
// - Authenticated users skip lobby, but meeting UI takes seconds to load
|
|
// - Page is transitioning between states
|
|
// Only give up after several consecutive cycles with no signal
|
|
consecutiveNoSignal++;
|
|
const currentUrl = this._page?.url() || 'unknown';
|
|
this._logger.info(`No lobby/meeting signal detected (attempt ${consecutiveNoSignal}/${maxNoSignal}), URL: ${currentUrl}`);
|
|
|
|
if (consecutiveNoSignal >= maxNoSignal) {
|
|
// Take a screenshot and log page content for debugging before giving up
|
|
await this._takeScreenshot('no-meeting-signal');
|
|
try {
|
|
const bodySnippet = await this._page?.evaluate(() =>
|
|
document.body?.innerText?.substring(0, 500) || '(empty)'
|
|
);
|
|
this._logger.warn(`Page content before giving up: ${bodySnippet}`);
|
|
} catch { /* ignore */ }
|
|
throw new Error('Bot was removed from lobby or meeting ended');
|
|
}
|
|
}
|
|
|
|
throw new Error('Timeout waiting to be admitted from lobby');
|
|
}
|
|
|
|
/**
|
|
* Determine the effective transfer mode based on config and join mode.
|
|
* auto: anonymous → audio, authenticated → caption
|
|
*/
|
|
private _getEffectiveTransferMode(): 'caption' | 'audio' {
|
|
const mode = this._options.transferMode || 'auto';
|
|
if (mode === 'caption') return 'caption';
|
|
if (mode === 'audio') return 'audio';
|
|
// auto: use audio for anonymous (Teams only provides English captions), caption for auth
|
|
const isAuth = !!(this._options.botAccountEmail && this._options.botAccountPassword);
|
|
return isAuth ? 'caption' : 'audio';
|
|
}
|
|
|
|
/**
|
|
* Enable captions and start scraping.
|
|
*/
|
|
private async _enableCaptions(): Promise<void> {
|
|
try {
|
|
await this._captionsProcedure!.enableCaptionsFlow();
|
|
await this._captionsProcedure!.subscribeToCaptions();
|
|
this._logger.info('Captions enabled and subscribed');
|
|
} catch (error) {
|
|
this._logger.warn('Could not enable captions:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enable audio capture from meeting participants.
|
|
*/
|
|
private async _enableAudioCapture(): Promise<void> {
|
|
if (!this._audioCaptureProcedure) {
|
|
this._logger.warn('Audio capture procedure not initialized');
|
|
return;
|
|
}
|
|
try {
|
|
await this._audioCaptureProcedure.startCapture();
|
|
this._logger.info('Audio capture started (PCM16 16kHz mono)');
|
|
} catch (error) {
|
|
this._logger.warn('Could not start audio capture:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Enable transcript capture (captions or audio) based on transfer mode.
|
|
*/
|
|
private async _enableTranscriptCapture(): Promise<void> {
|
|
this._logger.info(
|
|
`Aggressive hybrid mode active: audio STT + background captions for speaker hints (configured transferMode: ${this._options.transferMode || 'auto'})`,
|
|
);
|
|
await this._enableAudioCapture();
|
|
await this._enableCaptions();
|
|
}
|
|
|
|
/**
|
|
* Enable chat monitoring.
|
|
*/
|
|
private async _enableChat(): Promise<void> {
|
|
try {
|
|
await this._chatProcedure!.enableChatMonitoring();
|
|
await this._chatProcedure!.subscribeToChatMessages();
|
|
this._logger.info('Chat monitoring enabled and subscribed');
|
|
} catch (error) {
|
|
this._logger.warn('Could not enable chat monitoring:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a greeting message in the meeting chat AND via voice after joining.
|
|
* Uses the bot's display name and the configured language.
|
|
* Voice greeting confirms that the audio pipeline (TTS -> mic) is working.
|
|
*/
|
|
private async _sendJoinGreeting(): Promise<void> {
|
|
try {
|
|
const firstName = this._botName.split(' ')[0] || this._botName;
|
|
const lang = (this._options.language || 'de-DE').toLowerCase();
|
|
|
|
let greeting: string;
|
|
if (lang.startsWith('de')) {
|
|
greeting = `Hallo, hier ist ${firstName}. Ich bin bereit.`;
|
|
} else if (lang.startsWith('fr')) {
|
|
greeting = `Bonjour, c'est ${firstName}. Je suis prête.`;
|
|
} else if (lang.startsWith('it')) {
|
|
greeting = `Ciao, sono ${firstName}. Sono pronta.`;
|
|
} else {
|
|
greeting = `Hello, this is ${firstName}. I'm ready.`;
|
|
}
|
|
|
|
this._logger.info(`Sending join greeting (chat + voice): ${greeting}`);
|
|
|
|
// Chat greeting
|
|
await this.sendChatMessageToMeeting(greeting);
|
|
|
|
// Voice greeting — ask Gateway to generate TTS and send back playAudio
|
|
this._sendToGateway({
|
|
type: 'voiceGreeting',
|
|
sessionId: this._sessionId,
|
|
text: greeting,
|
|
language: this._options.language || 'de-DE',
|
|
});
|
|
} catch (error) {
|
|
this._logger.warn('Could not send join greeting:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a chat message event to the Gateway.
|
|
*/
|
|
private _sendChatMessage(speaker: string, text: string): void {
|
|
const message: ChatMessage = {
|
|
type: 'chatMessage',
|
|
sessionId: this._sessionId,
|
|
chat: {
|
|
speaker,
|
|
text,
|
|
timestamp: new Date().toISOString(),
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send an audio chunk to the Gateway for STT processing.
|
|
*/
|
|
private _sendAudioChunk(
|
|
base64Data: string,
|
|
sampleRate: number,
|
|
captureDiagnostics?: {
|
|
trackId?: string;
|
|
readyState?: string;
|
|
rms?: number;
|
|
nativeSampleRate?: number;
|
|
},
|
|
): void {
|
|
const message: AudioChunkMessage = {
|
|
type: 'audioChunk',
|
|
sessionId: this._sessionId,
|
|
audio: {
|
|
format: 'pcm16',
|
|
sampleRate,
|
|
data: base64Data,
|
|
timestamp: new Date().toISOString(),
|
|
captureDiagnostics,
|
|
},
|
|
};
|
|
this._sendToGateway(message);
|
|
}
|
|
|
|
/**
|
|
* Send a text message to the meeting chat.
|
|
*/
|
|
async sendChatMessageToMeeting(text: string): Promise<void> {
|
|
if (this._isShuttingDown || this._state !== 'in_meeting' || !this._chatProcedure) {
|
|
this._logger.warn('Cannot send chat message - not in meeting');
|
|
return;
|
|
}
|
|
await this._chatProcedure.sendChatMessage(text);
|
|
}
|
|
|
|
/**
|
|
* Handle structured commands from the Gateway (issued by AI).
|
|
*/
|
|
private async _handleBotCommand(command: string, params: Record<string, any>): Promise<void> {
|
|
if (!this._teamsActions || this._state !== 'in_meeting') {
|
|
this._logger.warn(`Cannot execute command '${command}' — not in meeting`);
|
|
return;
|
|
}
|
|
|
|
this._logger.info(`Executing bot command: ${command} params=${JSON.stringify(params)}`);
|
|
|
|
try {
|
|
switch (command) {
|
|
case 'toggleTranscript':
|
|
await this._teamsActions.toggleTranscript(params.enable !== false);
|
|
break;
|
|
case 'toggleMic':
|
|
await this._teamsActions.toggleMic(params.enable !== false);
|
|
break;
|
|
case 'toggleCamera':
|
|
await this._teamsActions.toggleCamera(params.enable !== false);
|
|
break;
|
|
default:
|
|
this._logger.warn(`Unknown bot command: ${command}`);
|
|
}
|
|
} catch (err) {
|
|
this._logger.error(`Bot command '${command}' failed: ${err}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update the bot state and notify callbacks + Gateway.
|
|
*/
|
|
private _setState(state: BotState, message?: string): void {
|
|
this._state = state;
|
|
this._logger.info(`State changed: ${state}${message ? ` - ${message}` : ''}`);
|
|
this._callbacks.onStateChange(state, message);
|
|
|
|
// Send status to Gateway
|
|
const statusMap: Record<BotState, StatusMessage['status']> = {
|
|
'idle': 'connecting',
|
|
'launching': 'connecting',
|
|
'navigating': 'connecting',
|
|
'in_lobby': 'in_lobby',
|
|
'in_meeting': 'joined',
|
|
'leaving': 'left',
|
|
'error': 'error',
|
|
'disconnected': 'left',
|
|
};
|
|
this._sendStatus(statusMap[state], message);
|
|
}
|
|
|
|
/**
|
|
* Take a screenshot for debugging.
|
|
* Logs screenshot as base64 for easy viewing from Azure logs.
|
|
*/
|
|
private async _takeScreenshot(name: string, force: boolean = false): Promise<void> {
|
|
if ((!config.screenshotOnError && !force) || !this._page) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
// Save to file
|
|
const screenshotDir = config.screenshotDir;
|
|
if (!fs.existsSync(screenshotDir)) {
|
|
fs.mkdirSync(screenshotDir, { recursive: true });
|
|
}
|
|
|
|
const filename = `${this._sessionId}-${name}-${Date.now()}.png`;
|
|
const filepath = path.join(screenshotDir, filename);
|
|
const buffer = await this._page.screenshot({ fullPage: true });
|
|
fs.writeFileSync(filepath, buffer);
|
|
this._logger.info(`Screenshot saved: ${filepath}`);
|
|
|
|
// Also log as base64 for Azure logs (truncated for readability)
|
|
const base64 = buffer.toString('base64');
|
|
this._logger.info(`SCREENSHOT_BASE64_START:${name}`);
|
|
// Log in chunks to avoid log line limits
|
|
const chunkSize = 50000;
|
|
for (let i = 0; i < base64.length; i += chunkSize) {
|
|
this._logger.info(`SCREENSHOT_CHUNK:${base64.substring(i, i + chunkSize)}`);
|
|
}
|
|
this._logger.info(`SCREENSHOT_BASE64_END:${name}`);
|
|
} catch (error) {
|
|
this._logger.error('Error taking screenshot:', error);
|
|
}
|
|
}
|
|
}
|