diff --git a/src/bot/authProcedure.ts b/src/bot/authProcedure.ts new file mode 100644 index 0000000..ed137ac --- /dev/null +++ b/src/bot/authProcedure.ts @@ -0,0 +1,244 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; + +/** + * AuthProcedure - Handles Microsoft account authentication in the browser. + * + * Used for dedicated bot accounts (e.g. bot@valueon.ch) to join Teams meetings + * as an authenticated user instead of an anonymous guest. + * + * Benefits of authenticated join: + * - Full access to Teams features (language settings, background effects) + * - No lobby wait (if user is member of the meeting org) + * - Bot name is the account display name + * - Can set spoken language for captions + */ + +const _LOGIN_URL = 'https://login.microsoftonline.com'; +const _TEAMS_URL = 'https://teams.microsoft.com'; + +export class AuthProcedure { + private _page: Page; + private _logger: Logger; + + constructor(page: Page, logger: Logger) { + this._page = page; + this._logger = logger; + } + + /** + * Authenticate with Microsoft using email + password. + * Navigates to Microsoft login, enters credentials, and waits for successful sign-in. + * + * @returns true if authentication was successful, false otherwise + */ + async authenticateWithMicrosoft(email: string, password: string): Promise { + try { + this._logger.info(`Authenticating as ${email}...`); + + // Navigate to Microsoft login + await this._page.goto(_LOGIN_URL, { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + + // Wait for email input + const emailInput = await this._page.waitForSelector( + 'input[type="email"], input[name="loginfmt"]', + { timeout: 10000 } + ); + if (!emailInput) { + this._logger.error('Could not find email input field'); + return false; + } + + // Enter email + await emailInput.fill(email); + this._logger.info('Email entered'); + + // Click Next + const nextButton = await this._page.$('input[type="submit"], button[type="submit"]'); + if (nextButton) { + await nextButton.click(); + } else { + await this._page.keyboard.press('Enter'); + } + await this._page.waitForTimeout(2000); + + // Check for "account not found" error + const errorElement = await this._page.$('#usernameError, [data-tid="error"]'); + if (errorElement) { + const errorText = await errorElement.textContent(); + this._logger.error(`Login error after email: ${errorText}`); + return false; + } + + // Wait for password input + const passwordInput = await this._page.waitForSelector( + 'input[type="password"], input[name="passwd"]', + { timeout: 10000 } + ); + if (!passwordInput) { + this._logger.error('Could not find password input field'); + return false; + } + + // Enter password + await passwordInput.fill(password); + this._logger.info('Password entered'); + + // Click Sign in + const signInButton = await this._page.$('input[type="submit"], button[type="submit"]'); + if (signInButton) { + await signInButton.click(); + } else { + await this._page.keyboard.press('Enter'); + } + await this._page.waitForTimeout(3000); + + // Check for MFA prompt + const mfaDetected = await this._detectMfa(); + if (mfaDetected) { + this._logger.error('MFA prompt detected - cannot authenticate automatically. Please disable MFA for the bot account.'); + return false; + } + + // Check for password error + const pwdError = await this._page.$('#passwordError, [data-tid="error"]'); + if (pwdError) { + const errorText = await pwdError.textContent(); + this._logger.error(`Login error after password: ${errorText}`); + return false; + } + + // Handle "Stay signed in?" prompt + await this._handleStaySignedIn(); + + // Verify authentication succeeded by checking for Teams or Microsoft landing + const isAuthenticated = await this._verifyAuthentication(); + if (isAuthenticated) { + this._logger.info(`Successfully authenticated as ${email}`); + } else { + this._logger.error('Authentication verification failed - may not be signed in'); + } + + return isAuthenticated; + + } catch (error) { + this._logger.error(`Authentication failed: ${error}`); + return false; + } + } + + /** + * Detect MFA prompts (authenticator app, SMS, phone call). + */ + private async _detectMfa(): Promise { + const mfaSelectors = [ + '#idDiv_SAOTCAS_Description', // Authenticator app prompt + '#idDiv_SAOTCC_Description', // Code entry + '#idDiv_SAASDS_Description', // SMS verification + '[data-tid="phoneVerification"]', // Phone verification + 'text=Approve sign in request', // Authenticator approval + 'text=Enter code', // Code entry + 'text=Verify your identity', // Generic MFA + 'text=Approve sign-in request', // Authenticator + ]; + + for (const selector of mfaSelectors) { + try { + const element = await this._page.$(selector); + if (element) { + return true; + } + } catch { + // Continue checking + } + } + return false; + } + + /** + * Handle the "Stay signed in?" prompt after successful login. + */ + private async _handleStaySignedIn(): Promise { + try { + // Look for "Stay signed in?" or "Angemeldet bleiben?" prompt + const staySignedInSelectors = [ + 'input#idSIButton9', // "Yes" button (Stay signed in) + 'button#idSIButton9', // Alternative + 'input[value="Yes"]', + 'input[value="Ja"]', + 'button:has-text("Yes")', + 'button:has-text("Ja")', + ]; + + for (const selector of staySignedInSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info('Clicked "Stay signed in" - Yes'); + await this._page.waitForTimeout(2000); + return; + } + } catch { + // Continue + } + } + + // No prompt found - that's OK, some tenants don't show it + this._logger.debug('No "Stay signed in" prompt detected'); + } catch (error) { + this._logger.debug(`Stay signed in handling: ${error}`); + } + } + + /** + * Verify that authentication was successful. + * Checks if we're on a Microsoft/Teams page with an authenticated session. + */ + private async _verifyAuthentication(): Promise { + try { + const url = this._page.url(); + + // If we're on Teams or Microsoft portal, we're authenticated + if (url.includes('teams.microsoft.com') || + url.includes('office.com') || + url.includes('microsoftonline.com/common/oauth2') || + url.includes('myapps.microsoft.com')) { + return true; + } + + // Wait a bit and check again (redirects may be in progress) + await this._page.waitForTimeout(3000); + const finalUrl = this._page.url(); + + if (finalUrl.includes('teams.microsoft.com') || + finalUrl.includes('office.com') || + finalUrl.includes('portal.office.com')) { + return true; + } + + // Check for any error states + const loginPage = finalUrl.includes('login.microsoftonline.com'); + if (loginPage) { + // Still on login page - authentication may have failed + const hasError = await this._page.$('[data-tid="error"], #passwordError, #usernameError'); + if (hasError) { + return false; + } + // Might still be processing + await this._page.waitForTimeout(5000); + const afterWaitUrl = this._page.url(); + return !afterWaitUrl.includes('login.microsoftonline.com'); + } + + // If we're somewhere else entirely, assume authenticated + return true; + } catch (error) { + this._logger.error(`Authentication verification error: ${error}`); + return false; + } + } +} diff --git a/src/bot/backgroundProcedure.ts b/src/bot/backgroundProcedure.ts new file mode 100644 index 0000000..c100dc9 --- /dev/null +++ b/src/bot/backgroundProcedure.ts @@ -0,0 +1,182 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +/** + * BackgroundProcedure - Handles setting a virtual background in Teams pre-join screen. + * + * Must be called AFTER the bot is on the pre-join screen but BEFORE clicking "Join now". + * Only works for authenticated joins (anonymous guests may not have background options). + */ +export class BackgroundProcedure { + private _page: Page; + private _logger: Logger; + + constructor(page: Page, logger: Logger) { + this._page = page; + this._logger = logger; + } + + /** + * Set a virtual background from a URL on the Teams pre-join screen. + * + * @param imageUrl - URL of the background image to download and apply + * @returns true if background was set successfully + */ + async setBackgroundFromUrl(imageUrl: string): Promise { + try { + this._logger.info(`Setting virtual background from: ${imageUrl}`); + + // Download the image to a temp file + const tempDir = os.tmpdir(); + const tempFile = path.join(tempDir, `poweron-bg-${Date.now()}.jpg`); + + const response = await fetch(imageUrl); + if (!response.ok) { + this._logger.error(`Failed to download background image: ${response.status} ${response.statusText}`); + return false; + } + + const buffer = Buffer.from(await response.arrayBuffer()); + fs.writeFileSync(tempFile, buffer); + this._logger.info(`Background image downloaded: ${buffer.length} bytes -> ${tempFile}`); + + // Open background effects panel + const panelOpened = await this._openBackgroundEffectsPanel(); + if (!panelOpened) { + this._logger.warn('Could not open background effects panel - skipping background'); + this._cleanup(tempFile); + return false; + } + + // Upload the image + const uploaded = await this._uploadBackgroundImage(tempFile); + this._cleanup(tempFile); + + if (uploaded) { + this._logger.info('Virtual background set successfully'); + } else { + this._logger.warn('Could not upload background image'); + } + + return uploaded; + + } catch (error) { + this._logger.error(`Background setup failed: ${error}`); + return false; + } + } + + /** + * Open the background effects panel on the pre-join screen. + */ + private async _openBackgroundEffectsPanel(): Promise { + const backgroundButtonSelectors = [ + 'button[data-tid="toggle-background-effect"]', + 'button[aria-label*="Background" i]', + 'button[aria-label*="Hintergrund" i]', + 'button[aria-label*="background filters" i]', + 'button[aria-label*="Hintergrundeffekte" i]', + '#video-background-effects-button', + ]; + + for (const selector of backgroundButtonSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info(`Clicked background effects button: ${selector}`); + await this._page.waitForTimeout(2000); + return true; + } + } catch { + // Continue trying + } + } + + this._logger.warn('Background effects button not found'); + return false; + } + + /** + * Upload a background image via the file input in the background effects panel. + */ + private async _uploadBackgroundImage(filePath: string): Promise { + try { + // Look for "Add new" or "+" button to upload custom image + const addButtonSelectors = [ + 'button[aria-label*="Add new" i]', + 'button[aria-label*="Neu hinzufügen" i]', + 'button[aria-label*="add image" i]', + 'button[aria-label*="Bild hinzufügen" i]', + 'button[data-tid="add-background-image"]', + '.add-image-button', + ]; + + let addButtonClicked = false; + for (const selector of addButtonSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info(`Clicked add image button: ${selector}`); + addButtonClicked = true; + await this._page.waitForTimeout(1000); + break; + } + } catch { + // Continue + } + } + + // Try to find file input (may appear after clicking add, or may already exist) + const fileInput = await this._page.$('input[type="file"]'); + if (fileInput) { + await fileInput.setInputFiles(filePath); + this._logger.info('Background image uploaded via file input'); + await this._page.waitForTimeout(2000); + + // The uploaded image should be auto-selected, but click it to be sure + // Look for the last image in the background gallery (newly uploaded) + try { + const images = await this._page.$$('[data-tid="background-image"], .background-image-item'); + if (images.length > 0) { + const lastImage = images[images.length - 1]; + await lastImage.click(); + this._logger.info('Selected uploaded background image'); + await this._page.waitForTimeout(1000); + } + } catch { + this._logger.debug('Could not click uploaded image - may be auto-selected'); + } + + return true; + } + + if (!addButtonClicked) { + this._logger.warn('No add-image button or file input found'); + } + + return false; + + } catch (error) { + this._logger.error(`Background image upload failed: ${error}`); + return false; + } + } + + /** + * Clean up temp file. + */ + private _cleanup(filePath: string): void { + try { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } catch { + // Ignore cleanup errors + } + } +} diff --git a/src/bot/joinProcedure.ts b/src/bot/joinProcedure.ts index c4cb931..debd78d 100644 --- a/src/bot/joinProcedure.ts +++ b/src/bot/joinProcedure.ts @@ -14,11 +14,13 @@ export class JoinProcedure { private _page: Page; private _logger: Logger; private _botName: string; + private _isAuthenticated: boolean; - constructor(page: Page, logger: Logger, botName: string) { + constructor(page: Page, logger: Logger, botName: string, isAuthenticated: boolean = false) { this._page = page; this._logger = logger; this._botName = botName; + this._isAuthenticated = isAuthenticated; } /** @@ -27,13 +29,15 @@ export class JoinProcedure { * Teams meeting URLs redirect through several hops. We resolve the redirect * and add params (suppressPrompt, msLaunch=false, etc.) to skip the * "Open in Teams app?" native dialog. Then we click "Continue on this browser". + * + * For authenticated joins, anon=true is omitted and the name input is skipped. */ async startMeetingLauncherFlow(meetingUrl: string): Promise { // Resolve the meeting URL redirect and add suppressPrompt params let launchUrl: string; try { - launchUrl = await resolveLaunchUrl(meetingUrl); - this._logger.info(`Resolved launch URL: ${launchUrl}`); + launchUrl = await resolveLaunchUrl(meetingUrl, this._isAuthenticated); + this._logger.info(`Resolved launch URL: ${launchUrl} (authenticated: ${this._isAuthenticated})`); } catch (error) { this._logger.warn(`Could not resolve launch URL, using fallback: ${error}`); launchUrl = getMeetingLaunchUrl(meetingUrl); @@ -104,10 +108,17 @@ export class JoinProcedure { * Fill in the bot name and click "Join now" to enter the lobby. */ async joinMeetingLobbyFlow(): Promise { - this._logger.info('Starting lobby join flow...'); + this._logger.info(`Starting lobby join flow... (authenticated: ${this._isAuthenticated})`); - // Enter the bot name (this also implicitly waits for the pre-join page to load) - await this._enterBotName(); + if (this._isAuthenticated) { + // Authenticated join: name comes from Microsoft account, no name input needed + // Wait for the pre-join page to load (look for Join now button) + this._logger.info('Authenticated join - skipping name input, waiting for Join button...'); + await this._page.waitForTimeout(3000); + } else { + // Anonymous join: enter bot name in the name input field + await this._enterBotName(); + } // Click "Join now" await this._clickJoinNow(); diff --git a/src/bot/meetingUrlParser.ts b/src/bot/meetingUrlParser.ts index fc8e4ce..cd8e571 100644 --- a/src/bot/meetingUrlParser.ts +++ b/src/bot/meetingUrlParser.ts @@ -67,7 +67,7 @@ export function isValidMeetingUrl(url: string): boolean { * Teams meeting URLs redirect through several hops. The final URL needs specific * search params to skip the "Open in Teams app?" dialog in the browser. */ -export async function resolveLaunchUrl(meetingUrl: string): Promise { +export async function resolveLaunchUrl(meetingUrl: string, isAuthenticated: boolean = false): Promise { const trimmed = meetingUrl.trim(); try { @@ -81,29 +81,36 @@ export async function resolveLaunchUrl(meetingUrl: string): Promise { resolvedUrl.searchParams.set('enableMobilePage', 'true'); resolvedUrl.searchParams.set('suppressPrompt', 'true'); + // Only add anon=true for anonymous joins + if (!isAuthenticated) { + resolvedUrl.searchParams.set('anon', 'true'); + } + return resolvedUrl.toString(); } catch { // Fallback: add params to the original URL - return _addLaunchParams(trimmed); + return _addLaunchParams(trimmed, isAuthenticated); } } /** * Fallback: adds launch params directly to the meeting URL without resolving redirects. */ -function _addLaunchParams(url: string): string { +function _addLaunchParams(url: string, isAuthenticated: boolean = false): string { try { const urlObj = new URL(url); urlObj.searchParams.set('msLaunch', 'false'); urlObj.searchParams.set('suppressPrompt', 'true'); urlObj.searchParams.set('directDl', 'true'); urlObj.searchParams.set('enableMobilePage', 'true'); - urlObj.searchParams.set('anon', 'true'); + if (!isAuthenticated) { + urlObj.searchParams.set('anon', 'true'); + } return urlObj.toString(); } catch { - // If URL parsing fails, append as query string const separator = url.includes('?') ? '&' : '?'; - return `${url}${separator}msLaunch=false&suppressPrompt=true&directDl=true&enableMobilePage=true&anon=true`; + const anonParam = isAuthenticated ? '' : '&anon=true'; + return `${url}${separator}msLaunch=false&suppressPrompt=true&directDl=true&enableMobilePage=true${anonParam}`; } } @@ -111,6 +118,6 @@ function _addLaunchParams(url: string): string { * Converts a meeting URL to the web app launch URL. * @deprecated Use resolveLaunchUrl() instead for proper redirect resolution. */ -export function getMeetingLaunchUrl(url: string): string { - return _addLaunchParams(url); +export function getMeetingLaunchUrl(url: string, isAuthenticated: boolean = false): string { + return _addLaunchParams(url, isAuthenticated); } diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index 90c3ce9..dd8deec 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -23,6 +23,9 @@ export interface OrchestratorOptions { gatewayWsUrl: string; instanceId: string; language?: string; + botAccountEmail?: string; + botAccountPassword?: string; + backgroundImageUrl?: string; } /** @@ -81,13 +84,15 @@ export class BotOrchestrator { } /** - * Start the bot - connect to Gateway, launch browser, join meeting, enable captions. + * Start the bot - connect to Gateway, launch browser, authenticate (if configured), join meeting, enable captions. */ async start(): Promise { if (!isValidMeetingUrl(this._meetingUrl)) { throw new Error(`Invalid meeting URL: ${this._meetingUrl}`); } + const isAuthenticated = !!(this._options.botAccountEmail && this._options.botAccountPassword); + try { this._setState('launching'); @@ -97,12 +102,36 @@ export class BotOrchestrator { // Launch browser await this._launchBrowser(); + // Authenticate with Microsoft if bot account is configured + if (isAuthenticated) { + const { AuthProcedure } = await import('./authProcedure'); + const authProcedure = new AuthProcedure(this._page!, this._logger); + const authSuccess = await authProcedure.authenticateWithMicrosoft( + this._options.botAccountEmail!, + this._options.botAccountPassword! + ); + if (!authSuccess) { + this._logger.warn('Microsoft authentication failed - falling back to anonymous join'); + } + } + this._setState('navigating'); // Navigate to meeting and handle launcher await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl); - // Join the meeting (enter lobby) + // Set virtual background if configured (must be done on pre-join screen, before "Join now") + if (this._options.backgroundImageUrl && this._page) { + try { + const { BackgroundProcedure } = await import('./backgroundProcedure'); + const bgProcedure = new BackgroundProcedure(this._page, this._logger); + await bgProcedure.setBackgroundFromUrl(this._options.backgroundImageUrl); + } catch (error) { + this._logger.warn(`Background image setup failed (non-fatal): ${error}`); + } + } + + // Join the meeting (enter lobby for anonymous, direct join for authenticated) await this._joinProcedure!.joinMeetingLobbyFlow(); // Check if we're in lobby @@ -380,7 +409,8 @@ export class BotOrchestrator { this._page = await this._context.newPage(); // Initialize procedures - this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName); + const isAuthenticated = !!(this._options.botAccountEmail && this._options.botAccountPassword); + this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName, isAuthenticated); this._captionsProcedure = new CaptionsProcedure( this._page, this._logger, diff --git a/src/index.ts b/src/index.ts index 064a951..9551dbe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,8 +19,8 @@ async function main(): Promise { // Start HTTP server httpServer = new HttpServer({ - onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language) => { - await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language); + onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl) => { + await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl); }, onLeaveRequest: async (sessionId) => { await sessionManager.endSession(sessionId); diff --git a/src/server/httpServer.ts b/src/server/httpServer.ts index 0ef1645..1fb2d33 100644 --- a/src/server/httpServer.ts +++ b/src/server/httpServer.ts @@ -4,7 +4,7 @@ import { logger } from '../utils/logger'; import { config } from '../config'; export interface HttpServerCallbacks { - onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string) => Promise; + onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, backgroundImageUrl?: string) => Promise; onLeaveRequest: (sessionId: string) => Promise; onStatusRequest: (sessionId: string) => { state: string; error?: string } | null; } @@ -77,14 +77,14 @@ export class HttpServer { // Deploy a new bot this._app.post('/api/bot', async (req: Request, res: Response) => { try { - const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language } = req.body; + const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl } = req.body; if (!sessionId || !meetingUrl) { res.status(400).json({ error: 'Missing required fields: sessionId, meetingUrl' }); return; } - await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language); + await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl); res.json({ success: true, diff --git a/src/sessionManager.ts b/src/sessionManager.ts index 2b2880d..8dc6828 100644 --- a/src/sessionManager.ts +++ b/src/sessionManager.ts @@ -38,7 +38,10 @@ export class SessionManager { botName?: string, instanceId?: string, gatewayWsUrl?: string, - language?: string + language?: string, + botAccountEmail?: string, + botAccountPassword?: string, + backgroundImageUrl?: string ): Promise { if (this._sessions.has(sessionId)) { logger.warn(`Session ${sessionId} already exists`); @@ -47,6 +50,9 @@ export class SessionManager { logger.info(`Creating session ${sessionId} for meeting: ${meetingUrl}`); logger.info(`Gateway WebSocket URL: ${gatewayWsUrl || 'not provided, using config'}`); + if (botAccountEmail) { + logger.info(`Authenticated join as: ${botAccountEmail}`); + } const callbacks: OrchestratorCallbacks = { onStateChange: (state, message) => { @@ -67,6 +73,9 @@ export class SessionManager { gatewayWsUrl: gatewayWsUrl || config.gatewayWsUrl, instanceId: instanceId || 'default', language: language, + botAccountEmail: botAccountEmail, + botAccountPassword: botAccountPassword, + backgroundImageUrl: backgroundImageUrl, }; const orchestrator = new BotOrchestrator(