feat: authenticated bot join, virtual background, language selectors

- New authProcedure.ts: Microsoft login flow (email/password, MFA detection, stay signed in)
- New backgroundProcedure.ts: download image URL, upload as Teams virtual background
- Orchestrator: authenticate before join when botAccountEmail provided
- JoinProcedure: skip name input for authenticated joins
- meetingUrlParser: anon=true only for anonymous joins
- SessionManager/HttpServer: pass new fields through the chain
- Updated Teams caption language selectors for current UI

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG 2026-02-15 11:56:04 +01:00
parent 496268e936
commit 79027f190b
8 changed files with 506 additions and 23 deletions

244
src/bot/authProcedure.ts Normal file
View file

@ -0,0 +1,244 @@
import { Page } from 'playwright';
import { Logger } from 'winston';
/**
* AuthProcedure - Handles Microsoft account authentication in the browser.
*
* Used for dedicated bot accounts (e.g. bot@valueon.ch) to join Teams meetings
* as an authenticated user instead of an anonymous guest.
*
* Benefits of authenticated join:
* - Full access to Teams features (language settings, background effects)
* - No lobby wait (if user is member of the meeting org)
* - Bot name is the account display name
* - Can set spoken language for captions
*/
const _LOGIN_URL = 'https://login.microsoftonline.com';
const _TEAMS_URL = 'https://teams.microsoft.com';
export class AuthProcedure {
private _page: Page;
private _logger: Logger;
constructor(page: Page, logger: Logger) {
this._page = page;
this._logger = logger;
}
/**
* Authenticate with Microsoft using email + password.
* Navigates to Microsoft login, enters credentials, and waits for successful sign-in.
*
* @returns true if authentication was successful, false otherwise
*/
async authenticateWithMicrosoft(email: string, password: string): Promise<boolean> {
try {
this._logger.info(`Authenticating as ${email}...`);
// Navigate to Microsoft login
await this._page.goto(_LOGIN_URL, {
waitUntil: 'domcontentloaded',
timeout: 30000,
});
// Wait for email input
const emailInput = await this._page.waitForSelector(
'input[type="email"], input[name="loginfmt"]',
{ timeout: 10000 }
);
if (!emailInput) {
this._logger.error('Could not find email input field');
return false;
}
// Enter email
await emailInput.fill(email);
this._logger.info('Email entered');
// Click Next
const nextButton = await this._page.$('input[type="submit"], button[type="submit"]');
if (nextButton) {
await nextButton.click();
} else {
await this._page.keyboard.press('Enter');
}
await this._page.waitForTimeout(2000);
// Check for "account not found" error
const errorElement = await this._page.$('#usernameError, [data-tid="error"]');
if (errorElement) {
const errorText = await errorElement.textContent();
this._logger.error(`Login error after email: ${errorText}`);
return false;
}
// Wait for password input
const passwordInput = await this._page.waitForSelector(
'input[type="password"], input[name="passwd"]',
{ timeout: 10000 }
);
if (!passwordInput) {
this._logger.error('Could not find password input field');
return false;
}
// Enter password
await passwordInput.fill(password);
this._logger.info('Password entered');
// Click Sign in
const signInButton = await this._page.$('input[type="submit"], button[type="submit"]');
if (signInButton) {
await signInButton.click();
} else {
await this._page.keyboard.press('Enter');
}
await this._page.waitForTimeout(3000);
// Check for MFA prompt
const mfaDetected = await this._detectMfa();
if (mfaDetected) {
this._logger.error('MFA prompt detected - cannot authenticate automatically. Please disable MFA for the bot account.');
return false;
}
// Check for password error
const pwdError = await this._page.$('#passwordError, [data-tid="error"]');
if (pwdError) {
const errorText = await pwdError.textContent();
this._logger.error(`Login error after password: ${errorText}`);
return false;
}
// Handle "Stay signed in?" prompt
await this._handleStaySignedIn();
// Verify authentication succeeded by checking for Teams or Microsoft landing
const isAuthenticated = await this._verifyAuthentication();
if (isAuthenticated) {
this._logger.info(`Successfully authenticated as ${email}`);
} else {
this._logger.error('Authentication verification failed - may not be signed in');
}
return isAuthenticated;
} catch (error) {
this._logger.error(`Authentication failed: ${error}`);
return false;
}
}
/**
* Detect MFA prompts (authenticator app, SMS, phone call).
*/
private async _detectMfa(): Promise<boolean> {
const mfaSelectors = [
'#idDiv_SAOTCAS_Description', // Authenticator app prompt
'#idDiv_SAOTCC_Description', // Code entry
'#idDiv_SAASDS_Description', // SMS verification
'[data-tid="phoneVerification"]', // Phone verification
'text=Approve sign in request', // Authenticator approval
'text=Enter code', // Code entry
'text=Verify your identity', // Generic MFA
'text=Approve sign-in request', // Authenticator
];
for (const selector of mfaSelectors) {
try {
const element = await this._page.$(selector);
if (element) {
return true;
}
} catch {
// Continue checking
}
}
return false;
}
/**
* Handle the "Stay signed in?" prompt after successful login.
*/
private async _handleStaySignedIn(): Promise<void> {
try {
// Look for "Stay signed in?" or "Angemeldet bleiben?" prompt
const staySignedInSelectors = [
'input#idSIButton9', // "Yes" button (Stay signed in)
'button#idSIButton9', // Alternative
'input[value="Yes"]',
'input[value="Ja"]',
'button:has-text("Yes")',
'button:has-text("Ja")',
];
for (const selector of staySignedInSelectors) {
try {
const button = await this._page.$(selector);
if (button) {
await button.click();
this._logger.info('Clicked "Stay signed in" - Yes');
await this._page.waitForTimeout(2000);
return;
}
} catch {
// Continue
}
}
// No prompt found - that's OK, some tenants don't show it
this._logger.debug('No "Stay signed in" prompt detected');
} catch (error) {
this._logger.debug(`Stay signed in handling: ${error}`);
}
}
/**
* Verify that authentication was successful.
* Checks if we're on a Microsoft/Teams page with an authenticated session.
*/
private async _verifyAuthentication(): Promise<boolean> {
try {
const url = this._page.url();
// If we're on Teams or Microsoft portal, we're authenticated
if (url.includes('teams.microsoft.com') ||
url.includes('office.com') ||
url.includes('microsoftonline.com/common/oauth2') ||
url.includes('myapps.microsoft.com')) {
return true;
}
// Wait a bit and check again (redirects may be in progress)
await this._page.waitForTimeout(3000);
const finalUrl = this._page.url();
if (finalUrl.includes('teams.microsoft.com') ||
finalUrl.includes('office.com') ||
finalUrl.includes('portal.office.com')) {
return true;
}
// Check for any error states
const loginPage = finalUrl.includes('login.microsoftonline.com');
if (loginPage) {
// Still on login page - authentication may have failed
const hasError = await this._page.$('[data-tid="error"], #passwordError, #usernameError');
if (hasError) {
return false;
}
// Might still be processing
await this._page.waitForTimeout(5000);
const afterWaitUrl = this._page.url();
return !afterWaitUrl.includes('login.microsoftonline.com');
}
// If we're somewhere else entirely, assume authenticated
return true;
} catch (error) {
this._logger.error(`Authentication verification error: ${error}`);
return false;
}
}
}

View file

@ -0,0 +1,182 @@
import { Page } from 'playwright';
import { Logger } from 'winston';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
/**
* BackgroundProcedure - Handles setting a virtual background in Teams pre-join screen.
*
* Must be called AFTER the bot is on the pre-join screen but BEFORE clicking "Join now".
* Only works for authenticated joins (anonymous guests may not have background options).
*/
export class BackgroundProcedure {
private _page: Page;
private _logger: Logger;
constructor(page: Page, logger: Logger) {
this._page = page;
this._logger = logger;
}
/**
* Set a virtual background from a URL on the Teams pre-join screen.
*
* @param imageUrl - URL of the background image to download and apply
* @returns true if background was set successfully
*/
async setBackgroundFromUrl(imageUrl: string): Promise<boolean> {
try {
this._logger.info(`Setting virtual background from: ${imageUrl}`);
// Download the image to a temp file
const tempDir = os.tmpdir();
const tempFile = path.join(tempDir, `poweron-bg-${Date.now()}.jpg`);
const response = await fetch(imageUrl);
if (!response.ok) {
this._logger.error(`Failed to download background image: ${response.status} ${response.statusText}`);
return false;
}
const buffer = Buffer.from(await response.arrayBuffer());
fs.writeFileSync(tempFile, buffer);
this._logger.info(`Background image downloaded: ${buffer.length} bytes -> ${tempFile}`);
// Open background effects panel
const panelOpened = await this._openBackgroundEffectsPanel();
if (!panelOpened) {
this._logger.warn('Could not open background effects panel - skipping background');
this._cleanup(tempFile);
return false;
}
// Upload the image
const uploaded = await this._uploadBackgroundImage(tempFile);
this._cleanup(tempFile);
if (uploaded) {
this._logger.info('Virtual background set successfully');
} else {
this._logger.warn('Could not upload background image');
}
return uploaded;
} catch (error) {
this._logger.error(`Background setup failed: ${error}`);
return false;
}
}
/**
* Open the background effects panel on the pre-join screen.
*/
private async _openBackgroundEffectsPanel(): Promise<boolean> {
const backgroundButtonSelectors = [
'button[data-tid="toggle-background-effect"]',
'button[aria-label*="Background" i]',
'button[aria-label*="Hintergrund" i]',
'button[aria-label*="background filters" i]',
'button[aria-label*="Hintergrundeffekte" i]',
'#video-background-effects-button',
];
for (const selector of backgroundButtonSelectors) {
try {
const button = await this._page.$(selector);
if (button) {
await button.click();
this._logger.info(`Clicked background effects button: ${selector}`);
await this._page.waitForTimeout(2000);
return true;
}
} catch {
// Continue trying
}
}
this._logger.warn('Background effects button not found');
return false;
}
/**
* Upload a background image via the file input in the background effects panel.
*/
private async _uploadBackgroundImage(filePath: string): Promise<boolean> {
try {
// Look for "Add new" or "+" button to upload custom image
const addButtonSelectors = [
'button[aria-label*="Add new" i]',
'button[aria-label*="Neu hinzufügen" i]',
'button[aria-label*="add image" i]',
'button[aria-label*="Bild hinzufügen" i]',
'button[data-tid="add-background-image"]',
'.add-image-button',
];
let addButtonClicked = false;
for (const selector of addButtonSelectors) {
try {
const button = await this._page.$(selector);
if (button) {
await button.click();
this._logger.info(`Clicked add image button: ${selector}`);
addButtonClicked = true;
await this._page.waitForTimeout(1000);
break;
}
} catch {
// Continue
}
}
// Try to find file input (may appear after clicking add, or may already exist)
const fileInput = await this._page.$('input[type="file"]');
if (fileInput) {
await fileInput.setInputFiles(filePath);
this._logger.info('Background image uploaded via file input');
await this._page.waitForTimeout(2000);
// The uploaded image should be auto-selected, but click it to be sure
// Look for the last image in the background gallery (newly uploaded)
try {
const images = await this._page.$$('[data-tid="background-image"], .background-image-item');
if (images.length > 0) {
const lastImage = images[images.length - 1];
await lastImage.click();
this._logger.info('Selected uploaded background image');
await this._page.waitForTimeout(1000);
}
} catch {
this._logger.debug('Could not click uploaded image - may be auto-selected');
}
return true;
}
if (!addButtonClicked) {
this._logger.warn('No add-image button or file input found');
}
return false;
} catch (error) {
this._logger.error(`Background image upload failed: ${error}`);
return false;
}
}
/**
* Clean up temp file.
*/
private _cleanup(filePath: string): void {
try {
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
} catch {
// Ignore cleanup errors
}
}
}

View file

@ -14,11 +14,13 @@ export class JoinProcedure {
private _page: Page;
private _logger: Logger;
private _botName: string;
private _isAuthenticated: boolean;
constructor(page: Page, logger: Logger, botName: string) {
constructor(page: Page, logger: Logger, botName: string, isAuthenticated: boolean = false) {
this._page = page;
this._logger = logger;
this._botName = botName;
this._isAuthenticated = isAuthenticated;
}
/**
@ -27,13 +29,15 @@ export class JoinProcedure {
* Teams meeting URLs redirect through several hops. We resolve the redirect
* and add params (suppressPrompt, msLaunch=false, etc.) to skip the
* "Open in Teams app?" native dialog. Then we click "Continue on this browser".
*
* For authenticated joins, anon=true is omitted and the name input is skipped.
*/
async startMeetingLauncherFlow(meetingUrl: string): Promise<void> {
// Resolve the meeting URL redirect and add suppressPrompt params
let launchUrl: string;
try {
launchUrl = await resolveLaunchUrl(meetingUrl);
this._logger.info(`Resolved launch URL: ${launchUrl}`);
launchUrl = await resolveLaunchUrl(meetingUrl, this._isAuthenticated);
this._logger.info(`Resolved launch URL: ${launchUrl} (authenticated: ${this._isAuthenticated})`);
} catch (error) {
this._logger.warn(`Could not resolve launch URL, using fallback: ${error}`);
launchUrl = getMeetingLaunchUrl(meetingUrl);
@ -104,10 +108,17 @@ export class JoinProcedure {
* Fill in the bot name and click "Join now" to enter the lobby.
*/
async joinMeetingLobbyFlow(): Promise<void> {
this._logger.info('Starting lobby join flow...');
this._logger.info(`Starting lobby join flow... (authenticated: ${this._isAuthenticated})`);
// Enter the bot name (this also implicitly waits for the pre-join page to load)
await this._enterBotName();
if (this._isAuthenticated) {
// Authenticated join: name comes from Microsoft account, no name input needed
// Wait for the pre-join page to load (look for Join now button)
this._logger.info('Authenticated join - skipping name input, waiting for Join button...');
await this._page.waitForTimeout(3000);
} else {
// Anonymous join: enter bot name in the name input field
await this._enterBotName();
}
// Click "Join now"
await this._clickJoinNow();

View file

@ -67,7 +67,7 @@ export function isValidMeetingUrl(url: string): boolean {
* Teams meeting URLs redirect through several hops. The final URL needs specific
* search params to skip the "Open in Teams app?" dialog in the browser.
*/
export async function resolveLaunchUrl(meetingUrl: string): Promise<string> {
export async function resolveLaunchUrl(meetingUrl: string, isAuthenticated: boolean = false): Promise<string> {
const trimmed = meetingUrl.trim();
try {
@ -81,29 +81,36 @@ export async function resolveLaunchUrl(meetingUrl: string): Promise<string> {
resolvedUrl.searchParams.set('enableMobilePage', 'true');
resolvedUrl.searchParams.set('suppressPrompt', 'true');
// Only add anon=true for anonymous joins
if (!isAuthenticated) {
resolvedUrl.searchParams.set('anon', 'true');
}
return resolvedUrl.toString();
} catch {
// Fallback: add params to the original URL
return _addLaunchParams(trimmed);
return _addLaunchParams(trimmed, isAuthenticated);
}
}
/**
* Fallback: adds launch params directly to the meeting URL without resolving redirects.
*/
function _addLaunchParams(url: string): string {
function _addLaunchParams(url: string, isAuthenticated: boolean = false): string {
try {
const urlObj = new URL(url);
urlObj.searchParams.set('msLaunch', 'false');
urlObj.searchParams.set('suppressPrompt', 'true');
urlObj.searchParams.set('directDl', 'true');
urlObj.searchParams.set('enableMobilePage', 'true');
urlObj.searchParams.set('anon', 'true');
if (!isAuthenticated) {
urlObj.searchParams.set('anon', 'true');
}
return urlObj.toString();
} catch {
// If URL parsing fails, append as query string
const separator = url.includes('?') ? '&' : '?';
return `${url}${separator}msLaunch=false&suppressPrompt=true&directDl=true&enableMobilePage=true&anon=true`;
const anonParam = isAuthenticated ? '' : '&anon=true';
return `${url}${separator}msLaunch=false&suppressPrompt=true&directDl=true&enableMobilePage=true${anonParam}`;
}
}
@ -111,6 +118,6 @@ function _addLaunchParams(url: string): string {
* Converts a meeting URL to the web app launch URL.
* @deprecated Use resolveLaunchUrl() instead for proper redirect resolution.
*/
export function getMeetingLaunchUrl(url: string): string {
return _addLaunchParams(url);
export function getMeetingLaunchUrl(url: string, isAuthenticated: boolean = false): string {
return _addLaunchParams(url, isAuthenticated);
}

View file

@ -23,6 +23,9 @@ export interface OrchestratorOptions {
gatewayWsUrl: string;
instanceId: string;
language?: string;
botAccountEmail?: string;
botAccountPassword?: string;
backgroundImageUrl?: string;
}
/**
@ -81,13 +84,15 @@ export class BotOrchestrator {
}
/**
* Start the bot - connect to Gateway, launch browser, join meeting, enable captions.
* Start the bot - connect to Gateway, launch browser, authenticate (if configured), join meeting, enable captions.
*/
async start(): Promise<void> {
if (!isValidMeetingUrl(this._meetingUrl)) {
throw new Error(`Invalid meeting URL: ${this._meetingUrl}`);
}
const isAuthenticated = !!(this._options.botAccountEmail && this._options.botAccountPassword);
try {
this._setState('launching');
@ -97,12 +102,36 @@ export class BotOrchestrator {
// Launch browser
await this._launchBrowser();
// Authenticate with Microsoft if bot account is configured
if (isAuthenticated) {
const { AuthProcedure } = await import('./authProcedure');
const authProcedure = new AuthProcedure(this._page!, this._logger);
const authSuccess = await authProcedure.authenticateWithMicrosoft(
this._options.botAccountEmail!,
this._options.botAccountPassword!
);
if (!authSuccess) {
this._logger.warn('Microsoft authentication failed - falling back to anonymous join');
}
}
this._setState('navigating');
// Navigate to meeting and handle launcher
await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl);
// Join the meeting (enter lobby)
// Set virtual background if configured (must be done on pre-join screen, before "Join now")
if (this._options.backgroundImageUrl && this._page) {
try {
const { BackgroundProcedure } = await import('./backgroundProcedure');
const bgProcedure = new BackgroundProcedure(this._page, this._logger);
await bgProcedure.setBackgroundFromUrl(this._options.backgroundImageUrl);
} catch (error) {
this._logger.warn(`Background image setup failed (non-fatal): ${error}`);
}
}
// Join the meeting (enter lobby for anonymous, direct join for authenticated)
await this._joinProcedure!.joinMeetingLobbyFlow();
// Check if we're in lobby
@ -380,7 +409,8 @@ export class BotOrchestrator {
this._page = await this._context.newPage();
// Initialize procedures
this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName);
const isAuthenticated = !!(this._options.botAccountEmail && this._options.botAccountPassword);
this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName, isAuthenticated);
this._captionsProcedure = new CaptionsProcedure(
this._page,
this._logger,

View file

@ -19,8 +19,8 @@ async function main(): Promise<void> {
// Start HTTP server
httpServer = new HttpServer({
onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language) => {
await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language);
onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl) => {
await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl);
},
onLeaveRequest: async (sessionId) => {
await sessionManager.endSession(sessionId);

View file

@ -4,7 +4,7 @@ import { logger } from '../utils/logger';
import { config } from '../config';
export interface HttpServerCallbacks {
onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string) => Promise<void>;
onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, backgroundImageUrl?: string) => Promise<void>;
onLeaveRequest: (sessionId: string) => Promise<void>;
onStatusRequest: (sessionId: string) => { state: string; error?: string } | null;
}
@ -77,14 +77,14 @@ export class HttpServer {
// Deploy a new bot
this._app.post('/api/bot', async (req: Request, res: Response) => {
try {
const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language } = req.body;
const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl } = req.body;
if (!sessionId || !meetingUrl) {
res.status(400).json({ error: 'Missing required fields: sessionId, meetingUrl' });
return;
}
await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language);
await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, backgroundImageUrl);
res.json({
success: true,

View file

@ -38,7 +38,10 @@ export class SessionManager {
botName?: string,
instanceId?: string,
gatewayWsUrl?: string,
language?: string
language?: string,
botAccountEmail?: string,
botAccountPassword?: string,
backgroundImageUrl?: string
): Promise<void> {
if (this._sessions.has(sessionId)) {
logger.warn(`Session ${sessionId} already exists`);
@ -47,6 +50,9 @@ export class SessionManager {
logger.info(`Creating session ${sessionId} for meeting: ${meetingUrl}`);
logger.info(`Gateway WebSocket URL: ${gatewayWsUrl || 'not provided, using config'}`);
if (botAccountEmail) {
logger.info(`Authenticated join as: ${botAccountEmail}`);
}
const callbacks: OrchestratorCallbacks = {
onStateChange: (state, message) => {
@ -67,6 +73,9 @@ export class SessionManager {
gatewayWsUrl: gatewayWsUrl || config.gatewayWsUrl,
instanceId: instanceId || 'default',
language: language,
botAccountEmail: botAccountEmail,
botAccountPassword: botAccountPassword,
backgroundImageUrl: backgroundImageUrl,
};
const orchestrator = new BotOrchestrator(