From 09dc63d75c8c9f61b2c8f2b8e083d76b29e53b4c Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Tue, 12 May 2026 19:16:07 +0200 Subject: [PATCH] anonymous bot working --- src/bot/audioProcedure.ts | 12 +++++ src/bot/mediaGetUserMediaPatch.ts | 82 +++++++++++++++++++++++++------ src/bot/orchestrator.ts | 24 ++++++--- src/index.ts | 4 +- src/server/httpServer.ts | 10 ++-- src/sessionManager.ts | 4 ++ 6 files changed, 108 insertions(+), 28 deletions(-) diff --git a/src/bot/audioProcedure.ts b/src/bot/audioProcedure.ts index 2d995b6..81d46ab 100644 --- a/src/bot/audioProcedure.ts +++ b/src/bot/audioProcedure.ts @@ -24,6 +24,10 @@ export type AudioProcedureOptions = { avatarBgColor?: string; /** Hex/CSS color of the centered display label (default: dark blue). */ avatarTextColor?: string; + /** Base64-encoded image/video data for custom bot avatar. */ + avatarMediaData?: string; + /** MIME type of the avatar media (e.g. image/png, video/mp4). */ + avatarMediaType?: string; }; export class AudioProcedure { @@ -33,6 +37,8 @@ export class AudioProcedure { private _displayLabel: string; private _avatarBgColor: string; private _avatarTextColor: string; + private _avatarMediaData?: string; + private _avatarMediaType?: string; private _audioContext: boolean = false; private _initScriptInjected: boolean = false; private _audioQueue: Array<{ audioData: string; format: 'mp3' | 'wav' | 'pcm' }> = []; @@ -46,6 +52,8 @@ export class AudioProcedure { this._displayLabel = (options?.displayLabel || 'Bot').trim() || 'Bot'; this._avatarBgColor = (options?.avatarBgColor || '').trim() || '#a8d4f0'; this._avatarTextColor = (options?.avatarTextColor || '').trim() || '#1a3552'; + this._avatarMediaData = options?.avatarMediaData; + this._avatarMediaType = options?.avatarMediaType; } /** @@ -69,6 +77,8 @@ export class AudioProcedure { displayLabel: this._displayLabel, avatarBgColor: this._avatarBgColor, avatarTextColor: this._avatarTextColor, + avatarMediaData: this._avatarMediaData, + avatarMediaType: this._avatarMediaType, }); this._initScriptInjected = true; @@ -85,6 +95,8 @@ export class AudioProcedure { displayLabel: this._displayLabel, avatarBgColor: this._avatarBgColor, avatarTextColor: this._avatarTextColor, + avatarMediaData: this._avatarMediaData, + avatarMediaType: this._avatarMediaType, }; for (const frame of this._page.frames()) { try { diff --git a/src/bot/mediaGetUserMediaPatch.ts b/src/bot/mediaGetUserMediaPatch.ts index 6988342..5737b07 100644 --- a/src/bot/mediaGetUserMediaPatch.ts +++ b/src/bot/mediaGetUserMediaPatch.ts @@ -12,6 +12,10 @@ export type MediaGetUserMediaPatchOptions = { avatarBgColor?: string; /** Hex/CSS color of the centered display label (default: dark blue). */ avatarTextColor?: string; + /** Base64-encoded image/video data for custom bot avatar. */ + avatarMediaData?: string; + /** MIME type of the avatar media (e.g. image/png, video/mp4). */ + avatarMediaType?: string; }; export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => { @@ -19,6 +23,8 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => const { useCanvasVideo, displayLabel } = opts; const avatarBgColor = opts.avatarBgColor || '#a8d4f0'; const avatarTextColor = opts.avatarTextColor || '#1a3552'; + const avatarMediaData = opts.avatarMediaData || ''; + const avatarMediaType = opts.avatarMediaType || ''; const w: any = window as any; if (!w.__gumChromium) { @@ -169,11 +175,9 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => return; } - // Low fps: the avatar is intentionally STATIC. The interval still ticks so - // captureStream() in headless Chromium gets fresh frames (some Chromium - // builds pause the track if no new frames arrive), but each tick draws an - // identical image — no animation, no flicker. - const _fps = 2; + const _hasCustomMedia = !!(avatarMediaData && avatarMediaType); + const _isCustomVideo = _hasCustomMedia && avatarMediaType.startsWith('video/'); + const _fps = _isCustomVideo ? 15 : 2; w.__startBotAvatarStream = () => { if ( w.__botAvatarStreamStarted @@ -196,8 +200,8 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => w.__botAvatarStreamStarted = true; w.__botAvatarDisplayLabel = displayLabel; const canvas = document.createElement('canvas'); - canvas.width = 640; - canvas.height = 360; + canvas.width = _hasCustomMedia ? 1280 : 640; + canvas.height = _hasCustomMedia ? 720 : 360; canvas.setAttribute('data-poweron-avatar', '1'); // Render at a real size so the compositor produces frames in headless mode. // captureStream() in headless Chromium can stall when the canvas is 0/invisible. @@ -206,10 +210,10 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => (document.body || document.documentElement).appendChild(canvas); w.__botAvatarCanvas = canvas; const c2d = canvas.getContext('2d'); - const draw = () => { - if (!c2d) { - return; - } + const _isVideo = avatarMediaType.startsWith('video/'); + + const _drawFallback = () => { + if (!c2d) return; const wPx = canvas.width; const hPx = canvas.height; c2d.fillStyle = avatarBgColor; @@ -221,6 +225,58 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => const line = (w.__botAvatarDisplayLabel || displayLabel).toString().slice(0, 72); c2d.fillText(line, wPx / 2, hPx / 2); }; + + let _mediaReady = false; + let _mediaElement: any = null; + + if (avatarMediaData && avatarMediaType) { + const dataUrl = 'data:' + avatarMediaType + ';base64,' + avatarMediaData; + if (_isVideo) { + const video = document.createElement('video'); + video.src = dataUrl; + video.loop = true; + video.muted = true; + video.playsInline = true; + video.style.display = 'none'; + (document.body || document.documentElement).appendChild(video); + video.play().catch(() => { /* autoplay blocked — fall back to static */ }); + video.addEventListener('playing', () => { _mediaReady = true; }, { once: true }); + _mediaElement = video; + } else { + const img = new Image(); + img.src = dataUrl; + img.onload = () => { _mediaReady = true; }; + _mediaElement = img; + } + } + + const draw = () => { + if (!c2d) return; + if (_mediaReady && _mediaElement) { + try { + const cW = canvas.width; + const cH = canvas.height; + const srcW = _isVideo + ? (_mediaElement as HTMLVideoElement).videoWidth || cW + : (_mediaElement as HTMLImageElement).naturalWidth || cW; + const srcH = _isVideo + ? (_mediaElement as HTMLVideoElement).videoHeight || cH + : (_mediaElement as HTMLImageElement).naturalHeight || cH; + const scale = Math.min(cW / srcW, cH / srcH); + const dW = srcW * scale; + const dH = srcH * scale; + const dX = (cW - dW) / 2; + const dY = (cH - dH) / 2; + c2d.fillStyle = avatarBgColor; + c2d.fillRect(0, 0, cW, cH); + c2d.drawImage(_mediaElement, dX, dY, dW, dH); + return; + } catch { + // corrupted frame — fall through to fallback + } + } + _drawFallback(); + }; draw(); // Capture at fps for compositor-driven frames AND also push manual frames // via requestFrame() each tick for headless reliability. @@ -230,9 +286,7 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => if (w.__botAvatarVideoTrack) { w.__botAvatarVideoTrack.enabled = true; try { - // 'detail' = static / low-motion content -> WebRTC uses lower - // bitrate + preserves text sharpness instead of motion smoothing. - w.__botAvatarVideoTrack.contentHint = 'detail'; + w.__botAvatarVideoTrack.contentHint = _isCustomVideo ? 'motion' : 'detail'; } catch { // ignore } diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts index becd907..9f0eff9 100644 --- a/src/bot/orchestrator.ts +++ b/src/bot/orchestrator.ts @@ -35,6 +35,10 @@ export interface OrchestratorOptions { botAccountPassword?: string; transferMode?: string; debugMode?: boolean; + /** Base64-encoded image/video data for custom bot avatar. */ + avatarMediaData?: string; + /** MIME type of the avatar media (e.g. image/png, video/mp4). */ + avatarMediaType?: string; } /** @@ -253,11 +257,13 @@ export class BotOrchestrator { this._startCanvasRebindAfterJoin(); } - // Enable transcript capture (captions or audio based on transferMode) - await this._enableTranscriptCapture(); - - // Enable chat monitoring - await this._enableChat(); + // Anonymous bots cannot activate Teams captions — skip caption wait, + // only start audio capture + chat in parallel. + await Promise.all([ + this._enableAudioCapture(), + this._enableChat(), + ]); + this._logger.info('Skipping captions for anonymous bot (no permission in Teams)'); await this._takeScreenshot('anon-step5-ready', this._isDebugMode); // Send greeting in meeting chat @@ -460,8 +466,10 @@ export class BotOrchestrator { await this._ensureCameraOnInMeeting(); this._startCanvasRebindAfterJoin(); } - await this._enableTranscriptCapture(); - await this._enableChat(); + await Promise.all([ + this._enableTranscriptCapture(), + this._enableChat(), + ]); await this._sendJoinGreeting(); } @@ -1164,6 +1172,8 @@ export class BotOrchestrator { displayLabel: this._botName, avatarBgColor: config.botAvatarBgColor, avatarTextColor: config.botAvatarTextColor, + avatarMediaData: this._options.avatarMediaData, + avatarMediaType: this._options.avatarMediaType, }); this._teamsActions = new TeamsActionsService(this._page, this._logger); this._chatProcedure = new ChatProcedure( diff --git a/src/index.ts b/src/index.ts index 5efc754..e1b605d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,8 +19,8 @@ async function main(): Promise { // Start HTTP server httpServer = new HttpServer({ - onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode) => { - await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode); + onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType) => { + await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType); }, onLeaveRequest: async (sessionId) => { await sessionManager.endSession(sessionId); diff --git a/src/server/httpServer.ts b/src/server/httpServer.ts index 087d457..5264509 100644 --- a/src/server/httpServer.ts +++ b/src/server/httpServer.ts @@ -7,7 +7,7 @@ import { config } from '../config'; import { runAuthTests, runSingleVariant, getVariantIds } from '../bot/authTestProcedure'; export interface HttpServerCallbacks { - onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, transferMode?: string, debugMode?: boolean) => Promise; + onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, transferMode?: string, debugMode?: boolean, avatarMediaData?: string, avatarMediaType?: string) => Promise; onLeaveRequest: (sessionId: string) => Promise; onStatusRequest: (sessionId: string) => { state: string; error?: string } | null; } @@ -62,7 +62,7 @@ export class HttpServer { } private _setupMiddleware(): void { - this._app.use(express.json()); + this._app.use(express.json({ limit: '50mb' })); // Request logging this._app.use((req, res, next) => { @@ -80,16 +80,16 @@ export class HttpServer { // Deploy a new bot this._app.post('/api/bot', async (req: Request, res: Response) => { try { - const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode } = req.body; + const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType } = req.body; - logger.debug(`POST /api/bot: sessionId=${sessionId}, hasEmail=${!!botAccountEmail}, hasPassword=${!!botAccountPassword}, transferMode=${transferMode}, debugMode=${debugMode}`); + logger.debug(`POST /api/bot: sessionId=${sessionId}, hasEmail=${!!botAccountEmail}, hasPassword=${!!botAccountPassword}, transferMode=${transferMode}, debugMode=${debugMode}, hasAvatar=${!!avatarMediaData}`); if (!sessionId || !meetingUrl) { res.status(400).json({ error: 'Missing required fields: sessionId, meetingUrl' }); return; } - await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode); + await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType); res.json({ success: true, diff --git a/src/sessionManager.ts b/src/sessionManager.ts index 5219a39..fcd32e5 100644 --- a/src/sessionManager.ts +++ b/src/sessionManager.ts @@ -43,6 +43,8 @@ export class SessionManager { botAccountPassword?: string, transferMode?: string, debugMode?: boolean, + avatarMediaData?: string, + avatarMediaType?: string, ): Promise { if (this._sessions.has(sessionId)) { logger.warn(`Session ${sessionId} already exists`); @@ -78,6 +80,8 @@ export class SessionManager { botAccountPassword: botAccountPassword, transferMode: transferMode, debugMode: debugMode, + avatarMediaData: avatarMediaData, + avatarMediaType: avatarMediaType, }; const orchestrator = new BotOrchestrator(