PowerOn/service-teams-browser-bot
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

anonymous bot working

Browse source
This commit is contained in:
ValueOn AG 2026-05-12 19:16:07 +02:00
parent bad6e67ca0
commit 09dc63d75c
6 changed files with 108 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/bot/audioProcedure.ts
View file

@ -24,6 +24,10 @@ export type AudioProcedureOptions = {
avatarBgColor?: string;
/** Hex/CSS color of the centered display label (default: dark blue). */
avatarTextColor?: string;
/** Base64-encoded image/video data for custom bot avatar. */
avatarMediaData?: string;
/** MIME type of the avatar media (e.g. image/png, video/mp4). */
avatarMediaType?: string;
};
export class AudioProcedure {
@ -33,6 +37,8 @@ export class AudioProcedure {
private _displayLabel: string;
private _avatarBgColor: string;
private _avatarTextColor: string;
private _avatarMediaData?: string;
private _avatarMediaType?: string;
private _audioContext: boolean = false;
private _initScriptInjected: boolean = false;
private _audioQueue: Array<{ audioData: string; format: 'mp3' | 'wav' | 'pcm' }> = [];
@ -46,6 +52,8 @@ export class AudioProcedure {
this._displayLabel = (options?.displayLabel || 'Bot').trim() || 'Bot';
this._avatarBgColor = (options?.avatarBgColor || '').trim() || '#a8d4f0';
this._avatarTextColor = (options?.avatarTextColor || '').trim() || '#1a3552';
this._avatarMediaData = options?.avatarMediaData;
this._avatarMediaType = options?.avatarMediaType;
}
/**
@ -69,6 +77,8 @@ export class AudioProcedure {
displayLabel: this._displayLabel,
avatarBgColor: this._avatarBgColor,
avatarTextColor: this._avatarTextColor,
avatarMediaData: this._avatarMediaData,
avatarMediaType: this._avatarMediaType,
});
this._initScriptInjected = true;
@ -85,6 +95,8 @@ export class AudioProcedure {
displayLabel: this._displayLabel,
avatarBgColor: this._avatarBgColor,
avatarTextColor: this._avatarTextColor,
avatarMediaData: this._avatarMediaData,
avatarMediaType: this._avatarMediaType,
};
for (const frame of this._page.frames()) {
try {

82
src/bot/mediaGetUserMediaPatch.ts
View file

@ -12,6 +12,10 @@ export type MediaGetUserMediaPatchOptions = {
avatarBgColor?: string;
/** Hex/CSS color of the centered display label (default: dark blue). */
avatarTextColor?: string;
/** Base64-encoded image/video data for custom bot avatar. */
avatarMediaData?: string;
/** MIME type of the avatar media (e.g. image/png, video/mp4). */
avatarMediaType?: string;
};
export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) => {
@ -19,6 +23,8 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
const { useCanvasVideo, displayLabel } = opts;
const avatarBgColor = opts.avatarBgColor || '#a8d4f0';
const avatarTextColor = opts.avatarTextColor || '#1a3552';
const avatarMediaData = opts.avatarMediaData || '';
const avatarMediaType = opts.avatarMediaType || '';
const w: any = window as any;
if (!w.__gumChromium) {
@ -169,11 +175,9 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
return;
}
// Low fps: the avatar is intentionally STATIC. The interval still ticks so
// captureStream() in headless Chromium gets fresh frames (some Chromium
// builds pause the track if no new frames arrive), but each tick draws an
// identical image — no animation, no flicker.
const _fps = 2;
const _hasCustomMedia = !!(avatarMediaData && avatarMediaType);
const _isCustomVideo = _hasCustomMedia && avatarMediaType.startsWith('video/');
const _fps = _isCustomVideo ? 15 : 2;
w.__startBotAvatarStream = () => {
if (
w.__botAvatarStreamStarted
@ -196,8 +200,8 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
w.__botAvatarStreamStarted = true;
w.__botAvatarDisplayLabel = displayLabel;
const canvas = document.createElement('canvas');
canvas.width = 640;
canvas.height = 360;
canvas.width = _hasCustomMedia ? 1280 : 640;
canvas.height = _hasCustomMedia ? 720 : 360;
canvas.setAttribute('data-poweron-avatar', '1');
// Render at a real size so the compositor produces frames in headless mode.
// captureStream() in headless Chromium can stall when the canvas is 0/invisible.
@ -206,10 +210,10 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
(document.body || document.documentElement).appendChild(canvas);
w.__botAvatarCanvas = canvas;
const c2d = canvas.getContext('2d');
const draw = () => {
if (!c2d) {
return;
}
const _isVideo = avatarMediaType.startsWith('video/');
const _drawFallback = () => {
if (!c2d) return;
const wPx = canvas.width;
const hPx = canvas.height;
c2d.fillStyle = avatarBgColor;
@ -221,6 +225,58 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
const line = (w.__botAvatarDisplayLabel || displayLabel).toString().slice(0, 72);
c2d.fillText(line, wPx / 2, hPx / 2);
};
let _mediaReady = false;
let _mediaElement: any = null;
if (avatarMediaData && avatarMediaType) {
const dataUrl = 'data:' + avatarMediaType + ';base64,' + avatarMediaData;
if (_isVideo) {
const video = document.createElement('video');
video.src = dataUrl;
video.loop = true;
video.muted = true;
video.playsInline = true;
video.style.display = 'none';
(document.body || document.documentElement).appendChild(video);
video.play().catch(() => { /* autoplay blocked — fall back to static */ });
video.addEventListener('playing', () => { _mediaReady = true; }, { once: true });
_mediaElement = video;
} else {
const img = new Image();
img.src = dataUrl;
img.onload = () => { _mediaReady = true; };
_mediaElement = img;
}
}
const draw = () => {
if (!c2d) return;
if (_mediaReady && _mediaElement) {
try {
const cW = canvas.width;
const cH = canvas.height;
const srcW = _isVideo
? (_mediaElement as HTMLVideoElement).videoWidth || cW
: (_mediaElement as HTMLImageElement).naturalWidth || cW;
const srcH = _isVideo
? (_mediaElement as HTMLVideoElement).videoHeight || cH
: (_mediaElement as HTMLImageElement).naturalHeight || cH;
const scale = Math.min(cW / srcW, cH / srcH);
const dW = srcW * scale;
const dH = srcH * scale;
const dX = (cW - dW) / 2;
const dY = (cH - dH) / 2;
c2d.fillStyle = avatarBgColor;
c2d.fillRect(0, 0, cW, cH);
c2d.drawImage(_mediaElement, dX, dY, dW, dH);
return;
} catch {
// corrupted frame — fall through to fallback
}
}
_drawFallback();
};
draw();
// Capture at fps for compositor-driven frames AND also push manual frames
// via requestFrame() each tick for headless reliability.
@ -230,9 +286,7 @@ export const poweronMediaPatchInstall = (opts: MediaGetUserMediaPatchOptions) =>
if (w.__botAvatarVideoTrack) {
w.__botAvatarVideoTrack.enabled = true;
try {
// 'detail' = static / low-motion content -> WebRTC uses lower
// bitrate + preserves text sharpness instead of motion smoothing.
w.__botAvatarVideoTrack.contentHint = 'detail';
w.__botAvatarVideoTrack.contentHint = _isCustomVideo ? 'motion' : 'detail';
} catch {
// ignore
}

24
src/bot/orchestrator.ts
View file

@ -35,6 +35,10 @@ export interface OrchestratorOptions {
botAccountPassword?: string;
transferMode?: string;
debugMode?: boolean;
/** Base64-encoded image/video data for custom bot avatar. */
avatarMediaData?: string;
/** MIME type of the avatar media (e.g. image/png, video/mp4). */
avatarMediaType?: string;
}
/**
@ -253,11 +257,13 @@ export class BotOrchestrator {
this._startCanvasRebindAfterJoin();
}
// Enable transcript capture (captions or audio based on transferMode)
await this._enableTranscriptCapture();
// Enable chat monitoring
await this._enableChat();
// Anonymous bots cannot activate Teams captions — skip caption wait,
// only start audio capture + chat in parallel.
await Promise.all([
this._enableAudioCapture(),
this._enableChat(),
]);
this._logger.info('Skipping captions for anonymous bot (no permission in Teams)');
await this._takeScreenshot('anon-step5-ready', this._isDebugMode);
// Send greeting in meeting chat
@ -460,8 +466,10 @@ export class BotOrchestrator {
await this._ensureCameraOnInMeeting();
this._startCanvasRebindAfterJoin();
}
await this._enableTranscriptCapture();
await this._enableChat();
await Promise.all([
this._enableTranscriptCapture(),
this._enableChat(),
]);
await this._sendJoinGreeting();
}
@ -1164,6 +1172,8 @@ export class BotOrchestrator {
displayLabel: this._botName,
avatarBgColor: config.botAvatarBgColor,
avatarTextColor: config.botAvatarTextColor,
avatarMediaData: this._options.avatarMediaData,
avatarMediaType: this._options.avatarMediaType,
});
this._teamsActions = new TeamsActionsService(this._page, this._logger);
this._chatProcedure = new ChatProcedure(

4
src/index.ts
View file

@ -19,8 +19,8 @@ async function main(): Promise<void> {
// Start HTTP server
httpServer = new HttpServer({
onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode) => {
await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode);
onJoinRequest: async (sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType) => {
await sessionManager.createSession(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType);
},
onLeaveRequest: async (sessionId) => {
await sessionManager.endSession(sessionId);

10
src/server/httpServer.ts
View file

@ -7,7 +7,7 @@ import { config } from '../config';
import { runAuthTests, runSingleVariant, getVariantIds } from '../bot/authTestProcedure';
export interface HttpServerCallbacks {
onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, transferMode?: string, debugMode?: boolean) => Promise<void>;
onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string, instanceId?: string, gatewayWsUrl?: string, language?: string, botAccountEmail?: string, botAccountPassword?: string, transferMode?: string, debugMode?: boolean, avatarMediaData?: string, avatarMediaType?: string) => Promise<void>;
onLeaveRequest: (sessionId: string) => Promise<void>;
onStatusRequest: (sessionId: string) => { state: string; error?: string } | null;
}
@ -62,7 +62,7 @@ export class HttpServer {
}
private _setupMiddleware(): void {
this._app.use(express.json());
this._app.use(express.json({ limit: '50mb' }));
// Request logging
this._app.use((req, res, next) => {
@ -80,16 +80,16 @@ export class HttpServer {
// Deploy a new bot
this._app.post('/api/bot', async (req: Request, res: Response) => {
try {
const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode } = req.body;
const { sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType } = req.body;
logger.debug(`POST /api/bot: sessionId=${sessionId}, hasEmail=${!!botAccountEmail}, hasPassword=${!!botAccountPassword}, transferMode=${transferMode}, debugMode=${debugMode}`);
logger.debug(`POST /api/bot: sessionId=${sessionId}, hasEmail=${!!botAccountEmail}, hasPassword=${!!botAccountPassword}, transferMode=${transferMode}, debugMode=${debugMode}, hasAvatar=${!!avatarMediaData}`);
if (!sessionId || !meetingUrl) {
res.status(400).json({ error: 'Missing required fields: sessionId, meetingUrl' });
return;
}
await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode);
await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName, instanceId, gatewayWsUrl, language, botAccountEmail, botAccountPassword, transferMode, debugMode, avatarMediaData, avatarMediaType);
res.json({
success: true,

4
src/sessionManager.ts
View file

@ -43,6 +43,8 @@ export class SessionManager {
botAccountPassword?: string,
transferMode?: string,
debugMode?: boolean,
avatarMediaData?: string,
avatarMediaType?: string,
): Promise<void> {
if (this._sessions.has(sessionId)) {
logger.warn(`Session ${sessionId} already exists`);
@ -78,6 +80,8 @@ export class SessionManager {
botAccountPassword: botAccountPassword,
transferMode: transferMode,
debugMode: debugMode,
avatarMediaData: avatarMediaData,
avatarMediaType: avatarMediaType,
};
const orchestrator = new BotOrchestrator(