feat: fix chat in both-mode, add TeamsActionsService for AI commands
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
ValueOn AG
parent
4120a97e9f
commit
777bc198a2
2 changed files with 338 additions and 6 deletions
|
|
@ -14,6 +14,7 @@ import { AudioProcedure } from './audioProcedure';
|
|||
import { AudioCaptureProcedure } from './audioCaptureProcedure';
|
||||
import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
|
||||
import { AuthProcedure } from './authProcedure';
|
||||
import { TeamsActionsService } from './teamsActionsService';
|
||||
import { isValidMeetingUrl } from './meetingUrlParser';
|
||||
|
||||
// Camera / fake video injection is disabled for now to focus on stability.
|
||||
|
|
@ -64,6 +65,7 @@ export class BotOrchestrator {
|
|||
private _audioProcedure: AudioProcedure | null = null;
|
||||
private _audioCaptureProcedure: AudioCaptureProcedure | null = null;
|
||||
private _chatProcedure: ChatProcedure | null = null;
|
||||
private _teamsActions: TeamsActionsService | null = null;
|
||||
|
||||
private _state: BotState = 'idle';
|
||||
private _isShuttingDown: boolean = false;
|
||||
|
|
@ -617,7 +619,9 @@ export class BotOrchestrator {
|
|||
});
|
||||
|
||||
this._gatewayWs.on('message', (data) => {
|
||||
this._handleGatewayMessage(data.toString());
|
||||
this._handleGatewayMessage(data.toString()).catch((err) => {
|
||||
this._logger.error('Unhandled error in gateway message handler:', err);
|
||||
});
|
||||
});
|
||||
|
||||
this._gatewayWs.on('close', (code, reason) => {
|
||||
|
|
@ -641,20 +645,27 @@ export class BotOrchestrator {
|
|||
|
||||
/**
|
||||
* Handle incoming messages from the Gateway.
|
||||
* Async operations are awaited to ensure proper error handling
|
||||
* and serialized execution (e.g. audio + chat don't interfere).
|
||||
*/
|
||||
private _handleGatewayMessage(data: string): void {
|
||||
private async _handleGatewayMessage(data: string): Promise<void> {
|
||||
try {
|
||||
const message = JSON.parse(data);
|
||||
|
||||
switch (message.type) {
|
||||
case 'playAudio':
|
||||
const audioMsg = message as PlayAudioMessage;
|
||||
this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
|
||||
await this.playAudio(audioMsg.audio.data, audioMsg.audio.format);
|
||||
break;
|
||||
|
||||
case 'sendChatMessage':
|
||||
const chatMsg = message as SendChatMessage;
|
||||
this.sendChatMessageToMeeting(chatMsg.text);
|
||||
this._logger.info(`Gateway sendChatMessage received: ${chatMsg.text?.substring(0, 60)}...`);
|
||||
try {
|
||||
await this.sendChatMessageToMeeting(chatMsg.text);
|
||||
} catch (chatErr) {
|
||||
this._logger.error(`Failed to send chat message to meeting: ${chatErr}`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'stopAudio':
|
||||
|
|
@ -664,15 +675,18 @@ export class BotOrchestrator {
|
|||
}
|
||||
break;
|
||||
|
||||
case 'botCommand':
|
||||
await this._handleBotCommand(message.command, message.params || {});
|
||||
break;
|
||||
|
||||
case 'pong':
|
||||
// Heartbeat response
|
||||
break;
|
||||
|
||||
default:
|
||||
this._logger.debug('Unknown Gateway message type:', message.type);
|
||||
}
|
||||
} catch (error) {
|
||||
this._logger.error('Error parsing Gateway message:', error);
|
||||
this._logger.error('Error handling Gateway message:', error);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -925,6 +939,7 @@ export class BotOrchestrator {
|
|||
this._options.language
|
||||
);
|
||||
this._audioProcedure = new AudioProcedure(this._page, this._logger);
|
||||
this._teamsActions = new TeamsActionsService(this._page, this._logger);
|
||||
this._chatProcedure = new ChatProcedure(
|
||||
this._page,
|
||||
this._logger,
|
||||
|
|
@ -1203,6 +1218,36 @@ export class BotOrchestrator {
|
|||
await this._chatProcedure.sendChatMessage(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle structured commands from the Gateway (issued by AI).
|
||||
*/
|
||||
private async _handleBotCommand(command: string, params: Record<string, any>): Promise<void> {
|
||||
if (!this._teamsActions || this._state !== 'in_meeting') {
|
||||
this._logger.warn(`Cannot execute command '${command}' — not in meeting`);
|
||||
return;
|
||||
}
|
||||
|
||||
this._logger.info(`Executing bot command: ${command} params=${JSON.stringify(params)}`);
|
||||
|
||||
try {
|
||||
switch (command) {
|
||||
case 'toggleTranscript':
|
||||
await this._teamsActions.toggleTranscript(params.enable !== false);
|
||||
break;
|
||||
case 'toggleMic':
|
||||
await this._teamsActions.toggleMic(params.enable !== false);
|
||||
break;
|
||||
case 'toggleCamera':
|
||||
await this._teamsActions.toggleCamera(params.enable !== false);
|
||||
break;
|
||||
default:
|
||||
this._logger.warn(`Unknown bot command: ${command}`);
|
||||
}
|
||||
} catch (err) {
|
||||
this._logger.error(`Bot command '${command}' failed: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the bot state and notify callbacks + Gateway.
|
||||
*/
|
||||
|
|
|
|||
287
src/bot/teamsActionsService.ts
Normal file
287
src/bot/teamsActionsService.ts
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
import { Page } from 'playwright';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
/**
|
||||
* Service center for all Teams meeting UI actions.
|
||||
*
|
||||
* Each method is idempotent and state-aware: it checks the current state
|
||||
* before acting (e.g. won't toggle transcript OFF if it's already ON).
|
||||
* This makes calls safe to repeat and resilient to UI flicker.
|
||||
*/
|
||||
export class TeamsActionsService {
|
||||
private _page: Page;
|
||||
private _logger: Logger;
|
||||
|
||||
constructor(page: Page, logger: Logger) {
|
||||
this._page = page;
|
||||
this._logger = logger;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Transcript / Captions
|
||||
// =========================================================================
|
||||
|
||||
async toggleTranscript(enable: boolean): Promise<boolean> {
|
||||
this._logger.info(`TeamsActions: toggleTranscript enable=${enable}`);
|
||||
|
||||
try {
|
||||
// Open "More" menu
|
||||
if (!(await this._openMoreMenu())) return false;
|
||||
|
||||
// Try "Language and speech" → submenu → "Show live captions"
|
||||
const captionsBtn = await this._findCaptionsButton();
|
||||
if (captionsBtn) {
|
||||
const ariaChecked = await captionsBtn.getAttribute('aria-checked');
|
||||
const text = await captionsBtn.evaluate((el: HTMLElement) => el.textContent?.trim() || '');
|
||||
const isCurrentlyOn = ariaChecked === 'true' || text.toLowerCase().includes('hide');
|
||||
|
||||
if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) {
|
||||
this._logger.info(`TeamsActions: Transcript already ${enable ? 'ON' : 'OFF'} — no action needed`);
|
||||
await this._page.keyboard.press('Escape');
|
||||
return true;
|
||||
}
|
||||
|
||||
await captionsBtn.click();
|
||||
this._logger.info(`TeamsActions: Transcript toggled ${enable ? 'ON' : 'OFF'}`);
|
||||
await this._page.waitForTimeout(1000);
|
||||
return true;
|
||||
}
|
||||
|
||||
await this._page.keyboard.press('Escape');
|
||||
this._logger.warn('TeamsActions: Could not find captions button');
|
||||
return false;
|
||||
} catch (err) {
|
||||
this._logger.warn(`TeamsActions: toggleTranscript error: ${err}`);
|
||||
await this._page.keyboard.press('Escape').catch(() => {});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Chat
|
||||
// =========================================================================
|
||||
|
||||
async sendChatMessage(text: string): Promise<boolean> {
|
||||
this._logger.info(`TeamsActions: sendChatMessage: ${text.substring(0, 60)}...`);
|
||||
|
||||
try {
|
||||
await this._ensureChatPanelOpen();
|
||||
|
||||
const inputSelectors = [
|
||||
'[data-tid="ckeditor-replyConversation"]',
|
||||
'div[role="textbox"][data-tid*="chat"]',
|
||||
'div[role="textbox"][aria-label*="message" i]',
|
||||
'div[role="textbox"][aria-label*="Nachricht" i]',
|
||||
'div[contenteditable="true"][data-tid*="message"]',
|
||||
'div[contenteditable="true"][aria-label*="message" i]',
|
||||
];
|
||||
|
||||
let input: any = null;
|
||||
for (const selector of inputSelectors) {
|
||||
input = await this._page.$(selector);
|
||||
if (input) break;
|
||||
}
|
||||
|
||||
if (!input) {
|
||||
this._logger.warn('TeamsActions: Could not find chat input field');
|
||||
return false;
|
||||
}
|
||||
|
||||
await input.click();
|
||||
await this._page.waitForTimeout(200);
|
||||
await this._page.keyboard.type(text, { delay: 10 });
|
||||
await this._page.waitForTimeout(200);
|
||||
await this._page.keyboard.press('Enter');
|
||||
|
||||
this._logger.info('TeamsActions: Chat message sent');
|
||||
return true;
|
||||
} catch (err) {
|
||||
this._logger.error(`TeamsActions: sendChatMessage error: ${err}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Microphone
|
||||
// =========================================================================
|
||||
|
||||
async toggleMic(enable: boolean): Promise<boolean> {
|
||||
this._logger.info(`TeamsActions: toggleMic enable=${enable}`);
|
||||
|
||||
try {
|
||||
const micBtn = await this._page.$('button#microphone-button');
|
||||
if (!micBtn) {
|
||||
const fallbacks = [
|
||||
'button[data-inp="microphone-button"]',
|
||||
'button[aria-label*="microphone" i]',
|
||||
'button[aria-label*="Mikrofon" i]',
|
||||
];
|
||||
for (const sel of fallbacks) {
|
||||
const btn = await this._page.$(sel);
|
||||
if (btn) return this._toggleMediaButton(btn, enable, 'mic');
|
||||
}
|
||||
this._logger.warn('TeamsActions: Microphone button not found');
|
||||
return false;
|
||||
}
|
||||
|
||||
return this._toggleMediaButton(micBtn, enable, 'mic');
|
||||
} catch (err) {
|
||||
this._logger.warn(`TeamsActions: toggleMic error: ${err}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Camera
|
||||
// =========================================================================
|
||||
|
||||
async toggleCamera(enable: boolean): Promise<boolean> {
|
||||
this._logger.info(`TeamsActions: toggleCamera enable=${enable}`);
|
||||
|
||||
try {
|
||||
const camBtn = await this._page.$('button#video-button');
|
||||
if (!camBtn) {
|
||||
const fallbacks = [
|
||||
'button[data-inp="video-button"]',
|
||||
'button[aria-label*="camera" i]',
|
||||
'button[aria-label*="Camera" i]',
|
||||
'button[aria-label*="Video" i]',
|
||||
];
|
||||
for (const sel of fallbacks) {
|
||||
const btn = await this._page.$(sel);
|
||||
if (btn) return this._toggleMediaButton(btn, enable, 'camera');
|
||||
}
|
||||
this._logger.warn('TeamsActions: Camera button not found');
|
||||
return false;
|
||||
}
|
||||
|
||||
return this._toggleMediaButton(camBtn, enable, 'camera');
|
||||
} catch (err) {
|
||||
this._logger.warn(`TeamsActions: toggleCamera error: ${err}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Internal Helpers
|
||||
// =========================================================================
|
||||
|
||||
private async _toggleMediaButton(
|
||||
btn: any,
|
||||
enable: boolean,
|
||||
name: string,
|
||||
): Promise<boolean> {
|
||||
const state = await btn.evaluate((el: HTMLElement) => ({
|
||||
dataState: el.getAttribute('data-state') || '',
|
||||
ariaLabel: el.getAttribute('aria-label') || '',
|
||||
}));
|
||||
|
||||
const isOff =
|
||||
state.dataState.includes('off') ||
|
||||
state.ariaLabel.toLowerCase().includes('turn on') ||
|
||||
state.ariaLabel.toLowerCase().includes('einschalten');
|
||||
|
||||
const isCurrentlyOn = !isOff;
|
||||
|
||||
if ((enable && isCurrentlyOn) || (!enable && !isCurrentlyOn)) {
|
||||
this._logger.info(`TeamsActions: ${name} already ${enable ? 'ON' : 'OFF'}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
await btn.click();
|
||||
this._logger.info(`TeamsActions: ${name} toggled ${enable ? 'ON' : 'OFF'}`);
|
||||
await this._page.waitForTimeout(500);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async _openMoreMenu(): Promise<boolean> {
|
||||
const selectors = [
|
||||
'button[id="callingButtons-showMoreBtn"]',
|
||||
'[data-tid="callingButtons-showMoreBtn"]',
|
||||
'button[aria-label*="More actions"]',
|
||||
'button[aria-label*="More"]',
|
||||
'[data-tid="more-button"]',
|
||||
];
|
||||
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
const button = await this._page.$(selector);
|
||||
if (button) {
|
||||
await button.click();
|
||||
await this._page.waitForTimeout(1000);
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
this._logger.warn('TeamsActions: Could not find More actions menu');
|
||||
return false;
|
||||
}
|
||||
|
||||
private async _findCaptionsButton(): Promise<any> {
|
||||
// Direct button
|
||||
let btn = await this._page.$('#closed-captions-button');
|
||||
if (btn) return btn;
|
||||
|
||||
// "Language and speech" → submenu
|
||||
const langSpeechSelectors = [
|
||||
'#LanguageSpeechMenuControl-id',
|
||||
'[data-tid="LanguageSpeechMenuControl-id"]',
|
||||
'div[role="menuitem"]:has-text("Language and speech")',
|
||||
'div[role="menuitem"]:has-text("Sprache und Spracheingabe")',
|
||||
];
|
||||
|
||||
for (const sel of langSpeechSelectors) {
|
||||
try {
|
||||
const item = await this._page.$(sel);
|
||||
if (item) {
|
||||
await item.click();
|
||||
await this._page.waitForTimeout(1500);
|
||||
btn = await this._page.$('#closed-captions-button');
|
||||
if (btn) return btn;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private async _ensureChatPanelOpen(): Promise<void> {
|
||||
const alreadyOpen = await this._page.evaluate(() => {
|
||||
const chatBtn = document.querySelector(
|
||||
'button[id="chat-button"], button[data-tid="chat-button"]',
|
||||
) as HTMLElement | null;
|
||||
if (chatBtn && chatBtn.getAttribute('aria-pressed') === 'true') return true;
|
||||
const chatInput = document.querySelector(
|
||||
'[data-tid="ckeditor-replyConversation"], div[role="textbox"][data-tid*="chat"]',
|
||||
) as HTMLElement | null;
|
||||
return chatInput ? chatInput.offsetHeight > 0 : false;
|
||||
});
|
||||
|
||||
if (alreadyOpen) return;
|
||||
|
||||
const chatButtonSelectors = [
|
||||
'button[id="chat-button"]',
|
||||
'button[data-tid="chat-button"]',
|
||||
'button[aria-label*="Chat" i]',
|
||||
'#chat-button',
|
||||
];
|
||||
|
||||
for (const selector of chatButtonSelectors) {
|
||||
try {
|
||||
const button = await this._page.$(selector);
|
||||
if (button) {
|
||||
await button.click();
|
||||
this._logger.info(`TeamsActions: Opened chat panel: ${selector}`);
|
||||
await this._page.waitForTimeout(1000);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue