PowerOn/service-teams-browser-bot
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

teamsbot auth fixes

Browse source
This commit is contained in:
ValueOn AG 2026-05-12 21:31:25 +02:00
parent 09dc63d75c
commit 414e2a5e40
3 changed files with 213 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
src/bot/captionsProcedure.ts
View file

@ -65,15 +65,17 @@ export class CaptionsProcedure {
* Works for both anonymous (light-meetings) and authenticated (full Teams) UI.
*/
private async _openMoreMenu(): Promise<void> {
const allSelectors = [
// Specific call-controls selectors first, broad ones last.
// In the full Teams web app, broad selectors like `aria-label*="More"`
// might match sidebar navigation buttons — restrict them to the
// call-controls area to avoid navigating away from the meeting.
const specificSelectors = [
'button[id="callingButtons-showMoreBtn"]',
'[data-tid="callingButtons-showMoreBtn"]',
'button[aria-label*="More actions"]',
'button[aria-label*="More"]',
'[data-tid="more-button"]',
];
for (const selector of allSelectors) {
for (const selector of specificSelectors) {
try {
const button = await this._page.$(selector);
if (button) {
@ -87,10 +89,37 @@ export class CaptionsProcedure {
}
}
// Broader selectors scoped to the calling/meeting toolbar only
const found = await this._page.evaluate(() => {
const btns = Array.from(document.querySelectorAll('button')) as HTMLElement[];
for (const btn of btns) {
const aria = (btn.getAttribute('aria-label') || '').toLowerCase();
if (!aria.includes('more')) continue;
if (btn.offsetHeight === 0 || btn.offsetWidth === 0) continue;
// Exclude sidebar navigation buttons
if (btn.closest('[data-tid="app-bar"], nav[role="navigation"]')) continue;
// Prefer buttons near calling controls
const nearCalling = btn.closest(
'[data-tid="calling-unified-bar"], [data-tid="call-controls"], '
+ '[data-tid="callingButtons-area"], [id="callingButtons-area"]',
);
if (nearCalling || aria.includes('more actions') || aria.includes('weitere aktionen')) {
btn.click();
return aria;
}
}
return null;
});
if (found) {
this._logger.info(`Clicked "More" button via scoped search: "${found}"`);
await this._page.waitForTimeout(1000);
return;
}
// Last resort: wait for the primary selector with a short timeout
try {
await this._page.waitForSelector(allSelectors[0], { timeout: 10000 });
await this._page.click(allSelectors[0]);
await this._page.waitForSelector(specificSelectors[0], { timeout: 10000 });
await this._page.click(specificSelectors[0]);
this._logger.info('Found "More" button (after wait)');
await this._page.waitForTimeout(1000);
return;

205
src/bot/chatProcedure.ts
View file

@ -26,15 +26,18 @@ export class ChatProcedure {
private _scanIntervalId: ReturnType<typeof setInterval> | null = null;
private _consecutiveOpenFailures: number = 0;
private static readonly _MAX_OPEN_FAILURES = 5;
private _isAuthMode: boolean;
constructor(
page: Page,
logger: Logger,
onChatMessage: (entry: ChatMessageEntry) => void
onChatMessage: (entry: ChatMessageEntry) => void,
isAuthMode: boolean = false,
) {
this._page = page;
this._logger = logger;
this._onChatMessage = onChatMessage;
this._isAuthMode = isAuthMode;
}
/**
@ -55,7 +58,7 @@ export class ChatProcedure {
* authenticated Teams meeting layout.
*/
async enableChatMonitoring(): Promise<boolean> {
this._logger.info('Enabling chat monitoring...');
this._logger.info(`Enabling chat monitoring (authMode=${this._isAuthMode})...`);
await this._dumpChatButtonDiagnostics();
await this._openChatPanel();
@ -64,11 +67,6 @@ export class ChatProcedure {
const isOpen = await this._isChatPanelOpen();
if (isOpen) {
this._logger.info('Chat panel opened successfully');
// Light-meetings ships a "simplified compose" with a collapsed
// placeholder + dedicated expand button. The real ckeditor textbox
// is rendered but Playwright considers it invisible until expanded.
// Expand once now so the periodic scan and send path see the
// canonical ckeditor surface.
await this._ensureComposeExpanded();
} else {
this._logger.warn('Chat panel could not be opened - chat send/receive will not work');
@ -178,23 +176,44 @@ export class ChatProcedure {
*/
private async _isChatPanelOpen(): Promise<boolean> {
return this._page.evaluate(() => {
// Strategy 1: light-meetings / standard meeting UI — side panel container
const sidePanel = document.querySelector(
'[data-tid="calling-right-side-panel"]',
) as HTMLElement | null;
if (!sidePanel) return false;
const isVisible = sidePanel.offsetWidth > 0
&& sidePanel.offsetHeight > 0
&& sidePanel.offsetParent !== null;
if (!isVisible) return false;
const chatHallmark = sidePanel.querySelector(
'[data-tid="message-pane-layout"], '
+ '[data-tid="message-pane-body"], '
+ '[data-tid="chat-pane-compose-message-footer"], '
+ '[data-tid="message-pane-footer"], '
+ '#chat-pane-list, '
+ '[data-app-name="chats"]',
);
return chatHallmark !== null;
if (sidePanel) {
const isVisible = sidePanel.offsetWidth > 0
&& sidePanel.offsetHeight > 0
&& sidePanel.offsetParent !== null;
if (isVisible) {
const chatHallmark = sidePanel.querySelector(
'[data-tid="message-pane-layout"], '
+ '[data-tid="message-pane-body"], '
+ '[data-tid="chat-pane-compose-message-footer"], '
+ '[data-tid="message-pane-footer"], '
+ '#chat-pane-list, '
+ '[data-app-name="chats"]',
);
if (chatHallmark) return true;
}
}
// Strategy 2: full Teams web app (auth mode) — the chat toggle button
// has aria-pressed="true" when the in-meeting chat panel is open.
// The correct toggle is the one with a keyboard shortcut in the label
// (e.g. "Chat (Ctrl+Shift+2)").
const allBtns = Array.from(document.querySelectorAll(
'button[aria-pressed], [role="button"][aria-pressed]',
));
for (const btn of allBtns) {
const aria = (btn.getAttribute('aria-label') || '').toLowerCase();
const hasChatHint = aria.includes('chat') || aria.includes('unterhalt');
const hasShortcutHint = aria.includes('ctrl') || aria.includes('strg');
if (hasChatHint && hasShortcutHint && btn.getAttribute('aria-pressed') === 'true') {
return true;
}
}
return false;
});
}
@ -250,6 +269,26 @@ export class ChatProcedure {
|| v.includes('conversation'),
);
};
const isDangerousNavButton = (el: Element): boolean => {
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const tid = (el.getAttribute('data-tid') || '').toLowerCase();
const id = (el.id || '').toLowerCase();
// Teams sidebar "Chats" navigation — clicking navigates away from meeting
if (aria === 'chats' || aria === 'unterhaltungen') return true;
// Tab items inside the chat panel (Files, Recap, Whiteboard, etc.)
if (tid.startsWith('tab-item-')) return true;
// "More chat options" submenu triggers — open dropdowns, not chat panel
if (aria.includes('more chat options') || aria.includes('weitere chatoptionen')) return true;
// Participant count link
if (tid === 'chat-header-participant-count') return true;
// Chat join button (call join, not chat toggle)
if (tid === 'chat-join-button') return true;
// Chat app tab IDs (com.microsoft.chattabs.*)
if (id.startsWith('com.microsoft.chattabs.')) return true;
// Buttons inside the Teams left app-bar / sidebar navigation
if (el.closest('[data-tid="app-bar"], [data-tid="app-bar-search"], nav[role="navigation"]')) return true;
return false;
};
const isVisible = (el: HTMLElement): boolean =>
el.offsetHeight > 0 && el.offsetWidth > 0 && el.offsetParent !== null;
const keyOf = (el: Element): string =>
@ -263,7 +302,7 @@ export class ChatProcedure {
document.querySelectorAll('button, [role="button"], [role="menuitem"]'),
) as HTMLElement[];
const candidates = all
.filter((el) => matchesChatHint(el) && isVisible(el))
.filter((el) => matchesChatHint(el) && isVisible(el) && !isDangerousNavButton(el))
.filter((el) => !alreadyTried.includes(keyOf(el)));
if (candidates.length === 0) return { picked: null as null | { key: string; id: string; tid: string; aria: string; toggle: boolean } };
@ -308,6 +347,31 @@ export class ChatProcedure {
this._logger.info('Chat panel opened successfully');
return true;
}
// Fallback for auth mode: if we clicked a toggle button and its
// aria-pressed is now "true", the panel IS open even if the DOM
// detection via calling-right-side-panel fails (different layout
// in the full Teams web app).
if (click.picked.toggle) {
const pressedNow = await this._page.evaluate((btnKey: string) => {
const all = Array.from(
document.querySelectorAll('button, [role="button"], [role="menuitem"]'),
);
for (const el of all) {
const k = `${el.id || ''}|${el.getAttribute('data-tid') || ''}|${el.getAttribute('aria-label') || ''}`;
if (k === btnKey) return el.getAttribute('aria-pressed');
}
return null;
}, click.picked.key);
if (pressedNow === 'true') {
this._logger.info(
'Chat panel DOM not detected, but toggle aria-pressed="true" — treating as open (auth mode fallback)',
);
return true;
}
}
this._logger.info(
'Chat button clicked but panel not detected — will try a different candidate next round',
);
@ -662,30 +726,36 @@ export class ChatProcedure {
if (!this._isSubscribed || !this._page) return;
try {
if (!(await this._isChatPanelOpen())) {
if (this._consecutiveOpenFailures >= ChatProcedure._MAX_OPEN_FAILURES) {
this._logger.warn(
`Chat panel failed to open ${this._consecutiveOpenFailures} consecutive times - ` +
'stopping periodic chat scan to avoid log noise',
);
if (this._scanIntervalId) {
clearInterval(this._scanIntervalId);
this._scanIntervalId = null;
}
return;
}
this._logger.info('Chat panel closed, reopening');
const opened = await this._openChatPanel();
if (opened) {
this._consecutiveOpenFailures = 0;
await this._ensureComposeExpanded();
// In auth mode, never try to reopen the chat panel — the toggle
// navigates to the Chat section and minimizes the meeting to PiP.
if (this._isAuthMode) {
this._logger.info('Chat panel closed (auth mode) — skipping reopen to avoid PiP');
} else {
this._consecutiveOpenFailures++;
this._logger.info(
`Chat reopen failed (${this._consecutiveOpenFailures}/${ChatProcedure._MAX_OPEN_FAILURES} before giving up)`,
);
return;
if (this._consecutiveOpenFailures >= ChatProcedure._MAX_OPEN_FAILURES) {
this._logger.warn(
`Chat panel failed to open ${this._consecutiveOpenFailures} consecutive times - ` +
'stopping periodic chat scan to avoid log noise',
);
if (this._scanIntervalId) {
clearInterval(this._scanIntervalId);
this._scanIntervalId = null;
}
return;
}
this._logger.info('Chat panel closed, reopening');
const opened = await this._openChatPanel();
if (opened) {
this._consecutiveOpenFailures = 0;
await this._ensureComposeExpanded();
} else {
this._consecutiveOpenFailures++;
this._logger.info(
`Chat reopen failed (${this._consecutiveOpenFailures}/${ChatProcedure._MAX_OPEN_FAILURES} before giving up)`,
);
return;
}
await this._page.waitForTimeout(1000);
}
await this._page.waitForTimeout(1000);
} else {
this._consecutiveOpenFailures = 0;
}
@ -958,8 +1028,6 @@ export class ChatProcedure {
// overlay used in anonymous / pre-join layouts has a generic
// [data-tid="ckeditor"] [role="textbox"] in a floating layer that
// looks like a chat input but does NOT post into the meeting chat).
// Surfacing the failure fast lets the periodic scan re-toggle the
// panel and the Gateway resend the message.
const panelOpen = await this._isChatPanelOpen();
if (!panelOpen) {
this._logger.warn('Chat panel not open — aborting send so the periodic scan can re-toggle it');
@ -1168,8 +1236,13 @@ export class ChatProcedure {
}
await this._page.waitForTimeout(200);
await this._page.keyboard.press('Enter');
this._logger.info(`Chat message sent (stage=${stageUsed})`);
// Try clicking the send button first; fall back to Enter key.
const sent = await this._clickSendButton();
if (!sent) {
await this._page.keyboard.press('Enter');
}
this._logger.info(`Chat message sent (stage=${stageUsed}, via=${sent ? 'sendBtn' : 'enter'})`);
return true;
}
@ -1192,6 +1265,42 @@ export class ChatProcedure {
return false;
}
/**
* Click the chat send button if visible. Teams renders a send button
* (arrow icon) once text is present in the compose box. Clicking it is
* more reliable than pressing Enter, which CKEditor sometimes swallows.
*/
private async _clickSendButton(): Promise<boolean> {
try {
const sendSelectors = [
'button[data-tid="newMessageCommands-send"]',
'button[data-tid="send-message-button"]',
'button[aria-label="Send" i]',
'button[aria-label="Senden" i]',
'button[aria-label*="Send message" i]',
'button[aria-label*="Nachricht senden" i]',
'[data-tid="chat-pane-compose-message-footer"] button[type="submit"]',
'[data-tid="message-pane-footer"] button[type="submit"]',
];
for (const sel of sendSelectors) {
const btn = await this._page.$(sel);
if (btn) {
const visible = await btn.isVisible().catch(() => false);
if (visible) {
await btn.click();
this._logger.info(`Clicked send button: ${sel}`);
await btn.dispose();
return true;
}
await btn.dispose();
}
}
} catch (err) {
this._logger.warn(`Send button click failed: ${err}`);
}
return false;
}
/**
* Dump rich diagnostics about contenteditable / textbox candidates in the
* page so we can adapt selectors when Teams ships a UI change.

30
src/bot/orchestrator.ts
View file

@ -15,7 +15,7 @@ import { AudioCaptureProcedure } from './audioCaptureProcedure';
import { ChatProcedure, ChatMessageEntry } from './chatProcedure';
import { AuthProcedure, MfaChallenge } from './authProcedure';
import { TeamsActionsService } from './teamsActionsService';
import { BackgroundProcedure } from './backgroundProcedure';
import { isValidMeetingUrl, getMeetingLaunchUrl, resolveLaunchUrl } from './meetingUrlParser';
// Optional: canvas "avatar" video (config.botUseCanvasVideo) replaces the Chromium
@ -85,6 +85,8 @@ export class BotOrchestrator {
private _frameNavMediaRebindTimer: ReturnType<typeof setTimeout> | null = null;
/** Re-apply gUM + video senders for a few seconds after join */
private _canvasRebindTimer: ReturnType<typeof setInterval> | null = null;
/** Whether the bot is running in authenticated mode (full Teams web app) */
private _isAuthMode: boolean = false;
constructor(
sessionId: string,
@ -218,8 +220,6 @@ export class BotOrchestrator {
await this._ensureMicOn();
if (config.botUseCanvasVideo) {
await this._ensureCameraOn();
const bg = new BackgroundProcedure(this._page!, this._logger);
void bg.trySelectNoVirtualBackground();
}
// STEP 2: Enter bot name and click "Join now"
@ -435,8 +435,6 @@ export class BotOrchestrator {
await this._ensureMicOn();
if (config.botUseCanvasVideo) {
await this._ensureCameraOn();
const bg = new BackgroundProcedure(this._page!, this._logger);
void bg.trySelectNoVirtualBackground();
}
// STEP 5: Poll for "Join now" on the pre-join screen
@ -466,10 +464,22 @@ export class BotOrchestrator {
await this._ensureCameraOnInMeeting();
this._startCanvasRebindAfterJoin();
}
await Promise.all([
this._enableTranscriptCapture(),
this._enableChat(),
]);
// Auth mode: wait for meeting UI to stabilize, then captions first,
// then chat. The chat toggle in the full Teams web app navigates to
// the Chat section and minimizes the meeting to PiP if triggered too
// early or while the More menu is still open from captions setup.
if (this._isAuthMode) {
await this._page!.waitForTimeout(3000);
await this._enableTranscriptCapture();
await this._page!.waitForTimeout(2000);
await this._enableChat();
} else {
await Promise.all([
this._enableTranscriptCapture(),
this._enableChat(),
]);
}
await this._sendJoinGreeting();
}
@ -1048,6 +1058,7 @@ export class BotOrchestrator {
* (`rejectMediaDescriptionsUpdateAsync`). Keep these flag sets separate.
*/
private async _launchBrowser(authMode: boolean = false): Promise<void> {
this._isAuthMode = authMode;
this._logger.info(`Launching browser (authMode=${authMode})...`);
// When BOT_ANON_USE_AUTH_BROWSER_SETUP is on, the anon path uses the
@ -1188,6 +1199,7 @@ export class BotOrchestrator {
isFinal: true,
});
},
this._isAuthMode,
);
// DEBUG TOGGLE: skip both wrappers when isolating the Teams anonymous