From 043349f5298825e02cda9192e82350a217a75ef1 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 13 Feb 2026 22:44:57 +0100 Subject: [PATCH] Initial commit: Browser-based Teams Meeting Bot Co-authored-by: Cursor --- .env.sample | 18 ++ .gitignore | 31 ++++ Dockerfile | 44 +++++ README.md | 173 +++++++++++++++++++ docker-compose.yml | 22 +++ package.json | 35 ++++ src/bot/audioProcedure.ts | 137 +++++++++++++++ src/bot/captionsProcedure.ts | 242 +++++++++++++++++++++++++++ src/bot/joinProcedure.ts | 291 ++++++++++++++++++++++++++++++++ src/bot/meetingUrlParser.ts | 83 ++++++++++ src/bot/orchestrator.ts | 311 +++++++++++++++++++++++++++++++++++ src/config.ts | 36 ++++ src/index.ts | 74 +++++++++ src/server/gatewayClient.ts | 238 +++++++++++++++++++++++++++ src/server/httpServer.ts | 140 ++++++++++++++++ src/sessionManager.ts | 210 +++++++++++++++++++++++ src/types/index.ts | 82 +++++++++ src/utils/logger.ts | 43 +++++ tsconfig.json | 19 +++ 19 files changed, 2229 insertions(+) create mode 100644 .env.sample create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 package.json create mode 100644 src/bot/audioProcedure.ts create mode 100644 src/bot/captionsProcedure.ts create mode 100644 src/bot/joinProcedure.ts create mode 100644 src/bot/meetingUrlParser.ts create mode 100644 src/bot/orchestrator.ts create mode 100644 src/config.ts create mode 100644 src/index.ts create mode 100644 src/server/gatewayClient.ts create mode 100644 src/server/httpServer.ts create mode 100644 src/sessionManager.ts create mode 100644 src/types/index.ts create mode 100644 src/utils/logger.ts create mode 100644 tsconfig.json diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..4de52ee --- /dev/null +++ b/.env.sample @@ -0,0 +1,18 @@ +# Service Configuration +PORT=4100 +NODE_ENV=development + +# Gateway WebSocket Connection +GATEWAY_WS_URL=wss://gateway-int.poweron-center.net/api/teamsbot/ws + +# Bot Configuration +BOT_NAME=PowerOn AI +BOT_HEADLESS=true + +# Logging +LOG_LEVEL=info +LOG_DIR=./output/logs + +# Screenshots (for debugging) +SCREENSHOT_DIR=./output/screenshots +SCREENSHOT_ON_ERROR=true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1b0326 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Dependencies +node_modules/ + +# Build output +dist/ + +# Environment +.env +.env.local + +# Output directories +output/ +logs/ +screenshots/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Debug +*.log +npm-debug.log* + +# Test +coverage/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..980dc9c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# Build stage +FROM node:20-slim AS builder + +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci + +# Copy source +COPY tsconfig.json ./ +COPY src ./src + +# Build +RUN npm run build + +# Production stage +FROM mcr.microsoft.com/playwright:v1.41.0-jammy + +WORKDIR /app + +# Copy built files and dependencies +COPY --from=builder /app/dist ./dist +COPY --from=builder /app/node_modules ./node_modules +COPY package*.json ./ + +# Create output directories +RUN mkdir -p output/logs output/screenshots + +# Set environment +ENV NODE_ENV=production +ENV BOT_HEADLESS=true + +# Expose port +EXPOSE 4100 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:4100/health || exit 1 + +# Run +CMD ["node", "dist/index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5a62ac --- /dev/null +++ b/README.md @@ -0,0 +1,173 @@ +# Teams Browser Bot Service + +Browser-based Microsoft Teams Meeting Bot using Playwright. This service joins Teams meetings via the web interface, captures live captions, and plays TTS audio responses. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Browser-based Architecture │ +│ │ +│ ┌───────────────────┐ ┌───────────────────────────────┐ │ +│ │ Gateway │ │ Browser Bot Service │ │ +│ │ (Python) │ │ (Node.js + Playwright) │ │ +│ │ │ │ │ │ +│ │ - STT (Google) │ WebSocket │ - Headless Chrome │ │ +│ │ - AI (OpenAI) │◄──────────────────►│ - Teams Web App │ │ +│ │ - TTS (Google) │ Transcripts │ - Meeting join flow │ │ +│ │ - Session Mgmt │ + TTS Audio │ - Captions scraping │ │ +│ │ │ │ - Audio playback │ │ +│ └───────────────────┘ └───────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +## Features + +- **Multi-tenant support**: Can join any Teams meeting (not limited to own tenant) +- **Browser-based**: Uses Teams web app, no Graph Communications SDK needed +- **Captions scraping**: Captures live captions for transcription +- **Audio playback**: Plays TTS audio through the browser into the meeting +- **WebSocket integration**: Real-time communication with Gateway + +## Prerequisites + +- Node.js 18+ +- Docker (for production deployment) + +## Quick Start + +### Local Development + +```bash +# Install dependencies +npm install + +# Install Playwright browsers +npx playwright install chromium + +# Copy and configure environment +cp .env.sample .env +# Edit .env with your settings + +# Run in development mode +npm run dev +``` + +### Docker + +```bash +# Build and run +docker-compose up --build + +# Or build image only +docker build -t teams-browser-bot . +``` + +## Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `PORT` | HTTP server port | `4100` | +| `GATEWAY_WS_URL` | Gateway WebSocket URL | `wss://gateway-int.poweron-center.net/api/teamsbot/ws` | +| `BOT_NAME` | Display name in meetings | `PowerOn AI` | +| `BOT_HEADLESS` | Run browser headless | `true` | +| `LOG_LEVEL` | Logging level | `info` | +| `SCREENSHOT_ON_ERROR` | Take screenshots on errors | `true` | + +## API Endpoints + +### Health Check +``` +GET /health +``` + +### Deploy Bot +``` +POST /api/bot +Content-Type: application/json + +{ + "sessionId": "uuid", + "meetingUrl": "https://teams.microsoft.com/meet/...", + "botName": "PowerOn AI" +} +``` + +### Leave Meeting +``` +POST /api/bot/:sessionId/leave +``` + +### Get Status +``` +GET /api/bot/:sessionId/status +``` + +## WebSocket Protocol + +### Gateway → Bot + +```typescript +// Join a meeting +{ type: "joinMeeting", sessionId: "uuid", meetingUrl: "...", botName?: "..." } + +// Leave meeting +{ type: "leaveMeeting", sessionId: "uuid" } + +// Play audio +{ type: "playAudio", sessionId: "uuid", audio: { format: "mp3", data: "base64..." } } +``` + +### Bot → Gateway + +```typescript +// Transcript +{ type: "transcript", sessionId: "uuid", transcript: { speaker: "...", text: "...", timestamp: "...", isFinal: true } } + +// Status +{ type: "status", sessionId: "uuid", status: "joined" | "in_lobby" | "left" | "error", message?: "..." } +``` + +## Meeting URL Formats + +Supports both classic and new (short) URL formats: + +``` +# Classic format +https://teams.microsoft.com/l/meetup-join/19%3ameeting_xxx/0?context=... + +# New format (since 2025) +https://teams.microsoft.com/meet/36438888781520?p=5fGqrujxzewPFjJacW +``` + +## Deployment + +### Azure Container Instance + +```bash +# Create resource group +az group create --name rg-teams-bot --location westeurope + +# Create container instance +az container create \ + --resource-group rg-teams-bot \ + --name teams-browser-bot \ + --image /teams-browser-bot:latest \ + --cpu 2 \ + --memory 4 \ + --ports 4100 \ + --environment-variables \ + GATEWAY_WS_URL=wss://gateway-int.poweron-center.net/api/teamsbot/ws \ + BOT_NAME="PowerOn AI" +``` + +## Debugging + +- Logs are written to `output/logs/` +- Screenshots (on error) are saved to `output/screenshots/` +- Set `BOT_HEADLESS=false` for local debugging with visible browser + +## Based On + +- [Recall.ai Microsoft Teams Meeting Bot](https://github.com/recallai/microsoft-teams-meeting-bot) +- [Recall.ai Blog: How to build a Microsoft Teams Bot](https://www.recall.ai/blog/how-to-build-a-microsoft-teams-bot) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d6dbc37 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.8' + +services: + teams-browser-bot: + build: . + ports: + - "4100:4100" + environment: + - NODE_ENV=production + - PORT=4100 + - GATEWAY_WS_URL=${GATEWAY_WS_URL:-wss://gateway-int.poweron-center.net/api/teamsbot/ws} + - BOT_NAME=${BOT_NAME:-PowerOn AI} + - BOT_HEADLESS=true + - LOG_LEVEL=info + - SCREENSHOT_ON_ERROR=true + volumes: + - ./output:/app/output + restart: unless-stopped + # Required for Playwright/Chrome + shm_size: '2gb' + security_opt: + - seccomp:unconfined diff --git a/package.json b/package.json new file mode 100644 index 0000000..e778a76 --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "service-teams-browser-bot", + "version": "1.0.0", + "description": "Browser-based Teams Meeting Bot using Playwright", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts", + "lint": "eslint src/**/*.ts", + "test": "jest" + }, + "dependencies": { + "playwright": "^1.41.0", + "ws": "^8.16.0", + "uuid": "^9.0.1", + "dotenv": "^16.4.1", + "express": "^4.18.2", + "winston": "^3.11.0" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/node": "^20.11.0", + "@types/uuid": "^9.0.7", + "@types/ws": "^8.5.10", + "typescript": "^5.3.3", + "ts-node": "^10.9.2", + "@typescript-eslint/eslint-plugin": "^6.19.0", + "@typescript-eslint/parser": "^6.19.0", + "eslint": "^8.56.0" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/src/bot/audioProcedure.ts b/src/bot/audioProcedure.ts new file mode 100644 index 0000000..97b2da1 --- /dev/null +++ b/src/bot/audioProcedure.ts @@ -0,0 +1,137 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; + +/** + * Handles audio playback in the Teams meeting. + * Injects TTS audio into the browser to be played through the meeting. + */ +export class AudioProcedure { + private _page: Page; + private _logger: Logger; + private _audioContext: boolean = false; + + constructor(page: Page, logger: Logger) { + this._page = page; + this._logger = logger; + } + + /** + * Initialize the audio context in the browser. + * Must be called after user interaction (joining meeting counts). + */ + async initialize(): Promise { + if (this._audioContext) { + return; + } + + this._logger.info('Initializing audio context...'); + + await this._page.evaluate(() => { + // Create a global audio context + const AudioContext = window.AudioContext || (window as any).webkitAudioContext; + (window as any).__audioContext = new AudioContext(); + (window as any).__audioQueue = []; + (window as any).__isPlaying = false; + }); + + this._audioContext = true; + this._logger.info('Audio context initialized'); + } + + /** + * Play audio in the browser. + * The audio will be heard by other meeting participants. + * + * @param audioData Base64 encoded audio data + * @param format Audio format (mp3, wav, pcm) + */ + async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise { + if (!this._audioContext) { + await this.initialize(); + } + + this._logger.info(`Playing audio (format: ${format}, size: ${audioData.length} bytes base64)`); + + try { + await this._page.evaluate(async ({ audioData, format }) => { + const ctx = (window as any).__audioContext as AudioContext; + + // Resume context if suspended + if (ctx.state === 'suspended') { + await ctx.resume(); + } + + // Decode base64 to ArrayBuffer + const binaryString = atob(audioData); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + let audioBuffer: AudioBuffer; + + if (format === 'pcm') { + // PCM: Assume 16-bit mono 16kHz + const pcmData = new Int16Array(bytes.buffer); + audioBuffer = ctx.createBuffer(1, pcmData.length, 16000); + const channelData = audioBuffer.getChannelData(0); + for (let i = 0; i < pcmData.length; i++) { + channelData[i] = pcmData[i] / 32768; // Convert to float + } + } else { + // MP3/WAV: Use decodeAudioData + audioBuffer = await ctx.decodeAudioData(bytes.buffer); + } + + // Create source and play + const source = ctx.createBufferSource(); + source.buffer = audioBuffer; + source.connect(ctx.destination); + source.start(0); + + // Return a promise that resolves when playback ends + return new Promise((resolve) => { + source.onended = () => resolve(); + }); + }, { audioData, format }); + + this._logger.info('Audio playback completed'); + } catch (error) { + this._logger.error('Error playing audio:', error); + throw error; + } + } + + /** + * Stop any currently playing audio. + */ + async stopAudio(): Promise { + try { + await this._page.evaluate(() => { + const ctx = (window as any).__audioContext as AudioContext; + if (ctx) { + ctx.suspend(); + } + }); + } catch { + // Ignore errors + } + } + + /** + * Clean up audio resources. + */ + async cleanup(): Promise { + try { + await this._page.evaluate(() => { + const ctx = (window as any).__audioContext as AudioContext; + if (ctx) { + ctx.close(); + } + }); + } catch { + // Page might be closed + } + this._audioContext = false; + } +} diff --git a/src/bot/captionsProcedure.ts b/src/bot/captionsProcedure.ts new file mode 100644 index 0000000..a6f4107 --- /dev/null +++ b/src/bot/captionsProcedure.ts @@ -0,0 +1,242 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; +import { TranscriptEntry } from '../types'; +import { config } from '../config'; + +/** + * Handles enabling and scraping captions from Teams meetings. + * Based on Recall.ai's open-source implementation. + */ +export class CaptionsProcedure { + private _page: Page; + private _logger: Logger; + private _onTranscript: (entry: TranscriptEntry) => void; + private _isSubscribed: boolean = false; + private _lastCaptionText: string = ''; + + constructor( + page: Page, + logger: Logger, + onTranscript: (entry: TranscriptEntry) => void + ) { + this._page = page; + this._logger = logger; + this._onTranscript = onTranscript; + } + + /** + * Enable live captions in the meeting. + * Opens the "More" menu and clicks "Turn on live captions". + */ + async enableCaptionsFlow(): Promise { + this._logger.info('Enabling live captions...'); + + // First, open the "More actions" menu + await this._openMoreMenu(); + + // Then click on "Turn on live captions" + await this._clickEnableCaptions(); + + this._logger.info('Live captions enabled'); + } + + /** + * Open the "More actions" (...) menu in the call controls. + */ + private async _openMoreMenu(): Promise { + const moreMenuSelectors = [ + '[data-tid="callingButtons-showMoreBtn"]', + 'button[aria-label*="More actions"]', + 'button[aria-label*="More"]', + '[data-tid="more-button"]', + ]; + + for (const selector of moreMenuSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + await this._page.waitForTimeout(1000); + this._logger.info('Opened more menu'); + return; + } + } catch (error) { + // Continue + } + } + + throw new Error('Could not find More actions menu'); + } + + /** + * Click the "Turn on live captions" option. + */ + private async _clickEnableCaptions(): Promise { + const captionsSelectors = [ + 'button:has-text("Turn on live captions")', + 'button:has-text("Live captions")', + '[data-tid="captions-toggle"]', + 'button[aria-label*="captions"]', + 'button[aria-label*="Captions"]', + // Menu item selectors + '[role="menuitem"]:has-text("captions")', + '[role="menuitemcheckbox"]:has-text("captions")', + ]; + + for (const selector of captionsSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + await this._page.waitForTimeout(1000); + this._logger.info('Clicked enable captions'); + return; + } + } catch (error) { + // Continue + } + } + + // Try clicking away to close menu if captions not found + await this._page.keyboard.press('Escape'); + this._logger.warn('Could not find captions option - may already be enabled or not available'); + } + + /** + * Start watching the captions DOM for updates. + * Emits transcript events when new captions appear. + */ + async subscribeToCaptions(): Promise { + if (this._isSubscribed) { + this._logger.warn('Already subscribed to captions'); + return; + } + + this._isSubscribed = true; + this._logger.info('Subscribing to captions...'); + + // Set up a MutationObserver in the browser to watch for caption changes + await this._page.evaluate(() => { + // Store captions data on window for retrieval + (window as any).__captionsBuffer = []; + (window as any).__lastCaptionId = 0; + + // Function to extract caption text + const extractCaptions = () => { + // Common caption container selectors in Teams + const captionSelectors = [ + '[data-tid="closed-captions-renderer"]', + '[data-tid="captions-container"]', + '.captions-container', + '[class*="caption"]', + ]; + + for (const selector of captionSelectors) { + const container = document.querySelector(selector); + if (container) { + // Find individual caption entries + const entries = container.querySelectorAll('[data-tid="caption-entry"], [class*="caption-line"], [class*="captionLine"]'); + + entries.forEach((entry, index) => { + const speakerEl = entry.querySelector('[data-tid="caption-speaker"], [class*="speaker"]'); + const textEl = entry.querySelector('[data-tid="caption-text"], [class*="text"]'); + + const speaker = speakerEl?.textContent?.trim() || 'Unknown'; + const text = textEl?.textContent?.trim() || entry.textContent?.trim() || ''; + + if (text && text.length > 0) { + const captionId = `${speaker}-${text}-${index}`; + if (captionId !== (window as any).__lastCaptionId) { + (window as any).__lastCaptionId = captionId; + (window as any).__captionsBuffer.push({ + speaker, + text, + timestamp: new Date().toISOString(), + isFinal: true, // Teams captions are typically final + }); + } + } + }); + break; + } + } + }; + + // Set up observer + const observer = new MutationObserver(() => { + extractCaptions(); + }); + + // Observe the entire body for changes (captions can appear anywhere) + observer.observe(document.body, { + childList: true, + subtree: true, + characterData: true, + }); + + // Initial extraction + extractCaptions(); + + // Store observer reference for cleanup + (window as any).__captionsObserver = observer; + }); + + // Start polling for new captions + this._pollCaptions(); + } + + /** + * Poll the browser for new captions and emit them. + */ + private async _pollCaptions(): Promise { + while (this._isSubscribed) { + try { + const captions = await this._page.evaluate(() => { + const buffer = (window as any).__captionsBuffer || []; + (window as any).__captionsBuffer = []; + return buffer; + }); + + for (const caption of captions) { + // Deduplicate based on text + if (caption.text !== this._lastCaptionText) { + this._lastCaptionText = caption.text; + this._onTranscript({ + speaker: caption.speaker, + text: caption.text, + timestamp: new Date(caption.timestamp), + isFinal: caption.isFinal, + }); + } + } + } catch (error) { + // Page might be closed + if (this._isSubscribed) { + this._logger.error('Error polling captions:', error); + } + } + + // Poll every 500ms + await new Promise(resolve => setTimeout(resolve, 500)); + } + } + + /** + * Stop watching for captions. + */ + async unsubscribe(): Promise { + this._isSubscribed = false; + + try { + await this._page.evaluate(() => { + if ((window as any).__captionsObserver) { + (window as any).__captionsObserver.disconnect(); + } + }); + } catch { + // Page might already be closed + } + + this._logger.info('Unsubscribed from captions'); + } +} diff --git a/src/bot/joinProcedure.ts b/src/bot/joinProcedure.ts new file mode 100644 index 0000000..dbdc22c --- /dev/null +++ b/src/bot/joinProcedure.ts @@ -0,0 +1,291 @@ +import { Page } from 'playwright'; +import { Logger } from 'winston'; +import { config } from '../config'; +import { getMeetingLaunchUrl } from './meetingUrlParser'; + +/** + * Handles the Teams meeting join flow. + * Based on Recall.ai's open-source implementation. + */ +export class JoinProcedure { + private _page: Page; + private _logger: Logger; + private _botName: string; + + constructor(page: Page, logger: Logger, botName: string) { + this._page = page; + this._logger = logger; + this._botName = botName; + } + + /** + * Navigate to the meeting URL and handle the launcher dialog. + * Teams shows a "How do you want to join?" dialog first. + */ + async startMeetingLauncherFlow(meetingUrl: string): Promise { + const launchUrl = getMeetingLaunchUrl(meetingUrl); + this._logger.info(`Navigating to meeting: ${launchUrl}`); + + await this._page.goto(launchUrl, { + waitUntil: 'domcontentloaded', + timeout: config.timeouts.pageLoad, + }); + + // Wait for the page to stabilize + await this._page.waitForTimeout(2000); + + // Handle "Continue on this browser" or similar prompts + await this._handleLauncherDialog(); + } + + /** + * Handle the launcher dialog that asks how to join. + * We want to select "Continue on this browser" / "Join on the web instead" + */ + private async _handleLauncherDialog(): Promise { + this._logger.info('Looking for launcher dialog...'); + + // Common selectors for "Continue on browser" button + const browserJoinSelectors = [ + 'button:has-text("Continue on this browser")', + 'button:has-text("Join on the web instead")', + 'a:has-text("Continue on this browser")', + 'a:has-text("Join on the web")', + '[data-tid="joinOnWeb"]', + '[data-tid="prejoin-join-button"]', + ]; + + for (const selector of browserJoinSelectors) { + try { + const element = await this._page.$(selector); + if (element) { + this._logger.info(`Found launcher button: ${selector}`); + await element.click(); + await this._page.waitForTimeout(2000); + return; + } + } catch (error) { + // Continue to next selector + } + } + + this._logger.info('No launcher dialog found, may already be on join page'); + } + + /** + * Fill in the bot name and click "Join now" to enter the lobby. + */ + async joinMeetingLobbyFlow(): Promise { + this._logger.info('Starting lobby join flow...'); + + // Wait for the pre-join screen + await this._page.waitForTimeout(2000); + + // Handle microphone/camera permissions - we want them OFF + await this._disableMediaDevices(); + + // Enter the bot name + await this._enterBotName(); + + // Click "Join now" + await this._clickJoinNow(); + } + + /** + * Disable microphone and camera toggles. + */ + private async _disableMediaDevices(): Promise { + this._logger.info('Disabling media devices...'); + + // Microphone toggle selectors + const micSelectors = [ + '[data-tid="toggle-mute"]', + '[aria-label*="microphone"]', + '[aria-label*="Microphone"]', + 'button[id*="microphone"]', + ]; + + // Camera toggle selectors + const cameraSelectors = [ + '[data-tid="toggle-video"]', + '[aria-label*="camera"]', + '[aria-label*="Camera"]', + 'button[id*="camera"]', + ]; + + // Try to turn off microphone (click if it's ON) + for (const selector of micSelectors) { + try { + const mic = await this._page.$(selector); + if (mic) { + const ariaPressed = await mic.getAttribute('aria-pressed'); + const ariaChecked = await mic.getAttribute('aria-checked'); + if (ariaPressed === 'true' || ariaChecked === 'true') { + await mic.click(); + this._logger.info('Disabled microphone'); + } + break; + } + } catch (error) { + // Continue + } + } + + // Try to turn off camera + for (const selector of cameraSelectors) { + try { + const camera = await this._page.$(selector); + if (camera) { + const ariaPressed = await camera.getAttribute('aria-pressed'); + const ariaChecked = await camera.getAttribute('aria-checked'); + if (ariaPressed === 'true' || ariaChecked === 'true') { + await camera.click(); + this._logger.info('Disabled camera'); + } + break; + } + } catch (error) { + // Continue + } + } + } + + /** + * Enter the bot name in the name input field. + */ + private async _enterBotName(): Promise { + this._logger.info(`Entering bot name: ${this._botName}`); + + const nameSelectors = [ + 'input[data-tid="prejoin-display-name-input"]', + 'input[placeholder*="name"]', + 'input[placeholder*="Name"]', + 'input[aria-label*="name"]', + '#username', + ]; + + for (const selector of nameSelectors) { + try { + const input = await this._page.$(selector); + if (input) { + await input.fill(this._botName); + this._logger.info('Bot name entered'); + return; + } + } catch (error) { + // Continue + } + } + + this._logger.warn('Could not find name input field'); + } + + /** + * Click the "Join now" button. + */ + private async _clickJoinNow(): Promise { + this._logger.info('Clicking Join now...'); + + const joinSelectors = [ + 'button[data-tid="prejoin-join-button"]', + 'button:has-text("Join now")', + 'button:has-text("Join meeting")', + '[data-tid="joinButton"]', + ]; + + for (const selector of joinSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info('Clicked join button'); + return; + } + } catch (error) { + // Continue + } + } + + throw new Error('Could not find Join button'); + } + + /** + * Check if the bot is currently in the lobby (waiting to be admitted). + */ + async isInMeetingLobby(options: { waitForSeconds?: number } = {}): Promise { + const timeout = (options.waitForSeconds || 5) * 1000; + + const lobbySelectors = [ + '[data-tid="lobby-screen"]', + ':has-text("waiting for someone to let you in")', + ':has-text("Someone in the meeting should let you in soon")', + '[data-tid="waiting-screen"]', + ]; + + try { + await this._page.waitForSelector(lobbySelectors.join(', '), { + timeout, + state: 'visible', + }); + return true; + } catch { + return false; + } + } + + /** + * Check if the bot is currently in the meeting (admitted from lobby). + */ + async isInMeeting(options: { waitForSeconds?: number } = {}): Promise { + const timeout = (options.waitForSeconds || 5) * 1000; + + // Indicators that we're in the meeting + const inMeetingSelectors = [ + '[data-tid="call-composite"]', + '[data-tid="meeting-roster"]', + '[data-tid="hangup-button"]', + 'button[aria-label*="Leave"]', + '[data-tid="callingButtons-showMoreBtn"]', + ]; + + try { + await this._page.waitForSelector(inMeetingSelectors.join(', '), { + timeout, + state: 'visible', + }); + return true; + } catch { + return false; + } + } + + /** + * Leave the meeting gracefully. + */ + async leaveMeetingFlow(): Promise { + this._logger.info('Leaving meeting...'); + + const leaveSelectors = [ + '[data-tid="hangup-button"]', + 'button[aria-label*="Leave"]', + 'button:has-text("Leave")', + '[data-tid="call-hangup"]', + ]; + + for (const selector of leaveSelectors) { + try { + const button = await this._page.$(selector); + if (button) { + await button.click(); + this._logger.info('Clicked leave button'); + await this._page.waitForTimeout(2000); + return; + } + } catch (error) { + // Continue + } + } + + this._logger.warn('Could not find leave button, closing page'); + } +} diff --git a/src/bot/meetingUrlParser.ts b/src/bot/meetingUrlParser.ts new file mode 100644 index 0000000..3db4d7e --- /dev/null +++ b/src/bot/meetingUrlParser.ts @@ -0,0 +1,83 @@ +import { ParsedMeetingUrl } from '../types'; + +/** + * Parses Teams meeting URLs into a standardized format. + * Supports both classic and new (short) URL formats. + * + * Classic format: + * https://teams.microsoft.com/l/meetup-join/19%3ameeting_xxx/0?context=... + * + * New format (since 2025): + * https://teams.microsoft.com/meet/36438888781520?p=5fGqrujxzewPFjJacW + */ +export function parseMeetingUrl(url: string): ParsedMeetingUrl { + const trimmedUrl = url.trim(); + + // Check for new short format: /meet/{meetingId}?p={passcode} + const shortMatch = trimmedUrl.match(/teams\.microsoft\.com\/meet\/([^?]+)(?:\?p=([^&]+))?/); + if (shortMatch) { + return { + type: 'short', + originalUrl: trimmedUrl, + meetingId: shortMatch[1], + passcode: shortMatch[2], + }; + } + + // Check for classic format: /l/meetup-join/{encoded-meeting-info} + const classicMatch = trimmedUrl.match(/teams\.microsoft\.com\/l\/meetup-join\/([^/]+)/); + if (classicMatch) { + return { + type: 'classic', + originalUrl: trimmedUrl, + meetingId: classicMatch[1], + }; + } + + // Unknown format - return as-is + return { + type: 'classic', + originalUrl: trimmedUrl, + }; +} + +/** + * Validates if a URL is a valid Teams meeting URL. + */ +export function isValidMeetingUrl(url: string): boolean { + if (!url || typeof url !== 'string') { + return false; + } + + const trimmedUrl = url.trim().toLowerCase(); + + // Must be a teams.microsoft.com URL + if (!trimmedUrl.includes('teams.microsoft.com')) { + return false; + } + + // Must be either /meet/ or /l/meetup-join/ + return trimmedUrl.includes('/meet/') || trimmedUrl.includes('/l/meetup-join/'); +} + +/** + * Converts a meeting URL to the web app launch URL. + * Teams web app requires a specific format to join meetings. + */ +export function getMeetingLaunchUrl(url: string): string { + const parsed = parseMeetingUrl(url); + + // For short URLs, we can use them directly + if (parsed.type === 'short') { + return parsed.originalUrl; + } + + // For classic URLs, ensure we're using the web version + // Add ?anon=true to skip sign-in prompt for anonymous join + let launchUrl = parsed.originalUrl; + if (!launchUrl.includes('anon=')) { + launchUrl += (launchUrl.includes('?') ? '&' : '?') + 'anon=true'; + } + + return launchUrl; +} diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts new file mode 100644 index 0000000..242ceb7 --- /dev/null +++ b/src/bot/orchestrator.ts @@ -0,0 +1,311 @@ +import { Browser, BrowserContext, Page, chromium } from 'playwright'; +import { Logger } from 'winston'; +import { v4 as uuidv4 } from 'uuid'; +import path from 'path'; +import fs from 'fs'; + +import { config } from '../config'; +import { createSessionLogger } from '../utils/logger'; +import { BotSession, BotState, TranscriptEntry } from '../types'; +import { JoinProcedure } from './joinProcedure'; +import { CaptionsProcedure } from './captionsProcedure'; +import { AudioProcedure } from './audioProcedure'; +import { isValidMeetingUrl } from './meetingUrlParser'; + +export interface OrchestratorCallbacks { + onStateChange: (state: BotState, message?: string) => void; + onTranscript: (entry: TranscriptEntry) => void; + onError: (error: Error) => void; +} + +/** + * Orchestrates the entire bot lifecycle: + * - Launches browser + * - Joins meeting + * - Enables captions + * - Handles audio playback + * - Leaves meeting + */ +export class BotOrchestrator { + private _sessionId: string; + private _meetingUrl: string; + private _botName: string; + private _logger: Logger; + private _callbacks: OrchestratorCallbacks; + + private _browser: Browser | null = null; + private _context: BrowserContext | null = null; + private _page: Page | null = null; + + private _joinProcedure: JoinProcedure | null = null; + private _captionsProcedure: CaptionsProcedure | null = null; + private _audioProcedure: AudioProcedure | null = null; + + private _state: BotState = 'idle'; + private _isShuttingDown: boolean = false; + + constructor( + sessionId: string, + meetingUrl: string, + botName: string, + callbacks: OrchestratorCallbacks + ) { + this._sessionId = sessionId; + this._meetingUrl = meetingUrl; + this._botName = botName || config.botName; + this._callbacks = callbacks; + this._logger = createSessionLogger(sessionId); + } + + get sessionId(): string { + return this._sessionId; + } + + get state(): BotState { + return this._state; + } + + /** + * Start the bot - launch browser, join meeting, enable captions. + */ + async start(): Promise { + if (!isValidMeetingUrl(this._meetingUrl)) { + throw new Error(`Invalid meeting URL: ${this._meetingUrl}`); + } + + try { + this._setState('launching'); + + // Launch browser + await this._launchBrowser(); + + this._setState('navigating'); + + // Navigate to meeting and handle launcher + await this._joinProcedure!.startMeetingLauncherFlow(this._meetingUrl); + + // Join the meeting (enter lobby) + await this._joinProcedure!.joinMeetingLobbyFlow(); + + // Check if we're in lobby + const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 10 }); + if (inLobby) { + this._setState('in_lobby'); + this._logger.info('Bot is in lobby, waiting to be admitted...'); + } + + // Wait to be admitted to the meeting + await this._waitForMeetingAdmission(); + + this._setState('in_meeting'); + this._logger.info('Bot joined the meeting!'); + + // Initialize audio + await this._audioProcedure!.initialize(); + + // Enable and subscribe to captions + await this._enableCaptions(); + + } catch (error) { + this._logger.error('Error starting bot:', error); + this._setState('error', (error as Error).message); + await this._takeScreenshot('error'); + throw error; + } + } + + /** + * Stop the bot - leave meeting, close browser. + */ + async stop(): Promise { + if (this._isShuttingDown) { + return; + } + + this._isShuttingDown = true; + this._logger.info('Stopping bot...'); + + try { + this._setState('leaving'); + + // Unsubscribe from captions + if (this._captionsProcedure) { + await this._captionsProcedure.unsubscribe(); + } + + // Clean up audio + if (this._audioProcedure) { + await this._audioProcedure.cleanup(); + } + + // Leave the meeting + if (this._joinProcedure && this._state !== 'error') { + await this._joinProcedure.leaveMeetingFlow(); + } + + } catch (error) { + this._logger.error('Error during shutdown:', error); + } finally { + // Close browser + await this._closeBrowser(); + this._setState('disconnected'); + } + } + + /** + * Play audio in the meeting. + */ + async playAudio(audioData: string, format: 'mp3' | 'wav' | 'pcm'): Promise { + if (this._state !== 'in_meeting' || !this._audioProcedure) { + this._logger.warn('Cannot play audio - not in meeting'); + return; + } + + await this._audioProcedure.playAudio(audioData, format); + } + + /** + * Launch the browser and create a new page. + */ + private async _launchBrowser(): Promise { + this._logger.info('Launching browser...'); + + this._browser = await chromium.launch({ + headless: config.botHeadless, + args: [ + '--use-fake-ui-for-media-stream', // Auto-accept media permissions + '--use-fake-device-for-media-stream', // Use fake devices + '--disable-web-security', + '--disable-features=IsolateOrigins,site-per-process', + '--autoplay-policy=no-user-gesture-required', + ], + }); + + this._context = await this._browser.newContext({ + permissions: ['microphone', 'camera'], + viewport: { width: 1280, height: 720 }, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + }); + + this._page = await this._context.newPage(); + + // Initialize procedures + this._joinProcedure = new JoinProcedure(this._page, this._logger, this._botName); + this._captionsProcedure = new CaptionsProcedure(this._page, this._logger, (entry) => { + this._callbacks.onTranscript(entry); + }); + this._audioProcedure = new AudioProcedure(this._page, this._logger); + + // Handle page errors + this._page.on('pageerror', (error) => { + this._logger.error('Page error:', error); + }); + + // Handle page close + this._page.on('close', () => { + if (!this._isShuttingDown) { + this._logger.warn('Page closed unexpectedly'); + this._setState('disconnected'); + } + }); + + this._logger.info('Browser launched'); + } + + /** + * Close the browser. + */ + private async _closeBrowser(): Promise { + try { + if (this._page) { + await this._page.close(); + } + if (this._context) { + await this._context.close(); + } + if (this._browser) { + await this._browser.close(); + } + } catch (error) { + this._logger.error('Error closing browser:', error); + } + + this._page = null; + this._context = null; + this._browser = null; + this._logger.info('Browser closed'); + } + + /** + * Wait for the bot to be admitted from the lobby. + */ + private async _waitForMeetingAdmission(): Promise { + const startTime = Date.now(); + const timeout = config.timeouts.lobbyWait; + + while (Date.now() - startTime < timeout) { + // Check if we're in the meeting + const inMeeting = await this._joinProcedure!.isInMeeting({ waitForSeconds: 5 }); + if (inMeeting) { + return; + } + + // Check if still in lobby + const inLobby = await this._joinProcedure!.isInMeetingLobby({ waitForSeconds: 2 }); + if (!inLobby) { + // Might have been rejected or meeting ended + throw new Error('Bot was removed from lobby or meeting ended'); + } + + this._logger.info('Still waiting in lobby...'); + } + + throw new Error('Timeout waiting to be admitted from lobby'); + } + + /** + * Enable captions and start scraping. + */ + private async _enableCaptions(): Promise { + try { + await this._captionsProcedure!.enableCaptionsFlow(); + await this._captionsProcedure!.subscribeToCaptions(); + this._logger.info('Captions enabled and subscribed'); + } catch (error) { + this._logger.warn('Could not enable captions:', error); + // Continue without captions - not a fatal error + } + } + + /** + * Update the bot state and notify callbacks. + */ + private _setState(state: BotState, message?: string): void { + this._state = state; + this._logger.info(`State changed: ${state}${message ? ` - ${message}` : ''}`); + this._callbacks.onStateChange(state, message); + } + + /** + * Take a screenshot for debugging. + */ + private async _takeScreenshot(name: string): Promise { + if (!config.screenshotOnError || !this._page) { + return; + } + + try { + const screenshotDir = config.screenshotDir; + if (!fs.existsSync(screenshotDir)) { + fs.mkdirSync(screenshotDir, { recursive: true }); + } + + const filename = `${this._sessionId}-${name}-${Date.now()}.png`; + const filepath = path.join(screenshotDir, filename); + await this._page.screenshot({ path: filepath, fullPage: true }); + this._logger.info(`Screenshot saved: ${filepath}`); + } catch (error) { + this._logger.error('Error taking screenshot:', error); + } + } +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..5978364 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,36 @@ +import dotenv from 'dotenv'; +import path from 'path'; + +dotenv.config(); + +export const config = { + // Service + port: parseInt(process.env.PORT || '4100', 10), + nodeEnv: process.env.NODE_ENV || 'development', + + // Gateway + gatewayWsUrl: process.env.GATEWAY_WS_URL || 'wss://gateway-int.poweron-center.net/api/teamsbot/ws', + + // Bot + botName: process.env.BOT_NAME || 'PowerOn AI', + botHeadless: process.env.BOT_HEADLESS !== 'false', + + // Logging + logLevel: process.env.LOG_LEVEL || 'info', + logDir: process.env.LOG_DIR || './output/logs', + + // Screenshots + screenshotDir: process.env.SCREENSHOT_DIR || './output/screenshots', + screenshotOnError: process.env.SCREENSHOT_ON_ERROR === 'true', + + // Timeouts (in milliseconds) + timeouts: { + lobbyWait: 120000, // 2 minutes waiting in lobby + joinTimeout: 30000, // 30 seconds to join + captionsEnable: 10000, // 10 seconds to enable captions + pageLoad: 30000, // 30 seconds for page load + }, + + // Teams URLs + teamsBaseUrl: 'https://teams.microsoft.com', +}; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..6b5a3fe --- /dev/null +++ b/src/index.ts @@ -0,0 +1,74 @@ +import { SessionManager } from './sessionManager'; +import { HttpServer } from './server/httpServer'; +import { logger } from './utils/logger'; +import { config } from './config'; + +let sessionManager: SessionManager; +let httpServer: HttpServer; + +async function main(): Promise { + logger.info('Starting Teams Browser Bot Service...'); + logger.info(`Environment: ${config.nodeEnv}`); + logger.info(`Port: ${config.port}`); + logger.info(`Gateway URL: ${config.gatewayWsUrl}`); + logger.info(`Headless: ${config.botHeadless}`); + + // Initialize session manager + sessionManager = new SessionManager(); + await sessionManager.initialize(); + + // Start HTTP server + httpServer = new HttpServer({ + onJoinRequest: async (sessionId, meetingUrl, botName) => { + await sessionManager.createSession(sessionId, meetingUrl, botName); + }, + onLeaveRequest: async (sessionId) => { + await sessionManager.endSession(sessionId); + }, + onStatusRequest: (sessionId) => { + return sessionManager.getSessionStatus(sessionId); + }, + }); + + await httpServer.start(); + + logger.info('Teams Browser Bot Service started successfully'); +} + +// Graceful shutdown +async function shutdown(signal: string): Promise { + logger.info(`Received ${signal}, shutting down...`); + + try { + if (httpServer) { + await httpServer.stop(); + } + if (sessionManager) { + await sessionManager.shutdown(); + } + logger.info('Shutdown complete'); + process.exit(0); + } catch (error) { + logger.error('Error during shutdown:', error); + process.exit(1); + } +} + +process.on('SIGTERM', () => shutdown('SIGTERM')); +process.on('SIGINT', () => shutdown('SIGINT')); + +// Handle uncaught errors +process.on('uncaughtException', (error) => { + logger.error('Uncaught exception:', error); + shutdown('uncaughtException'); +}); + +process.on('unhandledRejection', (reason, promise) => { + logger.error('Unhandled rejection at:', promise, 'reason:', reason); +}); + +// Start the service +main().catch((error) => { + logger.error('Failed to start service:', error); + process.exit(1); +}); diff --git a/src/server/gatewayClient.ts b/src/server/gatewayClient.ts new file mode 100644 index 0000000..14c257e --- /dev/null +++ b/src/server/gatewayClient.ts @@ -0,0 +1,238 @@ +import WebSocket from 'ws'; +import { Logger } from 'winston'; +import { + GatewayToBot, + BotToGateway, + TranscriptMessage, + StatusMessage, + BotState +} from '../types'; +import { logger } from '../utils/logger'; + +export interface GatewayClientCallbacks { + onJoinMeeting: (sessionId: string, meetingUrl: string, botName?: string) => void; + onLeaveMeeting: (sessionId: string) => void; + onPlayAudio: (sessionId: string, audioData: string, format: 'mp3' | 'wav' | 'pcm') => void; + onDisconnect: () => void; +} + +/** + * WebSocket client that connects to the Gateway. + * Receives commands (join, leave, play audio) and sends events (transcript, status). + */ +export class GatewayClient { + private _wsUrl: string; + private _ws: WebSocket | null = null; + private _callbacks: GatewayClientCallbacks; + private _logger: Logger; + private _reconnectAttempts: number = 0; + private _maxReconnectAttempts: number = 10; + private _reconnectDelay: number = 1000; + private _isConnecting: boolean = false; + private _shouldReconnect: boolean = true; + + constructor(wsUrl: string, callbacks: GatewayClientCallbacks) { + this._wsUrl = wsUrl; + this._callbacks = callbacks; + this._logger = logger.child({ component: 'GatewayClient' }); + } + + /** + * Connect to the Gateway WebSocket. + */ + async connect(): Promise { + if (this._isConnecting || (this._ws && this._ws.readyState === WebSocket.OPEN)) { + return; + } + + this._isConnecting = true; + this._shouldReconnect = true; + + return new Promise((resolve, reject) => { + this._logger.info(`Connecting to Gateway: ${this._wsUrl}`); + + this._ws = new WebSocket(this._wsUrl); + + this._ws.on('open', () => { + this._logger.info('Connected to Gateway'); + this._isConnecting = false; + this._reconnectAttempts = 0; + resolve(); + }); + + this._ws.on('message', (data) => { + this._handleMessage(data.toString()); + }); + + this._ws.on('close', (code, reason) => { + this._logger.warn(`Gateway connection closed: ${code} - ${reason}`); + this._isConnecting = false; + this._ws = null; + this._callbacks.onDisconnect(); + + if (this._shouldReconnect) { + this._scheduleReconnect(); + } + }); + + this._ws.on('error', (error) => { + this._logger.error('Gateway WebSocket error:', error); + this._isConnecting = false; + + if (this._reconnectAttempts === 0) { + reject(error); + } + }); + }); + } + + /** + * Disconnect from the Gateway. + */ + disconnect(): void { + this._shouldReconnect = false; + + if (this._ws) { + this._ws.close(1000, 'Client disconnecting'); + this._ws = null; + } + } + + /** + * Send a transcript to the Gateway. + */ + sendTranscript( + sessionId: string, + speaker: string, + text: string, + isFinal: boolean = true + ): void { + const message: TranscriptMessage = { + type: 'transcript', + sessionId, + transcript: { + speaker, + text, + timestamp: new Date().toISOString(), + isFinal, + }, + }; + + this._send(message); + } + + /** + * Send a status update to the Gateway. + */ + sendStatus( + sessionId: string, + status: StatusMessage['status'], + message?: string + ): void { + const statusMessage: StatusMessage = { + type: 'status', + sessionId, + status, + message, + }; + + this._send(statusMessage); + } + + /** + * Map BotState to StatusMessage status. + */ + mapStateToStatus(state: BotState): StatusMessage['status'] { + switch (state) { + case 'launching': + case 'navigating': + return 'connecting'; + case 'in_lobby': + return 'in_lobby'; + case 'in_meeting': + return 'joined'; + case 'leaving': + case 'disconnected': + return 'left'; + case 'error': + return 'error'; + default: + return 'connecting'; + } + } + + /** + * Handle incoming messages from the Gateway. + */ + private _handleMessage(data: string): void { + try { + const message = JSON.parse(data) as GatewayToBot; + + this._logger.debug('Received message:', { type: message.type }); + + switch (message.type) { + case 'joinMeeting': + this._callbacks.onJoinMeeting( + message.sessionId, + message.meetingUrl, + message.botName + ); + break; + + case 'leaveMeeting': + this._callbacks.onLeaveMeeting(message.sessionId); + break; + + case 'playAudio': + this._callbacks.onPlayAudio( + message.sessionId, + message.audio.data, + message.audio.format + ); + break; + + default: + this._logger.warn('Unknown message type:', (message as any).type); + } + } catch (error) { + this._logger.error('Error parsing Gateway message:', error); + } + } + + /** + * Send a message to the Gateway. + */ + private _send(message: BotToGateway): void { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) { + this._logger.warn('Cannot send message - not connected'); + return; + } + + try { + this._ws.send(JSON.stringify(message)); + } catch (error) { + this._logger.error('Error sending message:', error); + } + } + + /** + * Schedule a reconnection attempt. + */ + private _scheduleReconnect(): void { + if (this._reconnectAttempts >= this._maxReconnectAttempts) { + this._logger.error('Max reconnection attempts reached'); + return; + } + + this._reconnectAttempts++; + const delay = this._reconnectDelay * Math.pow(2, this._reconnectAttempts - 1); + + this._logger.info(`Reconnecting in ${delay}ms (attempt ${this._reconnectAttempts}/${this._maxReconnectAttempts})`); + + setTimeout(() => { + this.connect().catch((error) => { + this._logger.error('Reconnection failed:', error); + }); + }, delay); + } +} diff --git a/src/server/httpServer.ts b/src/server/httpServer.ts new file mode 100644 index 0000000..21697db --- /dev/null +++ b/src/server/httpServer.ts @@ -0,0 +1,140 @@ +import express, { Express, Request, Response } from 'express'; +import { Server } from 'http'; +import { logger } from '../utils/logger'; +import { config } from '../config'; + +export interface HttpServerCallbacks { + onJoinRequest: (sessionId: string, meetingUrl: string, botName?: string) => Promise; + onLeaveRequest: (sessionId: string) => Promise; + onStatusRequest: (sessionId: string) => { state: string; error?: string } | null; +} + +/** + * HTTP server for the Bot Launcher API. + * Provides endpoints to deploy/control bots. + */ +export class HttpServer { + private _app: Express; + private _server: Server | null = null; + private _callbacks: HttpServerCallbacks; + + constructor(callbacks: HttpServerCallbacks) { + this._callbacks = callbacks; + this._app = express(); + this._setupMiddleware(); + this._setupRoutes(); + } + + /** + * Start the HTTP server. + */ + async start(): Promise { + return new Promise((resolve) => { + this._server = this._app.listen(config.port, () => { + logger.info(`HTTP server listening on port ${config.port}`); + resolve(); + }); + }); + } + + /** + * Stop the HTTP server. + */ + async stop(): Promise { + return new Promise((resolve, reject) => { + if (!this._server) { + resolve(); + return; + } + + this._server.close((err) => { + if (err) { + reject(err); + } else { + logger.info('HTTP server stopped'); + resolve(); + } + }); + }); + } + + private _setupMiddleware(): void { + this._app.use(express.json()); + + // Request logging + this._app.use((req, res, next) => { + logger.debug(`${req.method} ${req.path}`); + next(); + }); + } + + private _setupRoutes(): void { + // Health check + this._app.get('/health', (req: Request, res: Response) => { + res.json({ status: 'ok', timestamp: new Date().toISOString() }); + }); + + // Deploy a new bot + this._app.post('/api/bot', async (req: Request, res: Response) => { + try { + const { sessionId, meetingUrl, botName } = req.body; + + if (!sessionId || !meetingUrl) { + res.status(400).json({ error: 'Missing required fields: sessionId, meetingUrl' }); + return; + } + + await this._callbacks.onJoinRequest(sessionId, meetingUrl, botName); + + res.json({ + success: true, + sessionId, + message: 'Bot deployment started', + }); + } catch (error) { + logger.error('Error deploying bot:', error); + res.status(500).json({ error: (error as Error).message }); + } + }); + + // Leave meeting + this._app.post('/api/bot/:sessionId/leave', async (req: Request, res: Response) => { + try { + const { sessionId } = req.params; + + await this._callbacks.onLeaveRequest(sessionId); + + res.json({ + success: true, + sessionId, + message: 'Bot leave initiated', + }); + } catch (error) { + logger.error('Error leaving meeting:', error); + res.status(500).json({ error: (error as Error).message }); + } + }); + + // Get bot status + this._app.get('/api/bot/:sessionId/status', (req: Request, res: Response) => { + const { sessionId } = req.params; + const status = this._callbacks.onStatusRequest(sessionId); + + if (!status) { + res.status(404).json({ error: 'Session not found' }); + return; + } + + res.json({ + sessionId, + ...status, + }); + }); + + // List all active bots + this._app.get('/api/bots', (req: Request, res: Response) => { + // This would need access to the session manager + res.json({ message: 'Not implemented' }); + }); + } +} diff --git a/src/sessionManager.ts b/src/sessionManager.ts new file mode 100644 index 0000000..0c93924 --- /dev/null +++ b/src/sessionManager.ts @@ -0,0 +1,210 @@ +import { v4 as uuidv4 } from 'uuid'; +import { BotOrchestrator, OrchestratorCallbacks } from './bot/orchestrator'; +import { GatewayClient } from './server/gatewayClient'; +import { BotSession, BotState, TranscriptEntry } from './types'; +import { logger } from './utils/logger'; +import { config } from './config'; + +/** + * Manages all active bot sessions. + * Coordinates between the Gateway client and bot orchestrators. + */ +export class SessionManager { + private _sessions: Map = new Map(); + private _gatewayClient: GatewayClient | null = null; + + constructor() {} + + /** + * Initialize the session manager and connect to the Gateway. + */ + async initialize(): Promise { + logger.info('Initializing SessionManager...'); + + // Create Gateway client + this._gatewayClient = new GatewayClient(config.gatewayWsUrl, { + onJoinMeeting: (sessionId, meetingUrl, botName) => { + this.createSession(sessionId, meetingUrl, botName); + }, + onLeaveMeeting: (sessionId) => { + this.endSession(sessionId); + }, + onPlayAudio: (sessionId, audioData, format) => { + this.playAudio(sessionId, audioData, format); + }, + onDisconnect: () => { + logger.warn('Gateway disconnected'); + }, + }); + + // Connect to Gateway + try { + await this._gatewayClient.connect(); + logger.info('Connected to Gateway'); + } catch (error) { + logger.error('Failed to connect to Gateway:', error); + // Continue without Gateway - can still use HTTP API + } + } + + /** + * Create a new bot session and join the meeting. + */ + async createSession( + sessionId: string, + meetingUrl: string, + botName?: string + ): Promise { + if (this._sessions.has(sessionId)) { + logger.warn(`Session ${sessionId} already exists`); + return; + } + + logger.info(`Creating session ${sessionId} for meeting: ${meetingUrl}`); + + const callbacks: OrchestratorCallbacks = { + onStateChange: (state, message) => { + this._handleStateChange(sessionId, state, message); + }, + onTranscript: (entry) => { + this._handleTranscript(sessionId, entry); + }, + onError: (error) => { + this._handleError(sessionId, error); + }, + }; + + const orchestrator = new BotOrchestrator( + sessionId, + meetingUrl, + botName || config.botName, + callbacks + ); + + this._sessions.set(sessionId, orchestrator); + + // Start the bot asynchronously + orchestrator.start().catch((error) => { + logger.error(`Session ${sessionId} failed to start:`, error); + }); + } + + /** + * End a bot session and leave the meeting. + */ + async endSession(sessionId: string): Promise { + const orchestrator = this._sessions.get(sessionId); + if (!orchestrator) { + logger.warn(`Session ${sessionId} not found`); + return; + } + + logger.info(`Ending session ${sessionId}`); + + try { + await orchestrator.stop(); + } finally { + this._sessions.delete(sessionId); + } + } + + /** + * Play audio in a session's meeting. + */ + async playAudio( + sessionId: string, + audioData: string, + format: 'mp3' | 'wav' | 'pcm' + ): Promise { + const orchestrator = this._sessions.get(sessionId); + if (!orchestrator) { + logger.warn(`Session ${sessionId} not found for audio playback`); + return; + } + + await orchestrator.playAudio(audioData, format); + } + + /** + * Get the status of a session. + */ + getSessionStatus(sessionId: string): { state: string; error?: string } | null { + const orchestrator = this._sessions.get(sessionId); + if (!orchestrator) { + return null; + } + + return { + state: orchestrator.state, + }; + } + + /** + * Get all active session IDs. + */ + getActiveSessions(): string[] { + return Array.from(this._sessions.keys()); + } + + /** + * Shutdown all sessions and disconnect from Gateway. + */ + async shutdown(): Promise { + logger.info('Shutting down SessionManager...'); + + // End all sessions + const sessionIds = Array.from(this._sessions.keys()); + await Promise.all(sessionIds.map((id) => this.endSession(id))); + + // Disconnect from Gateway + if (this._gatewayClient) { + this._gatewayClient.disconnect(); + } + + logger.info('SessionManager shutdown complete'); + } + + /** + * Handle state changes from orchestrators. + */ + private _handleStateChange(sessionId: string, state: BotState, message?: string): void { + logger.info(`Session ${sessionId} state: ${state}${message ? ` - ${message}` : ''}`); + + if (this._gatewayClient) { + const status = this._gatewayClient.mapStateToStatus(state); + this._gatewayClient.sendStatus(sessionId, status, message); + } + + // Clean up if disconnected or error + if (state === 'disconnected' || state === 'error') { + this._sessions.delete(sessionId); + } + } + + /** + * Handle transcripts from orchestrators. + */ + private _handleTranscript(sessionId: string, entry: TranscriptEntry): void { + logger.debug(`Session ${sessionId} transcript: [${entry.speaker}] ${entry.text}`); + + if (this._gatewayClient) { + this._gatewayClient.sendTranscript( + sessionId, + entry.speaker, + entry.text, + entry.isFinal + ); + } + } + + /** + * Handle errors from orchestrators. + */ + private _handleError(sessionId: string, error: Error): void { + logger.error(`Session ${sessionId} error:`, error); + + if (this._gatewayClient) { + this._gatewayClient.sendStatus(sessionId, 'error', error.message); + } + } +} diff --git a/src/types/index.ts b/src/types/index.ts new file mode 100644 index 0000000..c449227 --- /dev/null +++ b/src/types/index.ts @@ -0,0 +1,82 @@ +// WebSocket Protocol Types (Gateway <-> Bot) + +export interface PlayAudioMessage { + type: 'playAudio'; + sessionId: string; + audio: { + format: 'mp3' | 'pcm' | 'wav'; + data: string; // base64 encoded + }; +} + +export interface TranscriptMessage { + type: 'transcript'; + sessionId: string; + transcript: { + speaker: string; + text: string; + timestamp: string; + isFinal: boolean; + }; +} + +export interface StatusMessage { + type: 'status'; + sessionId: string; + status: 'connecting' | 'in_lobby' | 'joined' | 'left' | 'error'; + message?: string; +} + +export interface JoinMeetingMessage { + type: 'joinMeeting'; + sessionId: string; + meetingUrl: string; + botName?: string; +} + +export interface LeaveMeetingMessage { + type: 'leaveMeeting'; + sessionId: string; +} + +export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage; +export type BotToGateway = TranscriptMessage | StatusMessage; + +// Bot State +export type BotState = + | 'idle' + | 'launching' + | 'navigating' + | 'in_lobby' + | 'in_meeting' + | 'leaving' + | 'error' + | 'disconnected'; + +// Session +export interface BotSession { + sessionId: string; + meetingUrl: string; + botName: string; + state: BotState; + createdAt: Date; + joinedAt?: Date; + leftAt?: Date; + error?: string; +} + +// Transcript Entry +export interface TranscriptEntry { + speaker: string; + text: string; + timestamp: Date; + isFinal: boolean; +} + +// Meeting URL Types +export interface ParsedMeetingUrl { + type: 'classic' | 'short'; + originalUrl: string; + meetingId?: string; + passcode?: string; +} diff --git a/src/utils/logger.ts b/src/utils/logger.ts new file mode 100644 index 0000000..38b3bf3 --- /dev/null +++ b/src/utils/logger.ts @@ -0,0 +1,43 @@ +import winston from 'winston'; +import path from 'path'; +import fs from 'fs'; +import { config } from '../config'; + +// Ensure log directory exists +if (!fs.existsSync(config.logDir)) { + fs.mkdirSync(config.logDir, { recursive: true }); +} + +const logFormat = winston.format.combine( + winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }), + winston.format.errors({ stack: true }), + winston.format.printf(({ level, message, timestamp, sessionId, ...meta }) => { + const sessionPrefix = sessionId ? `[${sessionId}] ` : ''; + const metaStr = Object.keys(meta).length ? ` ${JSON.stringify(meta)}` : ''; + return `${timestamp} ${level.toUpperCase()} ${sessionPrefix}${message}${metaStr}`; + }) +); + +export const logger = winston.createLogger({ + level: config.logLevel, + format: logFormat, + transports: [ + new winston.transports.Console({ + format: winston.format.combine( + winston.format.colorize(), + logFormat + ), + }), + new winston.transports.File({ + filename: path.join(config.logDir, 'error.log'), + level: 'error', + }), + new winston.transports.File({ + filename: path.join(config.logDir, 'combined.log'), + }), + ], +}); + +export function createSessionLogger(sessionId: string) { + return logger.child({ sessionId }); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..1951778 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}