From 4437576acc3bc0120fb8fb19e58ec8ba4ae7af4f Mon Sep 17 00:00:00 2001
From: ValueOn AG <p.motsch@valueon.ch>
Date: Thu, 26 Feb 2026 21:45:47 +0100
Subject: [PATCH] Add playback acknowledgement from bot to gateway.

Send ttsPlaybackAck events (queued/completed/failed) from bot and forward them as ttsDeliveryStatus SSE updates for session diagnostics.

Made-with: Cursor
---
 src/bot/orchestrator.ts | 31 +++++++++++++++++++++++++++++--
 src/types/index.ts      | 14 +++++++++++++-
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts
index 458b0de..37631c1 100644
--- a/src/bot/orchestrator.ts
+++ b/src/bot/orchestrator.ts
@@ -7,7 +7,7 @@ import WebSocket from 'ws';
 
 import { config } from '../config';
 import { createSessionLogger } from '../utils/logger';
-import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage } from '../types';
+import { BotSession, BotState, TranscriptEntry, StatusMessage, TranscriptMessage, PlayAudioMessage, ChatMessage, SendChatMessage, AudioChunkMessage, TtsPlaybackAckMessage } from '../types';
 import { JoinProcedure } from './joinProcedure';
 import { CaptionsProcedure } from './captionsProcedure';
 import { AudioProcedure } from './audioProcedure';
@@ -843,7 +843,34 @@ export class BotOrchestrator {
       }
     }
 
-    await this._audioProcedure.playAudio(audioData, format);
+    this._sendTtsPlaybackAck('queued', format, audioData.length, 'Audio queued for playback');
+    try {
+      await this._audioProcedure.playAudio(audioData, format);
+      this._sendTtsPlaybackAck('completed', format, audioData.length, 'Audio playback completed');
+    } catch (error) {
+      this._sendTtsPlaybackAck('failed', format, audioData.length, String(error));
+      throw error;
+    }
+  }
+
+  private _sendTtsPlaybackAck(
+    status: 'queued' | 'completed' | 'failed',
+    format: 'mp3' | 'wav' | 'pcm',
+    bytesBase64?: number,
+    message?: string,
+  ): void {
+    const ack: TtsPlaybackAckMessage = {
+      type: 'ttsPlaybackAck',
+      sessionId: this._sessionId,
+      playback: {
+        status,
+        format,
+        bytesBase64,
+        message,
+        timestamp: new Date().toISOString(),
+      },
+    };
+    this._sendToGateway(ack);
   }
 
   /**
diff --git a/src/types/index.ts b/src/types/index.ts
index 5912172..3d71dfb 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -73,8 +73,20 @@ export interface AudioChunkMessage {
   };
 }
 
+export interface TtsPlaybackAckMessage {
+  type: 'ttsPlaybackAck';
+  sessionId: string;
+  playback: {
+    status: 'queued' | 'completed' | 'failed';
+    format: 'mp3' | 'pcm' | 'wav';
+    bytesBase64?: number;
+    message?: string;
+    timestamp: string;
+  };
+}
+
 export type GatewayToBot = PlayAudioMessage | JoinMeetingMessage | LeaveMeetingMessage | SendChatMessage;
-export type BotToGateway = TranscriptMessage | StatusMessage | ChatMessage | AudioChunkMessage;
+export type BotToGateway = TranscriptMessage | StatusMessage | ChatMessage | AudioChunkMessage | TtsPlaybackAckMessage;
 
 // Bot State
 export type BotState =