From 39a422af257669968a11a55ccbbbed58d4930c44 Mon Sep 17 00:00:00 2001
From: ValueOn AG <p.motsch@valueon.ch>
Date: Thu, 26 Feb 2026 09:05:21 +0100
Subject: [PATCH] feat: add hybrid speaker hints for audio-mode transcripts

Made-with: Cursor
---
 src/bot/orchestrator.ts | 34 +++++++++++++++++++++++++++++-----
 src/types/index.ts      |  1 +
 2 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/src/bot/orchestrator.ts b/src/bot/orchestrator.ts
index eb9e7fb..f621457 100644
--- a/src/bot/orchestrator.ts
+++ b/src/bot/orchestrator.ts
@@ -726,7 +726,12 @@ export class BotOrchestrator {
   /**
    * Send a transcript to the Gateway.
    */
-  private _sendTranscript(speaker: string, text: string, isFinal: boolean): void {
+  private _sendTranscript(
+    speaker: string,
+    text: string,
+    isFinal: boolean,
+    source: 'caption' | 'audioCapture' | 'speakerHint' | 'chat' = 'caption',
+  ): void {
     const message: TranscriptMessage = {
       type: 'transcript',
       sessionId: this._sessionId,
@@ -735,6 +740,7 @@ export class BotOrchestrator {
         text,
         timestamp: new Date().toISOString(),
         isFinal,
+        source,
       },
     };
     this._sendToGateway(message);
@@ -920,10 +926,14 @@ export class BotOrchestrator {
       this._page,
       this._logger,
       (entry) => {
-        // Send transcript to Gateway
-        this._sendTranscript(entry.speaker, entry.text, entry.isFinal);
-        // Also notify local callbacks
-        this._callbacks.onTranscript(entry);
+        const transferMode = this._getEffectiveTransferMode();
+        if (transferMode === 'audio') {
+          // In audio mode, captions are only used as speaker hints.
+          this._sendTranscript(entry.speaker, entry.text, entry.isFinal, 'speakerHint');
+        } else {
+          this._sendTranscript(entry.speaker, entry.text, entry.isFinal, 'caption');
+          this._callbacks.onTranscript(entry);
+        }
       },
       this._options.language
     );
@@ -1098,6 +1108,19 @@ export class BotOrchestrator {
     }
   }
 
+  /**
+   * Subscribe to captions only as speaker hints (without enabling captions flow).
+   * This supports hybrid mode: audio text + caption-based speaker names.
+   */
+  private async _enableSpeakerHintsFromCaptions(): Promise<void> {
+    try {
+      await this._captionsProcedure!.subscribeToCaptions();
+      this._logger.info('Speaker hints from captions subscribed (audio mode)');
+    } catch (error) {
+      this._logger.warn('Could not subscribe to captions for speaker hints:', error);
+    }
+  }
+
   /**
    * Enable transcript capture (captions or audio) based on transfer mode.
    */
@@ -1109,6 +1132,7 @@ export class BotOrchestrator {
       await this._enableCaptions();
     } else {
       await this._enableAudioCapture();
+      await this._enableSpeakerHintsFromCaptions();
     }
   }
 
diff --git a/src/types/index.ts b/src/types/index.ts
index c0a6004..5912172 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -17,6 +17,7 @@ export interface TranscriptMessage {
     text: string;
     timestamp: string;
     isFinal: boolean;
+    source?: 'caption' | 'audioCapture' | 'speakerHint' | 'chat';
   };
 }