Merge pull request #1449 from iceljc/features/gpt-live-provider · SciSharp/BotSharp@eb23532 · GitHub
Skip to content

Commit eb23532

Browse files
iceljciceljc
authored
Merge pull request #1449 from iceljc/features/gpt-live-provider
Features/gpt live provider
2 parents a43fa45 + c0f7fea commit eb23532

29 files changed

Lines changed: 3050 additions & 46 deletions

‎src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs‎

Lines changed: 14 additions & 0 deletions

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/ILiveCompletion.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
namespace BotSharp.Abstraction.MLTasks;
2+
3+
/// <summary>
4+
/// A full duplex voice model that owns the conversation itself: it listens and speaks at the
5+
/// same time, arbitrates interruptions internally, and delegates reasoning and tool calls to a
6+
/// separate backend model.
7+
///
8+
/// It drives the same hub as <see cref="IRealTimeCompletion"/> and is used wherever a realtime
9+
/// completer is expected, so the interface adds nothing to that contract. It exists to keep the
10+
/// two families apart in DI: a live provider is never returned by
11+
/// <c>GetServices&lt;IRealTimeCompletion&gt;()</c>, so it is reachable only through the agent's
12+
/// live config, never as a realtime fallback.
13+
/// </summary>
14+
public interface ILiveCompletion : IRealTimeCompletion
15+
{
16+
}

‎src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,13 @@ public enum LlmModelType
185185
Embedding = 4,
186186
Audio = 5,
187187
Realtime = 6,
188-
Web = 7
188+
Web = 7,
189+
190+
/// <summary>
191+
/// Full duplex voice model that runs the conversation itself and delegates the thinking to
192+
/// a backend model. Kept apart from <see cref="Realtime"/>: the two are not interchangeable.
193+
/// </summary>
194+
Live = 8
189195
}
190196

191197
public enum LlmModelCapability
@@ -203,5 +209,6 @@ public enum LlmModelCapability
203209
AudioGeneration = 10,
204210
Realtime = 11,
205211
WebSearch = 12,
206-
PdfReading = 13
212+
PdfReading = 13,
213+
Live = 14
207214
}

‎src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,20 @@ namespace BotSharp.Abstraction.Realtime;
66

77
public interface IRealtimeHook : IHookBase
88
{
9+
/// <summary>
10+
/// Establishes the ambient identity for a stream connection, before anything runs on it.
11+
///
12+
/// Synchronous on purpose. Ambient identity is AsyncLocal-backed, and a write made inside an
13+
/// `async` method does not flow back out to its caller - so an awaited hook cannot establish
14+
/// identity for the connection that follows it, only for itself.
15+
/// </summary>
16+
/// <param name="userId">
17+
/// Who the client says it is. A browser cannot set headers on a WebSocket handshake, so this
18+
/// arrives as a query parameter and is UNVERIFIED: treat it as a claim to be checked, not as
19+
/// proof. Null when the client sent none.
20+
/// </param>
21+
void OnAuthenticate(string agentId, string conversationId, string? userId) { }
22+
923
Task OnModelReady(Agent agent, IRealTimeCompletion completer)
1024
=> Task.CompletedTask;
1125

‎src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,18 @@ public class RealtimeHubConnection
1616
public Func<string> OnModelUserInterrupted { get; set; } = null!;
1717
public Func<string> OnUserSpeechDetected { get; set; } = () => string.Empty;
1818

19+
/// <summary>
20+
/// Serializes a partial transcript (role, delta) for the user stream. Display only: a turn
21+
/// still reaches conversation storage only once it is complete.
22+
/// </summary>
23+
public Func<string, string, string>? OnModelTranscriptDelta { get; set; }
24+
25+
/// <summary>
26+
/// Set by the hub so a provider can push display-only events straight to the user stream,
27+
/// without going through the conversation hooks that write history.
28+
/// </summary>
29+
public Func<string, Task>? SendEventToUser { get; set; }
30+
1931
public void ResetResponseState()
2032
{
2133
LastAssistantItemId = null;

‎src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,15 @@ public async Task ConnectToModel(
5252
routing.Context.SetDialogs(dialogs);
5353
routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString());
5454

55-
var (provider, model) = GetLlmProviderModel(agent);
55+
var (provider, model, isLive) = GetLlmProviderModel(agent);
5656

57-
_completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == provider);
57+
_completer = ResolveCompleter(provider, isLive);
5858
_completer.SetModelName(model);
5959
_completer.SetOptions(options);
6060

61+
// Lets a provider stream display-only events, such as partial transcripts.
62+
_conn.SendEventToUser = responseToUser;
63+
6164
await _completer.Connect(
6265
conn: _conn,
6366
onModelReady: async () =>
@@ -189,27 +192,54 @@ public RealtimeHubConnection SetHubConnection(string conversationId)
189192
return _conn;
190193
}
191194

192-
private (string, string) GetLlmProviderModel(Agent agent)
195+
/// <summary>
196+
/// Picks the voice model for this conversation, and which family it belongs to.
197+
///
198+
/// The agent's live config wins when it is set, because a live session is a deliberate
199+
/// choice rather than a variation of a realtime one. Everything else resolves to realtime:
200+
/// RealtimeModelSettings describes a realtime provider, so a live provider is never reached
201+
/// by fallback, only by an agent asking for it.
202+
/// </summary>
203+
private (string provider, string model, bool isLive) GetLlmProviderModel(Agent agent)
193204
{
194-
var provider = agent?.LlmConfig?.Realtime?.Provider;
195-
var model = agent?.LlmConfig?.Realtime?.Model;
196-
var settingService = _services.GetRequiredService<ISettingService>();
205+
var live = agent?.LlmConfig?.Live;
206+
if (live?.IsValid == true)
207+
{
208+
return (live.Provider!, live.Model!, true);
209+
}
197210

198-
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
211+
var realtime = agent?.LlmConfig?.Realtime;
212+
if (realtime?.IsValid == true)
199213
{
200-
return (provider, model);
214+
return (realtime.Provider!, realtime.Model!, false);
201215
}
202216

203-
provider = _settings.Provider;
204-
model = _settings.Model;
217+
if (!string.IsNullOrEmpty(_settings.Provider) && !string.IsNullOrEmpty(_settings.Model))
218+
{
219+
return (_settings.Provider, _settings.Model, false);
220+
}
221+
222+
return ("openai", "gpt-realtime", false);
223+
}
224+
225+
/// <summary>
226+
/// Live providers register as <see cref="ILiveCompletion"/> only, so the two families are
227+
/// resolved from separate DI lists and a provider name is looked up in one of them, never
228+
/// both. A live completer still drives the hub as a realtime one - the interface derives
229+
/// from <see cref="IRealTimeCompletion"/> - so nothing downstream has to know the difference.
230+
/// </summary>
231+
private IRealTimeCompletion ResolveCompleter(string provider, bool isLive)
232+
{
233+
var completer = isLive
234+
? _services.GetServices<ILiveCompletion>().FirstOrDefault(x => x.Provider == provider)
235+
: _services.GetServices<IRealTimeCompletion>().FirstOrDefault(x => x.Provider == provider);
205236

206-
if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model))
237+
if (completer == null)
207238
{
208-
return (provider, model);
239+
var family = isLive ? "live" : "realtime";
240+
throw new InvalidOperationException($"Can't resolve the {family} completion provider by \"{provider}\".");
209241
}
210242

211-
provider = "openai";
212-
model = "gpt-realtime";
213-
return (provider, model);
243+
return completer;
214244
}
215245
}

‎src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,16 @@ public async Task<bool> Execute(RoleDialogModel message)
3838

3939
await Task.Delay(1500);
4040

41-
#if DEBUG
42-
var intermediateMsg = RoleDialogModel.From(message, AgentRole.Assistant, $"Here is your weather in {args?.City}");
43-
messageHub.Push(new()
44-
{
45-
EventName = ChatEvent.OnIntermediateMessageReceivedFromAssistant,
46-
Data = intermediateMsg,
47-
RefId = conv.ConversationId,
48-
SaveDataToDb = true
49-
});
50-
#endif
41+
//#if DEBUG
42+
// var intermediateMsg = RoleDialogModel.From(message, AgentRole.Assistant, $"Here is your weather in {args?.City}");
43+
// messageHub.Push(new()
44+
// {
45+
// EventName = ChatEvent.OnIntermediateMessageReceivedFromAssistant,
46+
// Data = intermediateMsg,
47+
// RefId = conv.ConversationId,
48+
// SaveDataToDb = true
49+
// });
50+
//#endif
5151

5252
message.Indication = $"Still working on it... Hold on, {args?.City}";
5353
messageHub.Push(new()
@@ -62,20 +62,20 @@ public async Task<bool> Execute(RoleDialogModel message)
6262
message.Content = $"It is a sunny day!";
6363
message.StopCompletion = false;
6464

65-
#if DEBUG
66-
var sidecar = _services.GetService<IConversationSideCar>();
67-
if (sidecar != null)
68-
{
69-
var text = $"I want to know fun events in {args?.City}";
70-
var states = new List<MessageState>
71-
{
72-
new() { Key = StateConst.CHANNEL, Value = ConversationChannel.Email }
73-
};
74-
75-
var msg = await sidecar.SendMessage(message.CurrentAgentId, text, states: states);
76-
message.Content = $"{message.Content} {msg.Content}";
77-
}
78-
#endif
65+
//#if DEBUG
66+
// var sidecar = _services.GetService<IConversationSideCar>();
67+
// if (sidecar != null)
68+
// {
69+
// var text = $"I want to know fun events in {args?.City}";
70+
// var states = new List<MessageState>
71+
// {
72+
// new() { Key = StateConst.CHANNEL, Value = ConversationChannel.Email }
73+
// };
74+
75+
// var msg = await sidecar.SendMessage(message.CurrentAgentId, text, states: states);
76+
// message.Content = $"{message.Content} {msg.Content}";
77+
// }
78+
//#endif
7979

8080
return true;
8181
}

‎src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ public static object GetCompletion(
4040
{
4141
return GetRealTimeCompletion(services, provider: provider, model: model);
4242
}
43+
else if (settings.Type == LlmModelType.Live)
44+
{
45+
return GetLiveCompletion(services, provider: provider, model: model);
46+
}
4347
else
4448
{
4549
return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig);
@@ -205,6 +209,35 @@ public static IRealTimeCompletion GetRealTimeCompletion(
205209
return completer;
206210
}
207211

212+
/// <summary>
213+
/// Live providers are registered as <see cref="ILiveCompletion"/> only, so they are resolved
214+
/// from their own list: a provider name is never shared between the two voice families here.
215+
/// </summary>
216+
public static ILiveCompletion GetLiveCompletion(
217+
IServiceProvider services,
218+
string? provider = null,
219+
string? model = null,
220+
string? modelId = null,
221+
bool? multiModal = null,
222+
AgentLlmConfig? agentConfig = null)
223+
{
224+
var completions = services.GetServices<ILiveCompletion>();
225+
(provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
226+
multiModal: multiModal,
227+
modelType: LlmModelType.Live,
228+
agentConfig: agentConfig);
229+
230+
var completer = completions.FirstOrDefault(x => x.Provider == provider);
231+
if (completer == null)
232+
{
233+
var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
234+
logger.LogError($"Can't resolve live completion provider by {provider}");
235+
}
236+
237+
completer?.SetModelName(model);
238+
return completer;
239+
}
240+
208241
private static (string, string) GetProviderAndModel(
209242
IServiceProvider services,
210243
string? provider = null,

‎src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs‎

Lines changed: 45 additions & 1 deletion

0 commit comments

Comments
 (0)