|
| 1 | +using BotSharp.Abstraction.Functions.Models; |
| 2 | +using BotSharp.Plugin.OpenAI.Providers.Chat; |
| 3 | +using OpenAI.Chat; |
| 4 | +using Xunit; |
| 5 | + |
| 6 | +namespace BotSharp.Core.UnitTests.Routing; |
| 7 | + |
| 8 | +/// <summary> |
| 9 | +/// The streamed counterpart of reading a reply's tool calls. A non-streaming reply hands over a |
| 10 | +/// finished list; a streamed one arrives as fragments that have to be reassembled, and that |
| 11 | +/// reassembly is the only part of the path with no production evidence behind it -- the streaming |
| 12 | +/// branch logs its calls under <c>#if DEBUG</c>, so a Release deployment that uses it shows |
| 13 | +/// nothing at all. |
| 14 | +/// </summary> |
| 15 | +/// <remarks> |
| 16 | +/// Why reassembly is guesswork in the first place: <see cref="StreamingChatToolCallUpdate"/> in |
| 17 | +/// the OpenAI SDK exposes only FunctionArgumentsUpdate, FunctionName, Kind and ToolCallId. There |
| 18 | +/// is no public index, so which call a fragment belongs to has to be inferred -- an update |
| 19 | +/// carrying a tool call id different from the one in progress starts a new call, and everything |
| 20 | +/// after it belongs to that one. |
| 21 | +/// <para> |
| 22 | +/// The case these exist for: two calls in one reply. Appending every fragment to a single string, |
| 23 | +/// as this did originally, is invisible while a reply asks for one tool and produces a single |
| 24 | +/// malformed argument blob the moment it asks for two. |
| 25 | +/// </para> |
| 26 | +/// </remarks> |
| 27 | +public class StreamingToolCallReconstructionTests |
| 28 | +{ |
| 29 | + /// <summary> |
| 30 | + /// One streamed fragment. Null <paramref name="id"/> means the update continues the call in |
| 31 | + /// progress, which is how the SDK reports every fragment after the first of a call. |
| 32 | + /// </summary> |
| 33 | + private static StreamingChatToolCallUpdate Update(string? id = null, string? name = null, string? args = null) |
| 34 | + => OpenAIChatModelFactory.StreamingChatToolCallUpdate( |
| 35 | + index: 0, |
| 36 | + toolCallId: id!, |
| 37 | + kind: ChatToolCallKind.Function, |
| 38 | + functionName: name!, |
| 39 | + functionArgumentsUpdate: args != null ? BinaryData.FromString(args) : null!); |
| 40 | + |
| 41 | + [Fact] |
| 42 | + public void Reconstruct_JoinsTheFragmentsOfOneCall() |
| 43 | + { |
| 44 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 45 | + [ |
| 46 | + Update(id: "call_1", name: "get_weather", args: "{\"ci"), |
| 47 | + Update(args: "ty\":\"Chi"), |
| 48 | + Update(args: "cago\"}") |
| 49 | + ]); |
| 50 | + |
| 51 | + var call = Assert.Single(calls); |
| 52 | + Assert.Equal("call_1", call.Id); |
| 53 | + Assert.Equal("get_weather", call.FunctionName); |
| 54 | + Assert.Equal("{\"city\":\"Chicago\"}", call.FunctionArgs); |
| 55 | + } |
| 56 | + |
| 57 | + /// <summary> |
| 58 | + /// The regression this reassembly exists for. Both calls must come out whole and separate; |
| 59 | + /// the failure it replaced produced one call whose arguments were the two JSON documents |
| 60 | + /// concatenated. |
| 61 | + /// </summary> |
| 62 | + [Fact] |
| 63 | + public void Reconstruct_KeepsTwoCallsArgumentsApart() |
| 64 | + { |
| 65 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 66 | + [ |
| 67 | + Update(id: "call_1", name: "get_weather", args: "{\"city\":"), |
| 68 | + Update(args: "\"Chicago\"}"), |
| 69 | + Update(id: "call_2", name: "get_time", args: "{\"zone\":"), |
| 70 | + Update(args: "\"CST\"}") |
| 71 | + ]); |
| 72 | + |
| 73 | + Assert.Equal(2, calls.Count); |
| 74 | + Assert.Equal(["call_1", "call_2"], calls.Select(x => x.Id)); |
| 75 | + Assert.Equal(["get_weather", "get_time"], calls.Select(x => x.FunctionName)); |
| 76 | + Assert.Equal("{\"city\":\"Chicago\"}", calls[0].FunctionArgs); |
| 77 | + Assert.Equal("{\"zone\":\"CST\"}", calls[1].FunctionArgs); |
| 78 | + } |
| 79 | + |
| 80 | + /// <summary> |
| 81 | + /// A reply really does ask for the same tool more than once -- production shows models doing |
| 82 | + /// it with up to four copies of one name in a single reply. The id is the only thing telling |
| 83 | + /// those apart, so grouping must not fall back to the name. |
| 84 | + /// </summary> |
| 85 | + [Fact] |
| 86 | + public void Reconstruct_SeparatesTwoCallsOfTheSameTool() |
| 87 | + { |
| 88 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 89 | + [ |
| 90 | + Update(id: "call_1", name: "process_wo_avoidance", args: "{\"key_words\":\"toilet leak\"}"), |
| 91 | + Update(id: "call_2", name: "process_wo_avoidance", args: "{\"key_words\":\"roof repair\"}") |
| 92 | + ]); |
| 93 | + |
| 94 | + Assert.Equal(2, calls.Count); |
| 95 | + Assert.Equal(["call_1", "call_2"], calls.Select(x => x.Id)); |
| 96 | + Assert.Equal("{\"key_words\":\"toilet leak\"}", calls[0].FunctionArgs); |
| 97 | + Assert.Equal("{\"key_words\":\"roof repair\"}", calls[1].FunctionArgs); |
| 98 | + } |
| 99 | + |
| 100 | + /// <summary> |
| 101 | + /// A provider that stamps the id on every fragment rather than only on the first must not be |
| 102 | + /// read as opening a new call per fragment. |
| 103 | + /// </summary> |
| 104 | + [Fact] |
| 105 | + public void Reconstruct_DoesNotReopenACallWhenTheIdRepeats() |
| 106 | + { |
| 107 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 108 | + [ |
| 109 | + Update(id: "call_1", name: "get_weather", args: "{\"city\":"), |
| 110 | + Update(id: "call_1", args: "\"Chicago\"}") |
| 111 | + ]); |
| 112 | + |
| 113 | + var call = Assert.Single(calls); |
| 114 | + Assert.Equal("{\"city\":\"Chicago\"}", call.FunctionArgs); |
| 115 | + } |
| 116 | + |
| 117 | + /// <summary> |
| 118 | + /// The name does not have to arrive with the id. When it comes in a later fragment it still |
| 119 | + /// has to land on the call in progress, or the call is dispatched under an empty name. |
| 120 | + /// </summary> |
| 121 | + [Fact] |
| 122 | + public void Reconstruct_TakesTheNameFromALaterFragment() |
| 123 | + { |
| 124 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 125 | + [ |
| 126 | + Update(id: "call_1"), |
| 127 | + Update(name: "get_weather"), |
| 128 | + Update(args: "{}") |
| 129 | + ]); |
| 130 | + |
| 131 | + var call = Assert.Single(calls); |
| 132 | + Assert.Equal("call_1", call.Id); |
| 133 | + Assert.Equal("get_weather", call.FunctionName); |
| 134 | + Assert.Equal("{}", call.FunctionArgs); |
| 135 | + } |
| 136 | + |
| 137 | + /// <summary> |
| 138 | + /// A first fragment with no id still opens a call rather than indexing into an empty list. |
| 139 | + /// </summary> |
| 140 | + [Fact] |
| 141 | + public void Reconstruct_OpensACallEvenWhenTheFirstFragmentHasNoId() |
| 142 | + { |
| 143 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 144 | + [ |
| 145 | + Update(name: "get_weather", args: "{\"city\":"), |
| 146 | + Update(args: "\"Chicago\"}") |
| 147 | + ]); |
| 148 | + |
| 149 | + var call = Assert.Single(calls); |
| 150 | + Assert.Equal("get_weather", call.FunctionName); |
| 151 | + Assert.Equal("{\"city\":\"Chicago\"}", call.FunctionArgs); |
| 152 | + } |
| 153 | + |
| 154 | + /// <summary> |
| 155 | + /// Arguments are never null on the way out: a call the model sent no arguments for has to |
| 156 | + /// serialize as something the provider can put in the request. |
| 157 | + /// </summary> |
| 158 | + [Fact] |
| 159 | + public void Reconstruct_GivesACallWithoutArgumentsAnEmptyString() |
| 160 | + { |
| 161 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 162 | + [ |
| 163 | + Update(id: "call_1", name: "list_work_orders") |
| 164 | + ]); |
| 165 | + |
| 166 | + var call = Assert.Single(calls); |
| 167 | + Assert.Equal(string.Empty, call.FunctionArgs); |
| 168 | + } |
| 169 | + |
| 170 | + [Fact] |
| 171 | + public void Reconstruct_ReturnsNothingForAnEmptyStream() |
| 172 | + { |
| 173 | + Assert.Empty(ChatCompletionProvider.ReconstructToolCalls([])); |
| 174 | + } |
| 175 | + |
| 176 | + /// <summary> |
| 177 | + /// Order is the contract the rest of the engine reads: results are appended in the order the |
| 178 | + /// model asked, and a reply that stops the turn part way through keeps the calls before it. |
| 179 | + /// </summary> |
| 180 | + [Fact] |
| 181 | + public void Reconstruct_KeepsTheOrderTheModelSentThemIn() |
| 182 | + { |
| 183 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 184 | + [ |
| 185 | + Update(id: "call_1", name: "a", args: "{}"), |
| 186 | + Update(id: "call_2", name: "b", args: "{}"), |
| 187 | + Update(id: "call_3", name: "c", args: "{}") |
| 188 | + ]); |
| 189 | + |
| 190 | + Assert.Equal(["a", "b", "c"], calls.Select(x => x.FunctionName)); |
| 191 | + } |
| 192 | + |
| 193 | + /// <summary> |
| 194 | + /// What the engine reads afterwards. <see cref="LlmToolCall"/> is the shape the routing side |
| 195 | + /// dispatches on, and a call is only dispatchable with both an id to answer under and a name |
| 196 | + /// to resolve. |
| 197 | + /// </summary> |
| 198 | + [Fact] |
| 199 | + public void Reconstruct_ProducesCallsTheEngineCanDispatch() |
| 200 | + { |
| 201 | + var calls = ChatCompletionProvider.ReconstructToolCalls( |
| 202 | + [ |
| 203 | + Update(id: "call_1", name: "get_weather", args: "{\"city\":\"Chicago\"}"), |
| 204 | + Update(id: "call_2", name: "get_time", args: "{\"zone\":\"CST\"}") |
| 205 | + ]); |
| 206 | + |
| 207 | + Assert.Equal(2, calls.Count); |
| 208 | + Assert.All(calls, call => |
| 209 | + { |
| 210 | + Assert.False(string.IsNullOrEmpty(call.Id)); |
| 211 | + Assert.False(string.IsNullOrEmpty(call.FunctionName)); |
| 212 | + Assert.NotNull(call.FunctionArgs); |
| 213 | + }); |
| 214 | + } |
| 215 | +} |
0 commit comments