{{ message }}
Repository navigation
Expand file tree
/
Copy pathtest-tool-relay.mjs
More file actions
375 lines (344 loc) · 18.2 KB
/
Copy pathtest-tool-relay.mjs
File metadata and controls
375 lines (344 loc) · 18.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
// Tool relay: which tools get registered, which get blocked, which get
// skipped. Guards both ends — a blocked-but-should-pass tool silently costs
// extra money; a passed-but-should-block tool leaves the agent unable to
// work.
import assert from "node:assert/strict";
import { RelayTurn, buildCustomTools, feedResults } from "./src/tool-relay.mjs";
const tests = [];
const passed = [];
const failed = [];
function test(name, fn) {
tests.push({ name, fn });
}
async function run() {
for (const { name, fn } of tests) {
try {
await fn();
passed.push(name);
} catch (e) {
failed.push({ name, error: e });
}
}
}
// buildCustomTools eats the protocol-neutral shape; converting each protocol
// into that shape is the adapters' job (see test-protocol.mjs). This layer
// must not know that OpenAI calls the schema `parameters` while Anthropic
// calls it `input_schema`.
const tool = (name) => ({ name, description: "", parameters: { type: "object", properties: {} } });
const names = (list) => Object.keys(buildCustomTools(list, new RelayTurn()));
// ── registration and blocking ───────────────────────────
test("registration: every normal tool registers — none may be lost", () => {
const keep = [
"read_file", "list_dir", "grep", "codebase_search", "run_terminal_cmd",
"edit_file", "write", "todo_write", "web_search", "delete_file",
// Names containing task/agent that do NOT spawn new conversations must
// not be collateral damage.
"agentic_search", "list_tasks", "delegate_review",
];
assert.deepEqual(names(keep.map((n) => tool(n))), keep);
});
test("blocking: subagent spawners are rejected", () => {
// Turns are billed anyway, but 5 subagents in one turn is 6 billings, and
// the client cannot see where the money went.
for (const n of ["Task", "task", "subagent", "spawn_agent", "launchAgent",
"create_agent", "run_agent", "dispatch_agent",
"best_of_n_runner", "best-of-n-runner"]) {
assert.deepEqual(names([tool(n)]), [], `${n} must be blocked`);
}
});
test("blocking: spawners are rejected even among normal tools", () => {
assert.deepEqual(names([tool("read_file"), tool("Task"), tool("grep")]), ["read_file", "grep"]);
});
test("skipping: nameless entries are skipped, never crash", () => {
assert.deepEqual(names([tool("read_file"), {}, { description: "nameless" }, tool("grep")]),
["read_file", "grep"]);
});
// ── tool results: resolving the pending call ────────────
test("results: a pending call resolves with the client's answer", async () => {
const turn = new RelayTurn();
const tools = buildCustomTools([tool("read_file")], turn);
// execute doesn't actually run; it suspends waiting for the client's result.
const pending = tools.read_file.execute({ path: "/x" });
assert.equal(turn.waiting, true, "after the call the turn must be waiting");
const callId = [...turn.pending.keys()][0];
assert.match(callId, /^call_/, "the default prefix is the OpenAI set");
assert.equal(feedResults(turn, [{ id: callId, content: "file content" }]), 1);
assert.equal(await pending, "file content", "the returned result must reach the agent verbatim");
assert.equal(turn.waiting, false);
});
test("results: a wrong id never resolves the pending call", () => {
// That would hand A's result to B.
const turn = new RelayTurn();
buildCustomTools([tool("read_file")], turn).read_file.execute({ path: "/x" });
assert.equal(feedResults(turn, [{ id: "call_nonexistent", content: "x" }]), 0);
assert.equal(turn.waiting, true, "the call stays suspended");
});
test("results: non-string results serialize to JSON", async () => {
// Never feed [object Object] to the model.
const turn2 = new RelayTurn();
const t2 = buildCustomTools([tool("read_file")], turn2);
const p2 = t2.read_file.execute({});
const id2 = [...turn2.pending.keys()][0];
feedResults(turn2, [{ id: id2, content: { a: 1 } }]);
assert.equal(await p2, '{"a":1}');
});
// ── call-id prefix ──────────────────────────────────────
test("prefix: the call-id prefix is switchable per protocol", () => {
// Anthropic clients echo tool_use.id back verbatim as
// tool_result.tool_use_id; `call_`-prefixed ids can be treated as
// anomalous data.
const turnA = new RelayTurn();
turnA.callIdPrefix = "toolu_";
buildCustomTools([tool("read_file")], turnA).read_file.execute({});
assert.match([...turnA.pending.keys()][0], /^toolu_/);
});
// ── usage and errors ────────────────────────────────────
test("usage: the sink receives the SDK's raw usage, unconverted", async () => {
// Format conversion is the sink's job (it knows the wire format). It used
// to convert to the OpenAI shape in the engine; the Anthropic endpoint
// exposed the bug: that side wants input_tokens, so one of the two was
// forever empty — never errored, just blank stat columns.
const turn3 = new RelayTurn();
const captured = { usage: undefined };
const fakeSink = {
closed: false, parts: [],
role() {}, text() {}, toolCall() {},
finish(_reason, usage) { captured.usage = usage; this.closed = true; },
};
turn3.attach(fakeSink);
turn3.usage = { inputTokens: 11296, outputTokens: 68, totalTokens: 19785, cacheReadTokens: 8421 };
// Run the full wrap-up path (consume's finally closes the sink).
await turn3.consume({
async *stream() {},
async wait() { return { status: "finished", result: "" }; },
});
assert.equal(captured.usage.inputTokens, 11296, "the relay must not convert formats here");
assert.equal(captured.usage.prompt_tokens, undefined, "conversion is the sink's job");
assert.equal(turn3.usage.inputTokens, 11296, "accounting uses the raw shape too");
});
test("errors: upstream error objects serialize, never [object Object]", async () => {
// Regression: the SDK's `result.error` is measured to be an object;
// string concatenation turns it into `run error: [object Object]`. 98
// consecutive production failures on 2026-08-12, every log line like
// that — the only diagnostic clue erased.
const { describe } = await import("./src/logger.mjs");
assert.equal(describe({ code: "quota_exceeded", detail: "no quota left" }),
'{"code":"quota_exceeded","detail":"no quota left"}');
assert.equal(describe(new Error("boom")), "boom");
assert.equal(describe("already a string"), "already a string");
assert.equal(describe(null), "null");
const circular = { a: 1 }; circular.self = circular;
assert.ok(describe(circular).length > 0, "circular refs must not take the logging down");
});
test("errors: a run failure surfaces the upstream's reason", async () => {
const turn4 = new RelayTurn();
await turn4.consume({
async *stream() {},
async wait() { return { status: "error", error: { code: "rate_limited", retryAfter: 30 } }; },
});
assert.ok(!turn4.error.includes("[object Object]"), `error info was swallowed: ${turn4.error}`);
assert.match(turn4.error, /rate_limited/, "the upstream's reason must be visible");
});
// ── parallel tool calls ─────────────────────────────────
test("parallel: cached while the sink is closed, replayed on attach", async () => {
// Regression: Cursor's agent emits several parallel tool_use at once
// (Claude Code's concurrent subagents are three or four at a time). After
// the first flush closes the sink, later arrivals dropped outright would
// show the client "connection interrupted". Correct behaviour: cache and
// replay when the client's next request attaches.
const turn5 = new RelayTurn();
const sent5 = [];
const fakeSink5a = { text() {}, finish() {}, toolCall(i, c) { sent5.push(c); } };
const fakeSink5b = { text() {}, finish() {}, toolCall(i, c) { sent5.push(c); } };
turn5.attach(fakeSink5a);
// First call: sink open -> batched out after 80ms and closed.
const q1 = turn5.delegate("agent_a", { prompt: "task 1" });
await new Promise((r) => setTimeout(r, 150));
assert.equal(sent5.length, 1, "the first call must go out normally");
assert.equal(sent5[0].name, "agent_a");
assert.equal(turn5.sink, null, "after the flush the sink closes (waiting for the client to run and return)");
// Sink closed, the model emits two more parallel calls -> must cache, not
// reject.
const q2 = turn5.delegate("agent_b", { prompt: "task 2" });
const q3 = turn5.delegate("agent_c", { prompt: "task 3" });
await new Promise((r) => setTimeout(r, 150));
const settled = await Promise.race([
Promise.allSettled([q2, q3]).then(() => "settled"),
new Promise((r) => setTimeout(() => r("timeout"), 300)),
]);
assert.equal(settled, "timeout", "with the sink closed, parallel calls must stay suspended, never reject");
// The client's next request (returning tool results) attaches -> replay
// the cache.
turn5.attach(fakeSink5b);
await new Promise((r) => setTimeout(r, 30));
const deferred = sent5.slice(1);
assert.equal(deferred.length, 2, "attaching must replay the cached parallel calls");
assert.equal(deferred[0].name, "agent_b");
assert.equal(deferred[1].name, "agent_c");
assert.equal(turn5.sink, null, "after the replay the sink closes again, waiting for results");
assert.ok(q2 instanceof Promise && q3 instanceof Promise, "the replay doesn't change the suspended semantics");
});
// ── description fallback ────────────────────────────────
test("description: empty descriptions distill from the schema", () => {
const turn6 = new RelayTurn();
const ct = buildCustomTools([
{ name: "no_desc", parameters: { type: "object", properties: { file_path: { type: "string" } }, required: ["file_path"] } },
{ name: "with_desc", description: "original description", parameters: { type: "object", properties: {} } },
], turn6);
assert.ok(ct.no_desc.description.length > 5, "empty descriptions need a generated summary");
assert.match(ct.no_desc.description, /file_path/, "the summary must include required param names");
assert.equal(ct.with_desc.description, "original description", "a given description stays verbatim");
});
test("description: non-string, blank and oversized descriptions are covered", () => {
const turn = new RelayTurn();
const ct = buildCustomTools([
{ name: "num_desc", description: 42, parameters: { type: "object", properties: { a: {} } } },
{ name: "blank_desc", description: " \n\t ", parameters: { type: "object", properties: { b: {} } } },
{ name: "long_desc", description: "x".repeat(12000), parameters: { type: "object", properties: {} } },
{ name: "emoji_desc", description: "a".repeat(9999) + "\ud83d\ude00tail", parameters: { type: "object", properties: {} } },
], turn);
assert.equal(ct.num_desc.description, "call a client-provided tool (params: a)", "non-string descriptions fall back to the schema summary");
assert.equal(ct.blank_desc.description, "call a client-provided tool (params: b)", "whitespace-only descriptions fall back too");
assert.equal(ct.long_desc.description.length, 10000, "oversized descriptions are truncated");
assert.equal(ct.emoji_desc.description, "a".repeat(9999),
"the truncation must not split a surrogate pair");
});
// ── schema normalization ────────────────────────────────
test("schema: client schemas are normalized for the SDK", () => {
const turn = new RelayTurn();
const ct = buildCustomTools([
{
name: "messy",
parameters: {
$schema: "http://json-schema.org/draft-07/schema#",
required: "file_path", // not an array
properties: null,
additionalProperties: "no", // string, not bool/schema
},
},
{
name: "nested",
parameters: {
type: "object",
required: ["top", 42, ""],
properties: {
file_path: { type: "string", required: "x", additionalProperties: "no" },
opts: { type: "object", properties: null },
},
},
},
], turn);
const messy = ct.messy.inputSchema;
assert.deepEqual(messy.required, [], "non-array required becomes an empty array");
assert.deepEqual(messy.properties, {}, "null properties become an empty object");
assert.equal(messy.type, "object", "a missing type defaults to object");
assert.equal(messy.additionalProperties, true, "a non-bool/object additionalProperties becomes true");
assert.equal(messy.$schema, undefined, "the $schema artifact is dropped");
const nested = ct.nested.inputSchema;
assert.deepEqual(nested.required, ["top", ""], "non-string required entries are filtered");
assert.deepEqual(nested.properties.file_path.required, [], "property schemas are normalized too");
assert.equal(nested.properties.file_path.additionalProperties, true);
assert.deepEqual(nested.properties.opts.properties, {}, "null properties inside a property schema are fixed");
});
// ── isError channel ────────────────────────────────────
test("results: an error result resolves as {content, isError} for the SDK", async () => {
// The SDK's result normalizer forwards execute()'s {content, isError}
// shape to the model; fed back as plain text, the model keeps reasoning
// on a failure it thinks succeeded.
const turn = new RelayTurn();
const tools = buildCustomTools([tool("read_file")], turn);
const pending = tools.read_file.execute({ path: "/x" });
const callId = [...turn.pending.keys()][0];
assert.equal(feedResults(turn, [{ id: callId, content: "permission denied", isError: true }]), 1);
assert.deepEqual(await pending,
{ content: [{ type: "text", text: "permission denied" }], isError: true });
assert.equal(turn.waiting, false);
});
test("results: an error result with non-string content serializes to JSON", async () => {
const turn = new RelayTurn();
const tools = buildCustomTools([tool("edit_file")], turn);
const pending = tools.edit_file.execute({});
const callId = [...turn.pending.keys()][0];
feedResults(turn, [{ id: callId, content: { code: 42 }, isError: true }]);
assert.deepEqual(await pending,
{ content: [{ type: "text", text: '{"code":42}' }], isError: true });
});
// ── cross-turn mixed batches ────────────────────────────
test("results: a mixed batch dispatches each id to its own turn", async () => {
// Two turns with tool calls in flight, results for both batched into one
// follow-up request. The old code fed everything into the first id's turn
// and silently dropped the second turn's results (its promise hung until
// the timeout).
const turnA = new RelayTurn();
const turnB = new RelayTurn();
const ta = buildCustomTools([tool("read_file")], turnA);
const tb = buildCustomTools([tool("grep")], turnB);
const pa = ta.read_file.execute({});
const pb = tb.grep.execute({});
const idA = [...turnA.pending.keys()][0];
const idB = [...turnB.pending.keys()][0];
assert.equal(feedResults(turnA, [
{ id: idA, content: "from A" },
{ id: idB, content: "from B" },
]), 2);
assert.equal(await pa, "from A");
assert.equal(await pb, "from B", "each result must reach its own turn, not the caller's");
assert.equal(turnA.waiting, false);
assert.equal(turnB.waiting, false);
});
test("results: an orphan id is reported, never silently dropped", () => {
const turn = new RelayTurn();
const tools = buildCustomTools([tool("read_file")], turn);
const pending = tools.read_file.execute({});
const callId = [...turn.pending.keys()][0];
assert.equal(feedResults(turn, [{ id: "call_unknown-orphan", content: "x" }]), 0,
"an id with no pending call feeds nothing");
assert.equal(turn.waiting, true, "the real call stays suspended");
feedResults(turn, [{ id: callId, content: "real" }]);
return pending.then((v) => assert.equal(v, "real", "the real id still resolves afterwards"));
});
// ── dead turns ─────────────────────────────────────────
test("dead turn: attach after the run finished never replays parked calls", async () => {
const turn = new RelayTurn();
const sinkA = { text() {}, finish() {}, toolCall() {} };
turn.attach(sinkA);
const qA = turn.delegate("agent_a", {});
await new Promise((r) => setTimeout(r, 150)); // flushed, sink closed
const qB = turn.delegate("agent_b", {}); // parked (sink closed)
// The run ends while calls are parked.
await turn.consume({
async *stream() {},
async wait() { return { status: "finished", result: "" }; },
});
assert.equal(turn.finished, true);
// consume's finally rejects every pending call ("run ended"); swallow the
// rejections the way the SDK's executor would.
await Promise.allSettled([qA, qB]);
let replayed = 0;
let sealed = false;
const sinkB = { text() {}, finish() { sealed = true; }, toolCall() { replayed += 1; } };
const end = turn.attach(sinkB);
assert.equal(replayed, 0, "a dead turn must not replay stale calls into a fresh response");
assert.equal(sealed, true, "the dead turn's sink is sealed immediately");
assert.equal(turn.sink, null);
const settled = await Promise.race([
end.then(() => "settled"),
new Promise((r) => setTimeout(() => r("pending"), 50)),
]);
assert.equal(settled, "pending", "the attach promise never resolves: no more output will come");
});
// ── SDK call-id diagnostics ─────────────────────────────
test("toolCallId: the SDK's call id is recorded on the pending call", () => {
const turn = new RelayTurn();
const tools = buildCustomTools([tool("read_file")], turn);
tools.read_file.execute({}, { toolCallId: "call_sdk-abc123" });
const rec = [...turn.pending.values()][0];
assert.equal(rec.toolCallId, "call_sdk-abc123", "the SDK-side id rides along for diagnostics");
assert.equal(rec.name, "read_file");
});
await run();
if (failed.length) {
for (const { name, error } of failed) console.error(`FAIL ${name}: ${error.message}`);
process.exit(1);
}
console.log(`tool relay: all passed (${passed.length} tests)`);
You can’t perform that action at this time.