feat(agents): AI Gateway dynamic routes in createAI (#2572) · cloudflare/agents@b7c6ab9 · GitHub
Skip to content

Commit b7c6ab9

Browse files
mattzcareymattzcarey
authored
feat(agents): AI Gateway dynamic routes in createAI (#2572)
* feat(agents): AI Gateway dynamic routes in createAI * docs(agents): dynamic routes in the createAI canon, precise id errors
1 parent 131af4b commit b7c6ab9

15 files changed

Lines changed: 386 additions & 30 deletions

File tree

‎.changeset/createai-dynamic-routes.md‎

Lines changed: 5 additions & 0 deletions

‎docs/agents/models-pi-ai.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ ai("@cf/zai-org/glm-4.7-flash");
8686
// 2. pi-ai's Cloudflare AI Gateway catalog — a string, resolved by pi, not us.
8787
ai("anthropic/claude-opus-4.8");
8888

89+
// 2b. An AI Gateway dynamic route — the route on the gateway picks the model.
90+
ai("dynamic/support");
91+
8992
// 3. Any pi-ai model object, from any registry you imported.
9093
import { anthropicProvider } from "@earendil-works/pi-ai/providers/anthropic";
9194
const claude = anthropicProvider()
@@ -100,6 +103,10 @@ cost, context window, max tokens. That metadata is why Claude Opus gets
100103
so, and this package does not second-guess it. Form 3 never mutates the object you
101104
passed; the options ride on a copy.
102105

106+
A `dynamic/<route>` id runs on `env.AI.run` with the Workers AI wire, under the gateway the
107+
route was created on. It answers in the chat-completions shape whichever model the route
108+
picked, and takes the same metadata defaults and overrides as a Workers AI id.
109+
103110
## Running a model
104111

105112
The instance carries pi-ai's four entry points, so nothing global is registered:

‎docs/agents/models.md‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@ we hand its request to AI Gateway and hand the answer straight back to it.
2020

2121
## The canon, in five lines
2222

23-
1. **`ai(string)` is a Workers AI id.** `@cf/<author>/<model>`, autocompleted
24-
from `@cloudflare/workers-types`. Any other string is a `TypeError` that
25-
tells you which provider package to install.
23+
1. **`ai(string)` is Cloudflare's.** A Workers AI id, `@cf/<author>/<model>`,
24+
autocompleted from `@cloudflare/workers-types`, or a
25+
[dynamic route](#dynamic-routes) on your gateway, `dynamic/<route>`. Any
26+
other string is a `TypeError` that tells you which provider package to
27+
install.
2628
2. **`ai(model)` is any AI SDK v4 model**, routed through AI Gateway. Its
2729
provider builds the request, parses the response, and handles its own
2830
errors; only the transport is ours.
@@ -235,6 +237,27 @@ a model on your key with one on unified billing.
235237
Only the gateway's _own_ failures — no such gateway, an unpaid account — become
236238
a `CloudflareAIError`. A vendor error stays the vendor's.
237239

240+
## Dynamic routes
241+
242+
An [AI Gateway dynamic route](https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/)
243+
is called by name, as `dynamic/<route>`. The route configured on the gateway
244+
picks the model (conditions, percentages, rate and budget limits, its own
245+
fallbacks), and the call goes through `env.AI.run` like a Workers AI model:
246+
247+
```ts
248+
const ai = createAI({ binding: env.AI, id: "prod" });
249+
250+
await generateText({ model: ai("dynamic/support"), prompt: "Hi" });
251+
ai("@cf/zai-org/glm-4.7-flash", { fallback: ["dynamic/support"] });
252+
```
253+
254+
The route belongs to a gateway, so set the gateway `id` it was created on. It
255+
answers in the OpenAI chat-completions shape whatever model served it, so the
256+
request is built as strict OpenAI: `chatTemplateKwargs` is dropped with a
257+
warning, because the route may not land on Workers AI. A dynamic route is a
258+
language model only, and the modality methods refuse it, including as a
259+
fallback leg.
260+
238261
## Other modalities
239262

240263
Images, speech, transcription and reranking come off the same provider, the

‎packages/agents/src/models/ai-sdk/errors.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { APICallError } from "@ai-sdk/provider";
2+
import { isDynamicRouteId, isRunPathLanguageModelId } from "../core/catalog";
23
import {
34
type CloudflareAIAttempt,
45
type CloudflareAIErrorCode,
@@ -30,14 +31,21 @@ const VENDOR_PACKAGES: Record<string, { package: string; call: string }> = {
3031
* this provider knows; a third-party model is a model object built by that
3132
* vendor's own provider package.
3233
*/
33-
export function notAWorkersAIId(modelId: string): TypeError {
34+
export function notAWorkersAIId(
35+
modelId: string,
36+
accepts: "workers-ai" | "workers-ai-or-route" = "workers-ai"
37+
): TypeError {
3438
const vendor = VENDOR_PACKAGES[modelId.split("/")[0] ?? ""];
3539
const advice =
3640
vendor === undefined
3741
? 'install its AI SDK provider package and pass the model object, e.g. ai(anthropic("claude-opus-4-8"))'
3842
: `install ${vendor.package} and pass the model object: ai(${vendor.call})`;
3943
return new TypeError(
40-
`"${modelId}" is not a Workers AI model id. Workers AI ids start with "@cf/", and they are the only ids this provider takes: a third-party model goes through its own provider and is only routed by AI Gateway — ${advice}.`
44+
`"${modelId}" is not a Workers AI model id. Workers AI ids start with "@cf/"${
45+
accepts === "workers-ai-or-route"
46+
? ' (and AI Gateway dynamic routes with "dynamic/")'
47+
: ""
48+
}, and they are the only ids this provider takes: a third-party model goes through its own provider and is only routed by AI Gateway — ${advice}.`
4149
);
4250
}
4351

@@ -48,10 +56,27 @@ export function notAWorkersAIId(modelId: string): TypeError {
4856
* the line that supplied the id rather than at the request it would have made.
4957
*/
5058
export function requireWorkersAIId<Id extends string>(modelId: Id): Id {
59+
if (isDynamicRouteId(modelId)) {
60+
throw new TypeError(
61+
`"${modelId}" is an AI Gateway dynamic route, and a dynamic route is a chat model: it answers in the chat-completions shape. Use it as ai("${modelId}"); the embedding, image, speech, transcription and reranking methods take "@cf/" ids.`
62+
);
63+
}
5164
if (!modelId.startsWith("@cf/")) throw notAWorkersAIId(modelId);
5265
return modelId;
5366
}
5467

68+
/**
69+
* The gate for ids that name a language model: a Workers AI id, or an AI
70+
* Gateway dynamic route. Both run on `env.AI.run`; any other string is a
71+
* vendor id, which is never a string here.
72+
*/
73+
export function requireLanguageModelId<Id extends string>(modelId: Id): Id {
74+
if (!isRunPathLanguageModelId(modelId)) {
75+
throw notAWorkersAIId(modelId, "workers-ai-or-route");
76+
}
77+
return modelId;
78+
}
79+
5580
/**
5681
* Error thrown for every failed Cloudflare AI call, whether it came back as a
5782
* non-2xx `Response` or as a thrown Workers AI binding error.

‎packages/agents/src/models/ai-sdk/index.ts‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
* One rule decides everything here: **Workers AI is the one catalog this
55
* package keeps up with; every other vendor brings its own provider.**
66
*
7-
* 1. **`ai("@cf/…")`** is a Workers AI model — ours, wire format and all.
7+
* 1. **`ai("@cf/…")`** is a Workers AI model — ours, wire format and all —
8+
* and **`ai("dynamic/<route>")`** is a dynamic route on your gateway.
89
* 2. **`ai(anthropic("claude-opus-4-8"))`** is the vendor's model, routed
910
* through AI Gateway. Their code builds and parses the request; we only
1011
* carry it. No vendor ids, wire formats or model metadata live here, and
@@ -45,9 +46,10 @@ import type {
4546
CloudflareRerankingModelId,
4647
CloudflareSpeechModelId,
4748
CloudflareTranscriptionModelId,
49+
DynamicRouteId,
4850
WorkersAIModelId
4951
} from "../core/catalog";
50-
import { requireWorkersAIId } from "./errors";
52+
import { requireLanguageModelId } from "./errors";
5153
import { CloudflareEmbeddingModel } from "./models/embedding";
5254
import { GatewayLanguageModel, routed } from "./models/gateway";
5355
import { CloudflareImageModel } from "./models/image";
@@ -77,6 +79,7 @@ export type {
7779
CloudflareRerankingModelId,
7880
CloudflareSpeechModelId,
7981
CloudflareTranscriptionModelId,
82+
DynamicRouteId,
8083
KnownImageModelId,
8184
KnownSpeechModelId,
8285
WorkersAIModelId
@@ -121,6 +124,11 @@ function isModel(value: unknown): value is LanguageModelV4 {
121124
export interface AI {
122125
/** A Workers AI model. Third-party ids are not ids here — pass the model. */
123126
(modelId: WorkersAIModelId, options?: ModelOptions): LanguageModelV4;
127+
/**
128+
* An AI Gateway dynamic route, `dynamic/<route>`: the route configured on
129+
* the gateway picks the model, and it answers like a chat model.
130+
*/
131+
(route: DynamicRouteId, options?: ModelOptions): LanguageModelV4;
124132
/** Any AI SDK v4 language model, routed through AI Gateway. */
125133
(model: LanguageModelV4, options?: GatewayModelOptions): LanguageModelV4;
126134
/** The AI SDK provider specification this implements. */
@@ -130,6 +138,7 @@ export interface AI {
130138
modelId: WorkersAIModelId,
131139
options?: ModelOptions
132140
): LanguageModelV4;
141+
languageModel(route: DynamicRouteId, options?: ModelOptions): LanguageModelV4;
133142
languageModel(
134143
model: LanguageModelV4,
135144
options?: GatewayModelOptions
@@ -254,14 +263,14 @@ export function createAI(settings: AISettings): AI {
254263
typeof leg === "string"
255264
? // The same gate the primary passes, at the line that wrote the
256265
// leg: a vendor model is never a string here, in a chain either.
257-
requireWorkersAIId(leg)
266+
requireLanguageModelId(leg)
258267
: languageModel(leg, inherited)
259268
)
260269
};
261270
};
262271

263272
function languageModel(
264-
model: WorkersAIModelId | LanguageModelV4,
273+
model: WorkersAIModelId | DynamicRouteId | LanguageModelV4,
265274
options?: GatewayModelOptions
266275
): LanguageModelV4 {
267276
const resolved = normalize(options);
@@ -283,7 +292,7 @@ export function createAI(settings: AISettings): AI {
283292
return new GatewayLanguageModel(model, resolved, config);
284293
}
285294
return new CloudflareLanguageModel(
286-
requireWorkersAIId(model),
295+
requireLanguageModelId(model),
287296
resolved,
288297
config
289298
);
@@ -316,7 +325,7 @@ export function createAI(settings: AISettings): AI {
316325
): RerankingModelV4 => new CloudflareRerankingModel(modelId, options, config);
317326

318327
const provider = (
319-
model: WorkersAIModelId | LanguageModelV4,
328+
model: WorkersAIModelId | DynamicRouteId | LanguageModelV4,
320329
options?: GatewayModelOptions
321330
): LanguageModelV4 => languageModel(model, options);
322331

‎packages/agents/src/models/ai-sdk/models/embedding.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import {
55
type EmbeddingModelV4Result
66
} from "@ai-sdk/provider";
77
import {
8-
parseCallOptions,
8+
parseModalityCallOptions,
99
resolveOptions,
1010
type ModelOptions
1111
} from "../settings";
@@ -75,7 +75,7 @@ export class CloudflareEmbeddingModel implements EmbeddingModelV4 {
7575
const resolved = resolveOptions(
7676
this.#config.providerGateway,
7777
this.#options,
78-
parseCallOptions(options.providerOptions)
78+
parseModalityCallOptions(options.providerOptions)
7979
);
8080
const input = { text: options.values };
8181
const transport: Transport = this.#config.transport;

‎packages/agents/src/models/ai-sdk/models/gateway.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import {
2626
type ResolvedGatewayProvider,
2727
requireGatewayProvider
2828
} from "../../core/gateway-providers";
29-
import { requireWorkersAIId } from "../errors";
29+
import { requireLanguageModelId } from "../errors";
3030
import {
3131
fallbackLegs,
3232
rememberOptions,
@@ -579,9 +579,9 @@ export class GatewayLanguageModel implements LanguageModelV4 {
579579
#workersAI(modelId: string): LanguageModelV4 {
580580
return new CloudflareLanguageModel(
581581
// Belt and braces: `ai()` and the per-call bag both gate their legs, so
582-
// a non-`@cf/` id can only arrive from a caller that built the model
582+
// any other id can only arrive from a caller that built the model
583583
// some other way — and it must not reach `env.AI.run` either.
584-
requireWorkersAIId(modelId),
584+
requireLanguageModelId(modelId),
585585
{ ...this.#options, fallback: undefined },
586586
this.#config
587587
);

‎packages/agents/src/models/ai-sdk/models/language.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import type {
99
SharedV4ProviderMetadata,
1010
SharedV4Warning
1111
} from "@ai-sdk/provider";
12-
import { CloudflareAIError, requireWorkersAIId } from "../errors";
12+
import { CloudflareAIError, requireLanguageModelId } from "../errors";
1313
import {
1414
fallbackLegs,
1515
rememberOptions,
@@ -473,7 +473,7 @@ export class CloudflareLanguageModel implements LanguageModelV4 {
473473
// legs, and a vendor id must never reach `env.AI.run`. Checked
474474
// as the chain is built, so the throw is not swallowed as one
475475
// more failed leg.
476-
{ model: requireWorkersAIId(leg), run: () => runSelf(leg) }
476+
{ model: requireLanguageModelId(leg), run: () => runSelf(leg) }
477477
: { model: leg.modelId, run: () => runLeg(leg) }
478478
)
479479
]);

‎packages/agents/src/models/ai-sdk/models/modality.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import type {
1212
} from "@ai-sdk/provider";
1313
import { CloudflareAIError, requireWorkersAIId } from "../errors";
1414
import {
15-
parseCallOptions,
15+
parseModalityCallOptions,
1616
resolveOptions,
1717
type ModelOptions
1818
} from "../settings";
@@ -142,7 +142,7 @@ export abstract class CloudflareModalityModel {
142142
const resolved = resolveOptions(
143143
this.#config.providerGateway,
144144
this.#options,
145-
parseCallOptions(call.providerOptions)
145+
parseModalityCallOptions(call.providerOptions)
146146
);
147147
const transport = this.#config.transport;
148148
return withFallback(

‎packages/agents/src/models/ai-sdk/settings.ts‎

Lines changed: 22 additions & 5 deletions

0 commit comments

Comments
 (0)