wip

examples/agent-server/.env.example

@@ -0,0 +1,17 @@
+ANTHROPIC_API_KEY=""
+AWS_ACCESS_KEY_ID=""
+AWS_SECRET_ACCESS_KEY=""
+AWS_REGION=""
+AZURE_API_KEY=""
+AZURE_RESOURCE_NAME=""
+COHERE_API_KEY=""
+FIREWORKS_API_KEY=""
+GOOGLE_GENERATIVE_AI_API_KEY=""
+GOOGLE_VERTEX_LOCATION=""
+GOOGLE_VERTEX_PROJECT=""
+GROQ_API_KEY=""
+MISTRAL_API_KEY=""
+OPENAI_API_KEY=""
+PERPLEXITY_API_KEY=""
+TOGETHER_AI_API_KEY=""
+XAI_API_KEY=""

examples/agent-server/.gitignore

@@ -0,0 +1,2 @@
+.agents
+.data

examples/agent-server/README.md

@@ -0,0 +1,9 @@
+## Agent Server Examples
+
+### Snippets
+
+```sh
+curl -X POST http://127.0.0.1:3000/agent/helloworld-completion/start \
+  -H "Content-Type: application/json" \
+  -d '{"prompt": "Hello, world!"}'
+```

examples/agent-server/agents/helloworld-completion/agent.ts

@@ -0,0 +1,24 @@
+import type { Agent } from '@ai-sdk/agent-server';
+import { z } from 'zod';
+
+// agent context is available to all states.
+// it can be used to pass data between states.
+export const contextSchema = z.object({ prompt: z.string() });
+export type Context = z.infer<typeof contextSchema>;
+
+export default {
+  // Explicit start method that receives the request, so that the user
+  // can perform any data validation, transformation, and loading
+  // that is required for their agent.
+  // The goal is to provide the initial context for the agent run.
+  async start({ request }) {
+    return {
+      context: contextSchema.parse(await request.json()),
+      initialState: 'main',
+    };
+  },
+
+  // Optional headers. Streams can be anything JSON-serializable,
+  // so we enable agents to set the headers that are needed.
+  headers: { 'X-Vercel-AI-Data-Stream': 'v1' },
+} satisfies Agent<Context>;

examples/agent-server/agents/helloworld-completion/states/main.ts

@@ -0,0 +1,19 @@
+import { StreamState } from '@ai-sdk/agent-server';
+import { openai } from '@ai-sdk/openai';
+import { streamText } from 'ai';
+import { Context } from '../agent';
+
+export default {
+  type: 'stream', // state type, in the future there will be other types
+  async execute({ context }) {
+    const result = streamText({
+      model: openai('gpt-4o'),
+      prompt: context.prompt,
+    });
+
+    return {
+      stream: result.toAgentStream(),
+      nextState: 'END',
+    };
+  },
+} satisfies StreamState<Context, string>;

examples/agent-server/agents/routing-example/agent.ts

@@ -0,0 +1,18 @@
+import type { Agent } from '@ai-sdk/agent-server';
+import { z } from 'zod';
+
+export const contextSchema = z.object({
+  prompt: z.string(),
+});
+
+export type Context = z.infer<typeof contextSchema>;
+
+export default {
+  async start({ request }) {
+    return {
+      context: contextSchema.parse(await request.json()),
+      initialState: 'main',
+    };
+  },
+  headers: { 'X-Vercel-AI-Data-Stream': 'v1' },
+} satisfies Agent<Context>;

examples/agent-server/agents/routing-example/states/chat.ts

@@ -0,0 +1,20 @@
+import { StreamState } from '@ai-sdk/agent-server';
+import { openai } from '@ai-sdk/openai';
+import { streamText } from 'ai';
+import { Context } from '../agent';
+
+export default {
+  type: 'stream',
+  async execute({ context }) {
+    const result = streamText({
+      model: openai('gpt-4o'),
+      system: 'You are a friendly chatbot. Respond briefly and concisely.',
+      prompt: context.prompt,
+    });
+
+    return {
+      stream: result.toAgentStream(),
+      nextState: 'END',
+    };
+  },
+} satisfies StreamState<Context, string>;

examples/agent-server/agents/routing-example/states/router.ts

@@ -0,0 +1,20 @@
+import { StreamState } from '@ai-sdk/agent-server';
+import { StreamData } from 'ai';
+import { Context } from '../agent';
+
+export default {
+  type: 'stream',
+  async execute({ context }) {
+    const streamData = new StreamData();
+    streamData.append({ status: 'selecting route' });
+    streamData.close();
+
+    return {
+      // will be simplified to streamData.toAgentStream() in the future
+      stream: streamData.stream.pipeThrough(new TextDecoderStream()),
+      nextState: context.prompt.toLowerCase().includes('write')
+        ? 'write-blog'
+        : 'chat',
+    };
+  },
+} satisfies StreamState<Context, string>;

examples/agent-server/agents/routing-example/states/write-blog.ts

@@ -0,0 +1,23 @@
+import { StreamState } from '@ai-sdk/agent-server';
+import { openai } from '@ai-sdk/openai';
+import { streamText } from 'ai';
+import { Context } from '../agent';
+
+export default {
+  type: 'stream',
+  async execute({ context }) {
+    const result = streamText({
+      model: openai('gpt-4o'),
+      system:
+        'You are an outstanding writer. ' +
+        'Write a blog post. ' +
+        'The blog post MUST BE at least 4 paragraphs long. ',
+      prompt: context.prompt,
+    });
+
+    return {
+      stream: result.toAgentStream(),
+      nextState: 'END',
+    };
+  },
+} satisfies StreamState<Context, string>;

examples/agent-server/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "agent-server-examples",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "start": "pnpm agent-server --port 3000 --host 127.0.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "22.9.1"
+  },
+  "dependencies": {
+    "@ai-sdk/agent-server": "0.0.0",
+    "@ai-sdk/openai": "1.0.4",
+    "ai": "4.0.3",
+    "zod": "3.23.8"
+  }
+}

examples/agent-server/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "declaration": true,
+    "sourceMap": true,
+    "target": "es2022",
+    "lib": ["es2022", "dom"],
+    "module": "esnext",
+    "types": ["node"],
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "rootDir": "./agents",
+    "outDir": "./.agents",
+    "skipLibCheck": true
+  },
+  "include": ["agents/**/*.ts"]
+}

packages/agent-server/README.md

@@ -0,0 +1,3 @@
+# AI SDK Server
+
+Experimental prototype. DO NOT USE.

packages/agent-server/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "@ai-sdk/agent-server",
+  "version": "0.0.0",
+  "license": "Apache-2.0",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist/**/*",
+    "CHANGELOG.md"
+  ],
+  "bin": {
+    "agent-server": "./dist/agent-server.js"
+  },
+  "scripts": {
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "clean": "rm -rf dist",
+    "lint": "eslint \"./**/*.ts*\"",
+    "type-check": "tsc --noEmit",
+    "prettier-check": "prettier --check \"./**/*.ts*\"",
+    "start": "pnpm tsx src/agent-server.ts --port 3000 --host 127.0.0.1",
+    "test": "pnpm test:node && pnpm test:edge",
+    "test:edge": "vitest --config vitest.edge.config.js --run --passWithNoTests",
+    "test:node": "vitest --config vitest.node.config.js --run --passWithNoTests"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@ai-sdk/provider": "1.0.1",
+    "@ai-sdk/provider-utils": "2.0.2",
+    "@hono/node-server": "1.13.7",
+    "commander": "12.1.0",
+    "hono": "4.6.11",
+    "pino": "9.5.0",
+    "zod": "3.23.8",
+    "dotenv": "16.4.5"
+  },
+  "devDependencies": {
+    "@types/node": "^18",
+    "@vercel/ai-tsconfig": "workspace:*",
+    "tsup": "^8",
+    "tsx": "4.19.2",
+    "typescript": "5.6.3"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "homepage": "https://sdk.vercel.ai/docs",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vercel/ai.git"
+  },
+  "bugs": {
+    "url": "https://github.com/vercel/ai/issues"
+  },
+  "keywords": [
+    "ai"
+  ]
+}

packages/agent-server/src/agent-server.ts

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+
+import { serve } from '@hono/node-server';
+import { Option } from 'commander';
+import { Hono } from 'hono';
+import { logger as honoLogger } from 'hono/logger';
+import { requestId } from 'hono/request-id';
+import * as path from 'node:path';
+import zod from 'zod';
+import { DataStore } from './data-store';
+import { ModuleLoader } from './module-loader';
+import { RunManager } from './run-manager';
+import { JobQueue } from './util/job-queue';
+import { startService } from './util/start-service';
+import { createWorker } from './worker';
+import { StreamManager } from './stream-manager';
+import { stream } from 'hono/streaming';
+import 'dotenv/config';
+
+startService({
+  name: '@ai-sdk/server',
+  configurationOptions: [
+    new Option('--port <number>', 'port number')
+      .env('PORT')
+      .argParser(value => +value)
+      .makeOptionMandatory(),
+    new Option('--host <string>', 'host name')
+      .env('HOST')
+      .makeOptionMandatory(),
+  ],
+  configurationSchema: zod.object({
+    host: zod.string(),
+    port: zod.number(),
+  }),
+  async initialize({ host, port }, logger) {
+    const moduleLoader = new ModuleLoader({
+      modulePath: path.join(process.cwd(), '.agents'),
+    });
+    const jobs = new JobQueue<{ runId: string }>();
+    const submitJob = jobs.push.bind(jobs);
+    const dataStore = new DataStore({
+      dataPath: path.join(process.cwd(), '.data'),
+    });
+    const streamManager = new StreamManager();
+    const runManager = new RunManager({
+      dataStore,
+      moduleLoader,
+      submitJob,
+      streamManager,
+      logger,
+    });
+
+    // setup workers
+    // the workers run in the same process in this prototype,
+    // so if they perform CPU-bound tasks, they will block
+    // TODO multiple workers
+    jobs.startWorker(
+      createWorker({
+        dataStore,
+        moduleLoader,
+        streamManager,
+        submitJob,
+        logger,
+      }),
+    );
+
+    // Hono setup
+    const app = new Hono();
+    app.use(requestId());
+    app.use(
+      honoLogger((message, ...rest) => {
+        logger.info(message, ...rest);
+      }),
+    );
+
+    // routes setup
+    app.post('/agent/:agent/start', async c => {
+      const { runId, headers } = await runManager.startAgent({
+        agent: c.req.param('agent'),
+        request: c.req.raw,
+      });
+
+      const runStream = streamManager.getStream(runId);
+
+      // headers
+      c.header('X-Agent-Run-Id', runId);
+      Object.entries(headers ?? {}).forEach(([key, value]) => {
+        c.header(key, value);
+      });
+
+      return stream(c, stream =>
+        stream.pipe(runStream.pipeThrough(new TextEncoderStream())),
+      );
+    });
+
+    const server = serve({ fetch: app.fetch, hostname: host, port });
+
+    return {
+      async shutdown() {
+        server.close(); // wait for requests to be finished
+      },
+    };
+  },
+});